@endo/compartment-mapper 1.1.5 → 1.2.0

package/src/archive.js

@@ -1,425 +1,91 @@
-// @ts-check
-/* eslint no-shadow: 0 */
-
-/** @import {ArchiveOptions} from './types.js' */
-/** @import {ArchiveWriter} from './types.js' */
-/** @import {CompartmentDescriptor} from './types.js' */
-/** @import {CompartmentMapDescriptor} from './types.js' */
-/** @import {ModuleDescriptor} from './types.js' */
-/** @import {ParserImplementation} from './types.js' */
-/** @import {ReadFn} from './types.js' */
-/** @import {CaptureSourceLocationHook} from './types.js' */
-/** @import {ReadPowers} from './types.js' */
-/** @import {HashPowers} from './types.js' */
-/** @import {Sources} from './types.js' */
-/** @import {WriteFn} from './types.js' */
-
-import { writeZip } from '@endo/zip';
-import { resolve } from './node-module-specifier.js';
-import { compartmentMapForNodeModules } from './node-modules.js';
-import { search } from './search.js';
-import { link } from './link.js';
-import {
-  exitModuleImportHookMaker,
-  makeImportHookMaker,
-} from './import-hook.js';
-import parserJson from './parse-json.js';
-import parserText from './parse-text.js';
-import parserBytes from './parse-bytes.js';
-import parserArchiveCjs from './parse-archive-cjs.js';
-import parserArchiveMjs from './parse-archive-mjs.js';
-import { parseLocatedJson } from './json.js';
-import { unpackReadPowers } from './powers.js';
-import {
-  assertCompartmentMap,
-  stringCompare,
-  pathCompare,
-} from './compartment-map.js';
-import { detectAttenuators } from './policy.js';
-
-const textEncoder = new TextEncoder();
-
-/** @type {Record<string, ParserImplementation>} */
-const parserForLanguage = {
-  mjs: parserArchiveMjs,
-  'pre-mjs-json': parserArchiveMjs,
-  cjs: parserArchiveCjs,
-  'pre-cjs-json': parserArchiveCjs,
-  json: parserJson,
-  text: parserText,
-  bytes: parserBytes,
-};
-
-/**
- * @param {string} rel - a relative URL
- * @param {string} abs - a fully qualified URL
- * @returns {string}
- */
-const resolveLocation = (rel, abs) => new URL(rel, abs).toString();
-
-const { keys, entries, fromEntries } = Object;
-
-/**
- * We attempt to produce compartment maps that are consistent regardless of
- * whether the packages were originally laid out on disk for development or
- * production, and other trivia like the fully qualified path of a specific
- * installation.
+/* Provides mechanisms for creating archives (zip files with a
+ * `compartmeent-map.json` and a file for every static dependency of an entry
+ * module).
  *
- * Naming compartments for the self-ascribed name and version of each Node.js
- * package is insufficient because they are not guaranteed to be unique.
- * Dependencies do not necessarilly come from the npm registry and may be
- * for example derived from fully qualified URL's or Github org and project
- * names.
- * Package managers are also not required to fully deduplicate the hard
- * copy of each package even when they are identical resources.
- * Duplication is undesirable, but we elect to defer that problem to solutions
- * in the package managers, as the alternative would be to consistently hash
- * the original sources of the packages themselves, which may not even be
- * available much less pristine for us.
+ * Uses the conventions of Node.js package managers and the `node_modules`.
+ * Archives will contain a copy for every physical copy of a package on disk,
+ * such that importing will construct a separate instance for each.
  *
- * So, instead, we use the lexically least path of dependency names, delimited
- * by hashes.
- * The compartment maps generated by the ./node-modules.js tooling pre-compute
- * these traces for our use here.
- * We sort the compartments lexically on their self-ascribed name and version,
- * and use the lexically least dependency name path as a tie-breaker.
- * The dependency path is logical and orthogonal to the package manager's
- * actual installation location, so should be orthogonal to the vagaries of the
- * package manager's deduplication algorithm.
+ * These archives will contain pre-compiled ESM and CJS and this module
+ * entrains a dependency on the core of Babel.
  *
- * @param {Record<string, CompartmentDescriptor>} compartments
- * @returns {Record<string, string>} map from old to new compartment names.
- */
-const renameCompartments = compartments => {
-  /** @type {Record<string, string>} */
-  const compartmentRenames = Object.create(null);
-  let index = 0;
-  let prev = '';
-
-  // The sort below combines two comparators to avoid depending on sort
-  // stability, which became standard as recently as 2019.
-  // If that date seems quaint, please accept my regards from the distant past.
-  // We are very proud of you.
-  const compartmentsByPath = Object.entries(compartments)
-    .map(([name, compartment]) => ({
-      name,
-      path: compartment.path,
-      label: compartment.label,
-    }))
-    .sort((a, b) => {
-      if (a.label === b.label) {
-        assert(a.path !== undefined && b.path !== undefined);
-        return pathCompare(a.path, b.path);
-      }
-      return stringCompare(a.label, b.label);
-    });
-
-  for (const { name, label } of compartmentsByPath) {
-    if (label === prev) {
-      compartmentRenames[name] = `${label}-n${index}`;
-      index += 1;
-    } else {
-      compartmentRenames[name] = label;
-      prev = label;
-      index = 1;
-    }
-  }
-  return compartmentRenames;
-};
-
-/**
- * @param {Record<string, CompartmentDescriptor>} compartments
- * @param {Sources} sources
- * @param {Record<string, string>} compartmentRenames
+ * See `archive-lite.js` for a variation that does not depend on Babel
+ * for cases like XS native Compartments where pre-compilation is not
+ * necessary or where the dependency on Babel can be dererred to runtime.
  */
-const translateCompartmentMap = (compartments, sources, compartmentRenames) => {
-  const result = Object.create(null);
-  for (const compartmentName of keys(compartmentRenames)) {
-    const compartment = compartments[compartmentName];
-    const { name, label, retained, policy } = compartment;
-    if (retained) {
-      // rename module compartments
-      /** @type {Record<string, ModuleDescriptor>} */
-      const modules = Object.create(null);
-      const compartmentModules = compartment.modules;
-      if (compartment.modules) {
-        for (const name of keys(compartmentModules).sort()) {
-          const module = compartmentModules[name];
-          if (module.compartment !== undefined) {
-            modules[name] = {
-              ...module,
-              compartment: compartmentRenames[module.compartment],
-            };
-          } else {
-            modules[name] = module;
-          }
-        }
-      }
-
-      // integrate sources into modules
-      const compartmentSources = sources[compartmentName];
-      if (compartmentSources) {
-        for (const name of keys(compartmentSources).sort()) {
-          const source = compartmentSources[name];
-          const { location, parser, exit, sha512, deferredError } = source;
-          if (location !== undefined) {
-            modules[name] = {
-              location,
-              parser,
-              sha512,
-            };
-          } else if (exit !== undefined) {
-            modules[name] = {
-              exit,
-            };
-          } else if (deferredError !== undefined) {
-            modules[name] = {
-              deferredError,
-            };
-          }
-        }
-      }
-
-      result[compartmentRenames[compartmentName]] = {
-        name,
-        label,
-        location: compartmentRenames[compartmentName],
-        modules,
-        policy,
-        // `scopes`, `types`, and `parsers` are not necessary since every
-        // loadable module is captured in `modules`.
-      };
-    }
-  }
-
-  return result;
-};
+// @ts-check
 
-/**
- * @param {Sources} sources
- * @param {Record<string, string>} compartmentRenames
- * @returns {Sources}
- */
-const renameSources = (sources, compartmentRenames) => {
-  return fromEntries(
-    entries(sources).map(([name, compartmentSources]) => [
-      compartmentRenames[name],
-      compartmentSources,
-    ]),
-  );
-};
+import { defaultParserForLanguage } from './archive-parsers.js';
+import { mapNodeModules } from './node-modules.js';
+import {
+  makeAndHashArchiveFromMap,
+  makeArchiveFromMap,
+  mapFromMap,
+  hashFromMap,
+  writeArchiveFromMap,
+} from './archive-lite.js';
 
-/**
- * @param {ArchiveWriter} archive
- * @param {Sources} sources
- */
-const addSourcesToArchive = async (archive, sources) => {
-  for (const compartment of keys(sources).sort()) {
-    const modules = sources[compartment];
-    const compartmentLocation = resolveLocation(`${compartment}/`, 'file:///');
-    for (const specifier of keys(modules).sort()) {
-      const { bytes, location } = modules[specifier];
-      if (location !== undefined) {
-        const moduleLocation = resolveLocation(location, compartmentLocation);
-        const path = new URL(moduleLocation).pathname.slice(1); // elide initial "/"
-        if (bytes !== undefined) {
-          // eslint-disable-next-line no-await-in-loop
-          await archive.write(path, bytes);
-        }
-      }
-    }
-  }
-};
+const { assign, create, freeze } = Object;
 
-/**
- * @param {Sources} sources
- * @param {CaptureSourceLocationHook} captureSourceLocation
- */
-const captureSourceLocations = async (sources, captureSourceLocation) => {
-  for (const compartmentName of keys(sources).sort()) {
-    const modules = sources[compartmentName];
-    for (const moduleSpecifier of keys(modules).sort()) {
-      const { sourceLocation } = modules[moduleSpecifier];
-      if (sourceLocation !== undefined) {
-        captureSourceLocation(compartmentName, moduleSpecifier, sourceLocation);
-      }
-    }
-  }
-};
+/** @import {ArchiveOptions} from './types.js' */
+/** @import {ReadFn} from './types.js' */
+/** @import {ReadPowers} from './types.js' */
+/** @import {HashPowers} from './types.js' */
+/** @import {WriteFn} from './types.js' */
 
 /**
- * @param {CompartmentMapDescriptor} compartmentMap
- * @param {Sources} sources
- * @returns {{archiveCompartmentMap: CompartmentMapDescriptor, archiveSources: Sources}}
+ * Add the default parserForLanguage option.
+ * @param {ArchiveOptions} [options]
+ * @returns {ArchiveOptions}
  */
-export const makeArchiveCompartmentMap = (compartmentMap, sources) => {
-  const {
-    compartments,
-    entry: { compartment: entryCompartmentName, module: entryModuleSpecifier },
-  } = compartmentMap;
-
-  const compartmentRenames = renameCompartments(compartments);
-  const archiveCompartments = translateCompartmentMap(
-    compartments,
-    sources,
-    compartmentRenames,
+const assignParserForLanguage = (options = {}) => {
+  const { parserForLanguage: parserForLanguageOption, ...rest } = options;
+  const parserForLanguage = freeze(
+    assign(create(null), defaultParserForLanguage, parserForLanguageOption),
   );
-  const archiveEntryCompartmentName = compartmentRenames[entryCompartmentName];
-  const archiveSources = renameSources(sources, compartmentRenames);
-
-  const archiveCompartmentMap = {
-    tags: [],
-    entry: {
-      compartment: archiveEntryCompartmentName,
-      module: entryModuleSpecifier,
-    },
-    compartments: archiveCompartments,
-  };
-
-  // Cross-check:
-  // We assert that we have constructed a valid compartment map, not because it
-  // might not be, but to ensure that the assertCompartmentMap function can
-  // accept all valid compartment maps.
-  assertCompartmentMap(archiveCompartmentMap);
-
-  return { archiveCompartmentMap, archiveSources };
+  return { ...rest, parserForLanguage };
 };
 
 /**
  * @param {ReadFn | ReadPowers} powers
  * @param {string} moduleLocation
  * @param {ArchiveOptions} [options]
- * @returns {Promise<{sources: Sources, compartmentMapBytes: Uint8Array, sha512?: string}>}
+ * @returns {Promise<{bytes: Uint8Array, sha512?: string}>}
  */
-const digestLocation = async (powers, moduleLocation, options) => {
-  const {
-    moduleTransforms,
-    modules: exitModules = {},
-    dev = false,
-    tags = new Set(),
-    captureSourceLocation = undefined,
-    searchSuffixes = undefined,
-    commonDependencies = undefined,
-    importHook: exitModuleImportHook = undefined,
-    policy = undefined,
-    sourceMapHook = undefined,
-  } = options || {};
-  const { read, computeSha512 } = unpackReadPowers(powers);
-  const {
-    packageLocation,
-    packageDescriptorText,
-    packageDescriptorLocation,
-    moduleSpecifier,
-  } = await search(read, moduleLocation);
-
-  tags.add('endo');
-  tags.add('import');
-  tags.add('default');
-
-  const packageDescriptor = parseLocatedJson(
-    packageDescriptorText,
-    packageDescriptorLocation,
-  );
-  const compartmentMap = await compartmentMapForNodeModules(
+export const makeAndHashArchive = async (
+  powers,
+  moduleLocation,
+  options = {},
+) => {
+  const compartmentMap = await mapNodeModules(powers, moduleLocation, options);
+  return makeAndHashArchiveFromMap(
     powers,
-    packageLocation,
-    tags,
-    packageDescriptor,
-    moduleSpecifier,
-    { dev, commonDependencies, policy },
-  );
-
-  const {
-    compartments,
-    entry: { module: entryModuleSpecifier, compartment: entryCompartmentName },
-  } = compartmentMap;
-
-  /** @type {Sources} */
-  const sources = Object.create(null);
-
-  const consolidatedExitModuleImportHook = exitModuleImportHookMaker({
-    modules: exitModules,
-    exitModuleImportHook,
-  });
-
-  const makeImportHook = makeImportHookMaker(read, packageLocation, {
-    sources,
-    compartmentDescriptors: compartments,
-    archiveOnly: true,
-    computeSha512,
-    searchSuffixes,
-    entryCompartmentName,
-    entryModuleSpecifier,
-    exitModuleImportHook: consolidatedExitModuleImportHook,
-    sourceMapHook,
-  });
-  // Induce importHook to record all the necessary modules to import the given module specifier.
-  const { compartment, attenuatorsCompartment } = link(compartmentMap, {
-    resolve,
-    makeImportHook,
-    moduleTransforms,
-    parserForLanguage,
-    archiveOnly: true,
-  });
-  await compartment.load(entryModuleSpecifier);
-  if (policy) {
-    // retain all attenuators.
-    await Promise.all(
-      detectAttenuators(policy).map(attenuatorSpecifier =>
-        attenuatorsCompartment.load(attenuatorSpecifier),
-      ),
-    );
-  }
-
-  const { archiveCompartmentMap, archiveSources } = makeArchiveCompartmentMap(
     compartmentMap,
-    sources,
+    assignParserForLanguage(options),
   );
-
-  const archiveCompartmentMapText = JSON.stringify(
-    archiveCompartmentMap,
-    null,
-    2,
-  );
-  const archiveCompartmentMapBytes = textEncoder.encode(
-    archiveCompartmentMapText,
-  );
-
-  if (captureSourceLocation !== undefined) {
-    captureSourceLocations(archiveSources, captureSourceLocation);
-  }
-
-  let archiveSha512;
-  if (computeSha512 !== undefined) {
-    archiveSha512 = computeSha512(archiveCompartmentMapBytes);
-  }
-
-  return {
-    compartmentMapBytes: archiveCompartmentMapBytes,
-    sources: archiveSources,
-    sha512: archiveSha512,
-  };
 };
 
 /**
  * @param {ReadFn | ReadPowers} powers
  * @param {string} moduleLocation
  * @param {ArchiveOptions} [options]
- * @returns {Promise<{bytes: Uint8Array, sha512?: string}>}
+ * @returns {Promise<Uint8Array>}
  */
-export const makeAndHashArchive = async (powers, moduleLocation, options) => {
-  const { compartmentMapBytes, sources, sha512 } = await digestLocation(
+export const makeArchive = async (powers, moduleLocation, options = {}) => {
+  const { dev, tags, conditions = tags, commonDependencies, policy } = options;
+
+  const compartmentMap = await mapNodeModules(powers, moduleLocation, {
+    dev,
+    conditions,
+    commonDependencies,
+    policy,
+  });
+
+  return makeArchiveFromMap(
     powers,
-    moduleLocation,
-    options,
+    compartmentMap,
+    assignParserForLanguage(options),
   );
-
-  const archive = writeZip();
-  await archive.write('compartment-map.json', compartmentMapBytes);
-  await addSourcesToArchive(archive, sources);
-  const bytes = await archive.snapshot();
-
-  return { bytes, sha512 };
 };
 
 /**
@@ -428,24 +94,17 @@ export const makeAndHashArchive = async (powers, moduleLocation, options) => {
  * @param {ArchiveOptions} [options]
  * @returns {Promise<Uint8Array>}
  */
-export const makeArchive = async (powers, moduleLocation, options) => {
-  const { bytes } = await makeAndHashArchive(powers, moduleLocation, options);
-  return bytes;
-};
+export const mapLocation = async (powers, moduleLocation, options = {}) => {
+  const { dev, tags, conditions = tags, commonDependencies, policy } = options;
+
+  const compartmentMap = await mapNodeModules(powers, moduleLocation, {
+    dev,
+    conditions,
+    commonDependencies,
+    policy,
+  });
 
-/**
- * @param {ReadFn | ReadPowers} powers
- * @param {string} moduleLocation
- * @param {ArchiveOptions} [options]
- * @returns {Promise<Uint8Array>}
- */
-export const mapLocation = async (powers, moduleLocation, options) => {
-  const { compartmentMapBytes } = await digestLocation(
-    powers,
-    moduleLocation,
-    options,
-  );
-  return compartmentMapBytes;
+  return mapFromMap(powers, compartmentMap, assignParserForLanguage(options));
 };
 
 /**
@@ -454,14 +113,17 @@ export const mapLocation = async (powers, moduleLocation, options) => {
  * @param {ArchiveOptions} [options]
  * @returns {Promise<string>}
  */
-export const hashLocation = async (powers, moduleLocation, options) => {
-  const { compartmentMapBytes } = await digestLocation(
-    powers,
-    moduleLocation,
-    options,
-  );
-  const { computeSha512 } = powers;
-  return computeSha512(compartmentMapBytes);
+export const hashLocation = async (powers, moduleLocation, options = {}) => {
+  const { dev, tags, conditions = tags, commonDependencies, policy } = options;
+
+  const compartmentMap = await mapNodeModules(powers, moduleLocation, {
+    dev,
+    conditions,
+    commonDependencies,
+    policy,
+  });
+
+  return hashFromMap(powers, compartmentMap, assignParserForLanguage(options));
 };
 
 /**
@@ -476,8 +138,20 @@ export const writeArchive = async (
   readPowers,
   archiveLocation,
   moduleLocation,
-  options,
+  options = {},
 ) => {
-  const archiveBytes = await makeArchive(readPowers, moduleLocation, options);
-  await write(archiveLocation, archiveBytes);
+  const { dev, tags, conditions = tags, commonDependencies, policy } = options;
+  const compartmentMap = await mapNodeModules(readPowers, moduleLocation, {
+    dev,
+    conditions,
+    commonDependencies,
+    policy,
+  });
+  return writeArchiveFromMap(
+    write,
+    readPowers,
+    archiveLocation,
+    compartmentMap,
+    assignParserForLanguage(options),
+  );
 };

package/src/bundle-cjs.d.ts

@@ -1,19 +1,8 @@
-declare namespace _default {
-    export { runtime };
-    export function getBundlerKit({ index, indexedImports, record: { cjsFunctor, exports: exportsList }, }: {
-        index: any;
-        indexedImports: any;
-        record: {
-            cjsFunctor: any;
-            exports?: {} | undefined;
-        };
-    }): {
-        getFunctor: () => string;
-        getCells: () => string;
-        getReexportsWiring: () => string;
-        getFunctorCall: () => string;
-    };
-}
+declare const _default: BundlerSupport<CjsModuleSource>;
 export default _default;
-declare const runtime: string;
+export type CjsModuleSource = VirtualModuleSource & {
+    cjsFunctor: string;
+};
+import type { BundlerSupport } from './bundle.js';
+import type { VirtualModuleSource } from 'ses';
 //# sourceMappingURL=bundle-cjs.d.ts.map

package/src/bundle-cjs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bundle-cjs.d.ts","sourceRoot":"","sources":["bundle-cjs.js"],"names":[],"mappings":";;IAuDE;;;;;;;;;;;;MAsBC;;;AA7DH,8BAmCa"}
+{"version":3,"file":"bundle-cjs.d.ts","sourceRoot":"","sources":["bundle-cjs.js"],"names":[],"mappings":"wBA4DW,eAAe,eAAe,CAAC;;8BAvD5B,mBAAmB,GAAG;IAAC,UAAU,EAAE,MAAM,CAAA;CAAC;oCAFtB,aAAa;yCADR,KAAK"}

package/src/bundle-cjs.js

@@ -1,4 +1,10 @@
-// @ts-nocheck
+/* Provides CommonJS support for `bundle.js`. */
+
+/** @import {VirtualModuleSource} from 'ses' */
+/** @import {BundlerSupport} from './bundle.js' */
+
+/** @typedef {VirtualModuleSource & {cjsFunctor: string}} CjsModuleSource */
+
 /** quotes strings */
 const q = JSON.stringify;
 
@@ -14,7 +20,8 @@ const exportsCellRecord = exportsList =>
   );
 
 // This function is serialized and references variables from its destination scope.
-const runtime = function wrapCjsFunctor(num) {
+const runtime = `\
+function wrapCjsFunctor(num) {
   /* eslint-disable no-undef */
   return ({ imports = {} }) => {
     const moduleCells = cells[num];
@@ -49,8 +56,9 @@ const runtime = function wrapCjsFunctor(num) {
     moduleCells['*'].set(Object.freeze(starExports));
   };
   /* eslint-enable no-undef */
-}.toString();
+}`;
 
+/** @type {BundlerSupport<CjsModuleSource>} */
 export default {
   runtime,
   getBundlerKit({

package/src/bundle-json.d.ts

@@ -0,0 +1,15 @@
+declare namespace _default {
+    let runtime: string;
+    function getBundlerKit({ index, indexedImports, bytes }: {
+        index: any;
+        indexedImports: any;
+        bytes: any;
+    }): {
+        getFunctor: () => string;
+        getCells: () => string;
+        getReexportsWiring: () => string;
+        getFunctorCall: () => string;
+    };
+}
+export default _default;
+//# sourceMappingURL=bundle-json.d.ts.map

package/src/bundle-json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bundle-json.d.ts","sourceRoot":"","sources":["bundle-json.js"],"names":[],"mappings":";;IAME;;;;;;;;;MAgBC"}

package/src/bundle-json.js

@@ -0,0 +1,24 @@
+/* Provides JSON support for `bundle.js`. */
+
+const textDecoder = new TextDecoder();
+
+export default {
+  runtime: '',
+  getBundlerKit({ index, indexedImports, bytes }) {
+    // Round-trip to revalidate JSON and squeeze out space.
+    const json = JSON.stringify(JSON.parse(textDecoder.decode(bytes)));
+    return {
+      getFunctor: () => `\
+// === functors[${index}] ===
+(set) => set(${json}),
+`,
+      getCells: () => `\
+    { default: cell('default') },
+`,
+      getReexportsWiring: () => '',
+      getFunctorCall: () => `\
+  functors[${index}](cells[${index}].default.set);
+`,
+    };
+  },
+};