@riboseinc/anafero-cli 0.0.47 → 0.0.49

package/dependencies.mts

@@ -1,5 +1,5 @@
 import { join, basename } from 'node:path';
-import { rmdir, readFile, cp, mkdtemp, stat } from 'node:fs/promises';
+import { readdir, rmdir, readFile, cp, mkdtemp, stat, mkdir, writeFile } from 'node:fs/promises';
 import vm, { type Module as VMModule } from 'node:vm';
 import { tmpdir } from 'node:os';
 import fs from 'node:fs';
@@ -31,7 +31,7 @@ const preloaded = {
   'prosemirror-schema-list': pmSchemaList,
   'react-helmet': helmet,
   'react': react,
-} as Record<string, unknown>;
+} as Readonly<Record<string, unknown>>;
 
 
 const depRegistry: Record<string, Promise<unknown>> = {};
@@ -140,6 +140,9 @@ const decoder = new TextDecoder();
  * with instantiated module and a map of any other generated files.
  * NOTE: Doesn’t really validate that returned module conforms to anything.
  *
+ * Handles caching (if a module was already requested, will resolve
+ * the same promise).
+ *
  * This currently relies on Node, and is therefore here in the CLI module;
  * see TODO about resolveDir about making dependencies buildable
  * in the browser.
@@ -149,194 +152,342 @@ async function fetchDependency(
   moduleRef,
   onProgress,
 ) {
-  if (depRegistry[moduleRef]) {
-    return await depRegistry[moduleRef];
+
+  if (!depRegistry[moduleRef]) {
+    depRegistry[moduleRef] = async function resolveDep() {
+      let sourceDir: string;
+
+      const localPath = moduleRef.split('file:')[1];
+      if (!localPath) {
+        //throw new Error("Only dependencies on local filesystem are supported for now.");
+        onProgress({ state: `fetching ${moduleRef} to ${localPath}` });
+        sourceDir = await fetchSourceFromGit(moduleRef, onProgress);
+      } else {
+        sourceDir = localPath;
+      }
+
+      const [bundledCode, otherFiles] = await getDependencyAssets(
+        moduleRef,
+        sourceDir,
+        onProgress,
+      );
+
+      const code = decoder.decode(bundledCode);
+
+      onProgress({ state: "instantiating module" });
+
+      // const fn = `${process.cwd()}/test.js`;
+      // console.debug("Will write to", fn);
+      // await writeFile(fn, result.outputFiles[0]!.contents, { encoding: 'utf-8' })
+
+      const context = vm.createContext({
+        Array,
+        Object,
+        crypto,
+        XPathResult,
+        document: getDoc(),
+        console,
+        Function,
+        setTimeout,
+        clearTimeout,
+        setInterval,
+        TextEncoder,
+        TextDecoder,
+        Blob,
+        btoa,
+        atob,
+      });
+      const mod = new vm.SourceTextModule(code, {
+        // TODO: Try moduleRef as VM module identifier?
+        // Take care of special characters, though.
+        identifier: 'anafero-dependency',
+        context,
+      });
+
+      async function link(specifier: string, referencingModule: VMModule) {
+        const isAbsoluteURL = specifier.startsWith('https://');
+        let base: string | undefined;
+        try {
+          new URL(referencingModule.identifier);
+          base = referencingModule.identifier;
+          // The module that does the importing is referenced by URL.
+        } catch (e) {
+          // The module that does the importing is not referenced by URL.
+          base = undefined;
+        }
+        const isRelativePath =
+          specifier.startsWith('/')
+          || specifier.startsWith('./')
+          || specifier.startsWith('../');
+        const isRelativeURL = isRelativePath && base !== undefined;
+
+        if (isAbsoluteURL || isRelativeURL) {
+          // Create a new absolute URL from the imported
+          // module's URL (specifier) and the parent module's
+          // URL (referencingModule.identifier).
+          //console.debug("anafero: building module URL", specifier, 'imported from', referencingModule.identifier);
+          if (isRelativeURL && base === undefined) {
+            throw new Error("Unable to resolve relative specifier without a base");
+          }
+          const url = new URL(specifier, base).toString();
+          // Download the raw source code.
+          //console.debug("anafero: fetching module", url);
+          const source = await (await fetch(url)).text();
+          // TODO: Fetched source needs to be cached appropriately
+          // Version needs to be taken into account?
+
+          // Instantiate a new module and return it.
+          return new vm.SourceTextModule(source, {
+            identifier: url,
+            context: referencingModule.context,
+          });
+        } else {
+          //console.debug("anafero: fetching preloaded", specifier);
+          const madeAvailable = preloaded[specifier]
+            ? preloaded[specifier]
+            // TODO: Don’t do the following
+            : await import(specifier);
+          const exportNames = Object.keys(madeAvailable);
+          // Construct a new module from the actual import
+          return new vm.SyntheticModule(
+            exportNames,
+            function () {
+              for (const name of exportNames) {
+                this.setExport(name, madeAvailable[name]);
+              }
+            },
+            { 
+              identifier: specifier,
+              context: referencingModule.context,
+            },
+          );
+        }
+      }
+
+      await mod.link(link);
+
+      await mod.evaluate();
+
+      const defaultExport = (mod.namespace as any).default as any;
+
+      dependencySources[moduleRef] = code;
+      dependencySupportingFiles[moduleRef] = otherFiles;
+
+      return defaultExport;
+
+      // It would be nice if this did work…
+      //let module = {};
+      //const req = (pkg: string) => {
+      //  console.debug("REQUIRED", pkg);
+      //  const result = import(pkg);
+      //  console.debug("GOT", result);
+      //  return result;
+      //};
+      //console.debug((new Function('module', 'require', decoder.decode(result.outputFiles[0]!.contents)))(module, req));
+      //const out = await import(fn);
+      //console.debug(out.name);
+      //throw new Error("TODO");
+      //const data = await readFile(outfile, { encoding: 'utf-8' });
+      //return new Function(data);
+    }();
   }
-  depRegistry[moduleRef] = async function resolveDep() {
-    let sourceDir: string;
-
-    const localPath = moduleRef.split('file:')[1];
-    if (!localPath) {
-      //throw new Error("Only dependencies on local filesystem are supported for now.");
-      onProgress({ state: `fetching ${moduleRef} to ${localPath}` });
-      sourceDir = await fetchSourceFromGit(moduleRef, onProgress);
-    } else {
-      sourceDir = localPath;
-    }
 
-    const buildDir = await mkdtemp(join(
-      tmpRoot,
-      `anafero-dist-${moduleRef.replace(/[^a-z0-9]/gi, '_')}-`,
-    ));
+  return await depRegistry[moduleRef] as any;
+};
 
-    onProgress({ state: `copying into build dir ${buildDir}` });
-
-    await cp(sourceDir, buildDir, { recursive: true });
-
-    //const outfile = join(buildDir, 'out.mjs');
-    //await esbuildTransform('oi');
-
-    onProgress({ state: "compiling" });
-
-    const result = await esbuild({
-      //entryPoints: [join(buildDir, 'index.mts')],
-      stdin: {
-        contents: await readFile(join(buildDir, 'index.mts')),
-        loader: 'ts',
-
-        // TODO: This means we use filesystem when resolving
-        // imports in the entry point. That’s not great, if we
-        // want to build e.g. in the browser.
-        // It may be possible to avoid this by writing a custom
-        // resolver plugin for esbuild. See
-        // - https://github.com/evanw/esbuild/issues/591#issuecomment-742962090
-        // - https://esbuild.github.io/plugins/#on-resolve-arguments
-        resolveDir: buildDir,
-
-        sourcefile: 'index.mts',
-      },
-      loader: {
-        '.mts': 'ts',
-        '.css': 'local-css',
-      },
-      entryNames: '[dir]/[name]',
-      assetNames: '[dir]/[name]',
-      format: 'esm',
-      target: ['es2022'],
-      tsconfigRaw: '{}',
-      //external: [],
-      packages: 'external',
-      //plugins: [{
-      //  name: 'plugin-resolver',
-      //  setup(build) {
-      //    build.onLoad({ filter: /^prosemirror-model-metanorma/ }, args => {
-      //      return import(args.path);
-      //    });
-      //  },
-      //}],
-      minify: false,
-      platform: 'browser',
-      write: false,
-      logLevel: 'silent',
-      //logLevel: 'info',
-      sourcemap: true,
-      bundle: true,
-      outfile: 'index.js',
-      treeShaking: true,
-      //outfile,
-    });
 
-    const otherFiles: Record<string, Uint8Array> = {};
+/**
+ * Builds dependency from TypeScript if needed,
+ * returns dependency source as string.
+ *
+ * Does not cache anything.
+ */
+async function getDependencyAssets(
+  moduleRef: string,
+  sourceDirPath: string,
+  onProgress: (progress: Progress) => void,
+): Promise<[
+  dependency: Uint8Array,
+  assets: Record<string, Uint8Array>,
+]> {
+  if (await isPreBuilt(sourceDirPath)) {
+    console.debug("Using pre-built", moduleRef);
+    return await readPreBuiltDependency(sourceDirPath);
+  } else {
+    return await buildDependency(moduleRef, sourceDirPath, onProgress);
+  }
+}
 
-    const mainOutput = result.outputFiles.
-      find(({ path }) => basename(path) === 'index.js')?.contents;
 
-    if (!mainOutput) {
-      throw new Error("Fetching dependency: no main output after building");
-    } else if (result.outputFiles.length > 1) {
-      for (const { path, contents } of result.outputFiles) {
-        if (!path.endsWith('/index.js')) {
-          otherFiles[basename(path)] = contents;
-        }
-      }
-    }
+function getPreBuiltRoot(sourceDirPath: string): string {
+  return join(sourceDirPath, 'dist');
+}
 
-    const code = decoder.decode(mainOutput);
-
-    onProgress({ state: "instantiating module" });
-
-    // const fn = `${process.cwd()}/test.js`;
-    // console.debug("Will write to", fn);
-    // await writeFile(fn, result.outputFiles[0]!.contents, { encoding: 'utf-8' })
-
-    const context = vm.createContext({
-      Array,
-      Object,
-      crypto,
-      XPathResult,
-      document: getDoc(),
-      console,
-      Function,
-      setTimeout,
-      setInterval,
-      TextEncoder,
-    });
-    const mod = new vm.SourceTextModule(code, {
-      // TODO: Try moduleRef as VM module identifier?
-      // Take care of special characters, though.
-      identifier: 'anafero-dependency',
-      context,
-    });
 
-    async function link(specifier: string, referencingModule: VMModule) {
-      if (specifier.startsWith('https://')) {
-        // Create a new absolute URL from the imported
-        // module's URL (specifier) and the parent module's
-        // URL (referencingModule.identifier).
-        const url = new URL(
-          specifier,
-          referencingModule.identifier,
-        ).toString();
-        // Download the raw source code.
-        const source = await (await fetch(url)).text();
-        // Instantiate a new module and return it.
-        return new vm.SourceTextModule(source, {
-          identifier: url,
-          context: referencingModule.context,
-        });
-      } else {
-        const madeAvailable = preloaded[specifier]
-          ? preloaded[specifier]
-          // TODO: Don’t do the following
-          : await import(specifier);
-        const exportNames = Object.keys(madeAvailable);
-        // Construct a new module from the actual import
-        return new vm.SyntheticModule(
-          exportNames,
-          function () {
-            for (const name of exportNames) {
-              this.setExport(name, madeAvailable[name]);
-            }
-          },
-          { 
-            identifier: specifier,
-            context: referencingModule.context,
-          },
-        );
+async function readPreBuiltJSBundle(sourceDirPath: string):
+Promise<Uint8Array> {
+  const bundlePath = join(
+    getPreBuiltRoot(sourceDirPath),
+    PRE_BUILT_JS_BUNDLE_FILENAME);
+  const bundleStat = await stat(bundlePath);
+  if (bundleStat.isFile()) {
+    return await readFile(bundlePath);
+  } else {
+    throw new Error("Pre-built entry point is not a file");
+  }
+}
 
-      }
-    }
 
-    await mod.link(link);
+const PRE_BUILT_JS_BUNDLE_FILENAME = 'index.js';
+
 
-    await mod.evaluate();
+async function readPreBuiltAssets(sourceDirPath: string):
+Promise<Record<string, Uint8Array>> {
+  const distroot = getPreBuiltRoot(sourceDirPath);
+  const filenames = await readdir(distroot);
+  const assets = filenames.filter(fn => fn !== PRE_BUILT_JS_BUNDLE_FILENAME);
+  return (
+    (await Promise.all(
+      assets.map(async (fn) => ({ [fn]: await readFile(join(distroot, fn)) }))
+    )).
+    reduce((prev, curr) => ({ ...prev, ...curr }), {})
+  );
+}
 
-    const defaultExport = (mod.namespace as any).default as any;
 
-    onProgress({ state: `removing build dir ${buildDir}` });
+export async function writePreBuiltAssets(
+  sourceDirPath: string,
+  bundledCode: Uint8Array,
+  assets: Record<string, Uint8Array>,
+) {
+  const distDir = getPreBuiltRoot(sourceDirPath);
+  await mkdir(distDir);
+  console.debug("Writing index.js");
+  await writeFile(join(distDir, PRE_BUILT_JS_BUNDLE_FILENAME), bundledCode);
+  for (const [fn, blob] of Object.entries(assets)) {
+    console.debug("Writing asset", fn);
+    await writeFile(join(distDir, fn), blob);
+  }
+}
 
-    await rmdir(buildDir, { recursive: true });
 
-    dependencySources[moduleRef] = code;
-    dependencySupportingFiles[moduleRef] = otherFiles;
+async function isPreBuilt(sourceDirPath: string): Promise<boolean> {
+  try {
+    // A pre-built dependency contains directory “dist”
+    // with index.js and possibly other assets.
+    // Subdirectories under dist are ignored.
+    await readPreBuiltJSBundle(sourceDirPath);
+    await readPreBuiltAssets(sourceDirPath);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
 
-    return defaultExport;
 
-    // It would be nice if this did work…
-    //let module = {};
-    //const req = (pkg: string) => {
-    //  console.debug("REQUIRED", pkg);
-    //  const result = import(pkg);
-    //  console.debug("GOT", result);
-    //  return result;
-    //};
-    //console.debug((new Function('module', 'require', decoder.decode(result.outputFiles[0]!.contents)))(module, req));
-    //const out = await import(fn);
-    //console.debug(out.name);
-    //throw new Error("TODO");
-    //const data = await readFile(outfile, { encoding: 'utf-8' });
-    //return new Function(data);
-  }();
+async function readPreBuiltDependency(
+  sourceDirPath: string,
+): Promise<[
+  bundledCode: Uint8Array,
+  supportingAssets: Record<string, Uint8Array>,
+]> {
+  return [
+    await readPreBuiltJSBundle(sourceDirPath),
+    await readPreBuiltAssets(sourceDirPath),
+  ];
+}
 
-  return await depRegistry[moduleRef] as any;
+
+export async function buildDependency(
+  /**
+   * Acts only as cache key to avoid build directories clashing.
+   */
+  moduleRef: string,
+  sourceDir: string,
+  onProgress: (progress: Progress) => void,
+): Promise<[
+  bundledCode: Uint8Array,
+  supportingAssets: Record<string, Uint8Array>,
+]> {
+  const buildDir = await mkdtemp(join(
+    tmpRoot,
+    `anafero-dist-${moduleRef.replace(/[^a-z0-9]/gi, '_')}-`,
+  ));
+
+  onProgress({ state: `copying into build dir ${buildDir}` });
+
+  await cp(sourceDir, buildDir, { recursive: true });
+
+  //const outfile = join(buildDir, 'out.mjs');
+  //await esbuildTransform('oi');
+
+  onProgress({ state: "compiling" });
+
+  const result = await esbuild({
+    //entryPoints: [join(buildDir, 'index.mts')],
+    stdin: {
+      contents: await readFile(join(buildDir, 'index.mts')),
+      loader: 'ts',
+
+      // TODO: This means we use filesystem when resolving
+      // imports in the entry point. That’s not great, if we
+      // want to build e.g. in the browser.
+      // It may be possible to avoid this by writing a custom
+      // resolver plugin for esbuild. See
+      // - https://github.com/evanw/esbuild/issues/591#issuecomment-742962090
+      // - https://esbuild.github.io/plugins/#on-resolve-arguments
+      resolveDir: buildDir,
+
+      sourcefile: 'index.mts',
+    },
+    loader: {
+      '.mts': 'ts',
+      '.css': 'local-css',
+    },
+    entryNames: '[dir]/[name]',
+    assetNames: '[dir]/[name]',
+    format: 'esm',
+    target: ['es2022'],
+    tsconfigRaw: '{}',
+    //external: [],
+    packages: 'external',
+    //plugins: [{
+    //  name: 'plugin-resolver',
+    //  setup(build) {
+    //    build.onLoad({ filter: /^prosemirror-model-metanorma/ }, args => {
+    //      return import(args.path);
+    //    });
+    //  },
+    //}],
+    minify: false,
+    platform: 'browser',
+    write: false,
+    logLevel: 'silent',
+    //logLevel: 'info',
+    sourcemap: true,
+    bundle: true,
+    outfile: 'index.js',
+    treeShaking: true,
+    //outfile,
+  });
+
+  onProgress({ state: `removing build dir ${buildDir}` });
+  await rmdir(buildDir, { recursive: true });
+
+  const otherFiles: Record<string, Uint8Array> = {};
+
+  const mainOutput = result.outputFiles.
+    find(({ path }) => basename(path) === 'index.js')?.contents;
+
+  if (!mainOutput) {
+    throw new Error("Fetching dependency: no main output after building");
+  } else if (result.outputFiles.length > 1) {
+    for (const { path, contents } of result.outputFiles) {
+      if (!path.endsWith('/index.js')) {
+        otherFiles[basename(path)] = contents;
+      }
+    }
+  }
+
+  return [mainOutput, otherFiles];
 }

package/generate-to-filesystem.tsx

@@ -42,6 +42,8 @@ import { type VersionBuildConfig, type VersionMeta } from 'anafero/index.mjs';
 
 import {
   fetchDependency,
+  buildDependency,
+  writePreBuiltAssets,
   getDependencySources,
   getDependencySupportingFiles,
 } from './dependencies.mjs';
@@ -57,6 +59,10 @@ console.debug("Package root", PACKAGE_ROOT);
 
 const decoder = new TextDecoder();
 
+const cwd = process.cwd();
+console.debug("Current working directory", cwd);
+
+
 Effect.
   suspend(() => main(process.argv)).
   pipe(
@@ -64,13 +70,50 @@ Effect.
     NodeRuntime.runMain,
   );
 
+
 function unpackOption<T>(opt: Option.Option<T>, df?: T): T | undefined {
   return Option.isNone(opt) ? df : opt.value;
 }
 
-const build = Command.
+
+const dispatch = Command.
+  make(
+    "npx --node-options='--experimental-vm-modules' -y @riboseinc/anafero-cli",
+    {},
+    () => Effect.log("Pass --help for usage instructions")).
+  pipe(
+    Command.withDescription("Anafero builder. Use with a subcommand."),
+  );
+
+const buildPackage = Command.
+  make('build-package', reportingOptions, () => Effect.gen(function * (_) {
+    const moduleRef = yield * _(
+      Effect.tryPromise(() => readFile(join(cwd, 'package.json'))),
+      Effect.flatMap(blob => Effect.try(() => decoder.decode(blob))),
+      Effect.flatMap(pkgRaw => Effect.try(() => JSON.parse(pkgRaw))),
+      Effect.flatMap(pkg => Effect.succeed(pkg['name'])),
+    );
+    yield * _(Effect.log(`Building package ${moduleRef}`));
+    const [bundledCode, assets] = yield * _(Effect.tryPromise(() => {
+      return buildDependency(
+        moduleRef,
+        cwd,
+        (progress) => console.debug(JSON.stringify(progress)),
+      );
+    }));
+    yield * _(Effect.tryPromise(() => writePreBuiltAssets(
+      cwd,
+      bundledCode,
+      assets,
+    )));
+  })).
+  pipe(
+    Command.withDescription("For developers: builds adapter in current directory into `./dist`."),
+  );
+
+const buildSite = Command.
   make(
-    'build',
+    'build-site',
     {
 
       targetDirectoryPath: Options.directory('target-dir'),
@@ -174,11 +217,11 @@ const build = Command.
     ),
   ).
   pipe(
-    Command.withDescription('build site'),
+    Command.withDescription("Builds a website using current directory as source."),
   );
 
 
-const dev = Command.
+const devSite = Command.
   make(
     'develop',
     {
@@ -198,14 +241,14 @@ const dev = Command.
     ({ pkg, skipBuild }) =>
       Effect.
         gen(function * (_) {
-          const buildCfg = yield * _(build);
+          const buildCfg = yield * _(buildSite);
 
           const { targetDirectoryPath } = buildCfg;
 
           // Maybe build
 
           if (!skipBuild) {
-            yield * build.handler(buildCfg);
+            yield * buildSite.handler(buildCfg);
           }
 
 
@@ -271,14 +314,17 @@ const dev = Command.
         })
   ).
   pipe(
-    Command.withDescription('dev mode (watching for changes & copying client-side JS)'),
+    Command.withDescription("Build site in dev mode (watching for changes & copying client-side JS)."),
   );
 
 
 
-const main = build.
+const main = dispatch.
   pipe(
-    Command.withSubcommands([dev]),
+    Command.withSubcommands([
+      buildSite.pipe(Command.withSubcommands([devSite])),
+      buildPackage,
+    ]),
     Command.run({
       name: "Anafero builder",
       version: "N/A",
@@ -286,8 +332,8 @@ const main = build.
   );
 
 
-// TODO: Refactor gitdir handling, avoid the globa
-const gitdir = join(process.cwd(), '.git');
+// TODO: Refactor gitdir handling, avoid the global
+const gitdir = join(cwd, '.git');
 
 async function areWeInGitRepoRoot(): Promise<boolean> {
   const gitRepoStat = await stat(gitdir);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@riboseinc/anafero-cli",
   "type": "module",
-  "version": "0.0.47",
+  "version": "0.0.49",
   "packageManager": "yarn@4.5.0",
   "bin": {
     "build-site": "build-site.mjs"