@nimbus-sh/core 0.1.0 → 0.3.0

package/src/runtime/installed-runtimes.ts

@@ -0,0 +1,235 @@
+/**
+ * installed-runtimes.ts — the runtimes a session already has, and the shell
+ * commands they answer.
+ *
+ * `nimbus install <name>` unpacks a runtime into
+ * `~/.nimbus/runtimes/<name>/<version>/` and this is what reads it back: the
+ * manifests on disk, the runner each entrypoint names, and the registration
+ * that turns the pair into an invokable command. It runs at install time and
+ * again at every boot, because a Durable Object that was evicted comes back
+ * with the filesystem and none of the registry.
+ *
+ * Nothing here fetches. Where a runtime came FROM — an R2 bucket and a digest
+ * chain in the Cloudflare deployment — is `@nimbus-sh/worker`'s
+ * `runtime/package-manager.ts`; a runtime that is already installed is the
+ * same runtime whichever publisher put it there, so an embedder that seeds the
+ * tree itself gets working commands out of this with nothing else in play.
+ */
+
+import type { CredentialedVfs, SqliteVFS } from '../vfs/sqlite-vfs.js';
+import {
+  CRED_KERNEL,
+  NIMBUS_ABI_TARGET,
+  NIMBUS_RUNTIME_ABIS,
+  NATIVE_UNSUPPORTED_ABI,
+  type RuntimePackageAbi,
+} from './os-contracts.js';
+import {
+  parseRuntimeManifest,
+  type ManifestEntrypoint,
+  type RuntimeManifest,
+} from './runtime-manifest.js';
+
+/** Minimal shell-registry shape we depend on. */
+export interface MinShellRegistry {
+  register(name: string, handler: (ctx: any) => Promise<number>): void;
+  unregister?(name: string): void;
+  resolve?(name: string): any;
+}
+
+/** Runner-factory contract. Each registered runner produces a shell-
+ *  command handler given the manifest + the installed root dir. The
+ *  package manager invokes the factory at install-time + at boot-time
+ *  rehydration. */
+export type RunnerFactory = (
+  manifest: RuntimeManifest,
+  installRoot: string,
+  binName: string,
+  binKind: string | undefined,
+) => (ctx: any) => Promise<number>;
+
+/**
+ * How a manifest entrypoint's `runner` key is resolved to code.
+ *
+ * A parameter rather than a fixed lookup because the process-global table
+ * below is a property of the Cloudflare session, not of the idea: a factory
+ * closes over one session's filesystem and one session's facet host, so an
+ * embedder holding two workspaces in one process must be able to give each its
+ * own without the second silently retargeting the first.
+ */
+export type RunnerLookup = (key: string) => RunnerFactory | undefined;
+
+/** Map of runner-key → factory. Populated by init.ts before install. */
+const runnerFactories: Record<string, RunnerFactory> = {};
+
+export function registerRunnerFactory(key: string, factory: RunnerFactory): void {
+  runnerFactories[key] = factory;
+}
+
+export function getRegisteredRunners(): string[] {
+  return Object.keys(runnerFactories);
+}
+
+/** The factory a manifest entrypoint's `runner` names, or undefined. */
+export function runnerFactoryFor(key: string): RunnerFactory | undefined {
+  return runnerFactories[key];
+}
+
+export interface RuntimeSummary {
+  name: string;
+  version: string;
+  root: string;
+  abi: RuntimePackageAbi;
+  bins: string[];
+  sizeBytes: number;
+  license: string;
+}
+
+export function runtimeAbiForManifest(manifest: RuntimeManifest): RuntimePackageAbi {
+  const byName = NIMBUS_RUNTIME_ABIS[manifest.name];
+  if (byName) return byName;
+  if (manifest.wasi_namespace) return NIMBUS_ABI_TARGET;
+  if (manifest.entrypoints.some((entrypoint) => entrypoint.runner === 'clang-runner')) {
+    return NIMBUS_ABI_TARGET;
+  }
+  return NATIVE_UNSUPPORTED_ABI;
+}
+
+/**
+ * Commands a runtime provides beyond its manifest entrypoints. The
+ * python/ruby package-manager front-ends (pip, gem, bundler) ride the
+ * language runner rather than shipping as manifest files, and already-
+ * deployed R2 manifests cannot retroactively declare them.
+ *
+ * This is the ONE hand-maintained command table: install aliasing
+ * (`nimbus install pip` → python), command-not-found hints, and bin
+ * registration all derive from `runtimeEntrypoints`, which merges this
+ * with the catalog manifest. Catalog-declared aliases (python3, ruby3,
+ * wasm-ld, …) come from manifest entrypoints and must NOT be repeated
+ * here. Mechanically validated against `NIMBUS_RUNTIME_ABIS` by
+ * tests/unit/runtime-command-aliases.mjs.
+ */
+export const RUNTIME_EXTRA_ENTRYPOINTS: Readonly<Record<string, readonly ManifestEntrypoint[]>> = {
+  bash: [
+    { binName: '/bin/bash', runner: 'bash-runner', args: [] },
+    { binName: '/usr/bin/bash', runner: 'bash-runner', args: [] },
+  ],
+  // `pip` belongs to whichever runtime provides the interpreter, and only one
+  // may claim it. The python row went with python-runner: Pyodide's manifest
+  // still names that runner, so it could not serve pip even if it were listed.
+  cpython: [
+    { binName: 'pip', runner: 'cpython-runner', kind: 'pip', args: [] },
+    { binName: 'pip3', runner: 'cpython-runner', kind: 'pip', args: [] },
+  ],
+  ruby: [
+    { binName: 'gem', runner: 'ruby-runner', kind: 'gem', args: [] },
+    { binName: 'bundle', runner: 'ruby-runner', kind: 'bundle', args: [] },
+    { binName: 'bundler', runner: 'ruby-runner', kind: 'bundle', args: [] },
+  ],
+};
+
+export function runtimeEntrypoints(manifest: RuntimeManifest): RuntimeManifest['entrypoints'] {
+  const out = [...manifest.entrypoints];
+  const seen = new Set(out.map((ep) => ep.binName));
+  for (const ep of RUNTIME_EXTRA_ENTRYPOINTS[manifest.name] ?? []) {
+    if (seen.has(ep.binName)) continue;
+    out.push({ ...ep });
+    seen.add(ep.binName);
+  }
+  return out;
+}
+
+/** Compute the per-user install root for (name, version). Uses
+ *  `process.env.HOME` if present; falls back to `/home/user`. */
+export function installRoot(homeDir: string, name: string, version: string): string {
+  // Strip leading slash so SqliteFS sees a relative-looking VFS path,
+  // matching the convention used elsewhere in src/session/init.ts.
+  const home = homeDir.replace(/^\/+/, '').replace(/\/+$/, '');
+  return `${home}/.nimbus/runtimes/${name}/${version}`;
+}
+
+/** Read all installed manifests off SqliteFS. Used by both `--list`
+ *  and boot-time rehydration. */
+export function listInstalledManifests(
+  vfs: SqliteVFS,
+  homeDir: string,
+): Array<{ root: string; manifest: RuntimeManifest }> {
+  return listInstalledManifestsView(vfs.as(CRED_KERNEL), homeDir);
+}
+
+export function listInstalledManifestsView(
+  fs: CredentialedVfs,
+  homeDir: string,
+): Array<{ root: string; manifest: RuntimeManifest }> {
+  const home = homeDir.replace(/^\/+/, '').replace(/\/+$/, '');
+  const runtimesRoot = `${home}/.nimbus/runtimes`;
+  const out: Array<{ root: string; manifest: RuntimeManifest }> = [];
+  if (!fs.exists(runtimesRoot)) return out;
+  // Each entry under runtimesRoot is a <name>; each entry under that
+  // is a <version>; each <version> dir has a manifest.json.
+  for (const nameEntry of fs.readdir(runtimesRoot)) {
+    if (nameEntry.type !== 'directory') continue;
+    const nameDir = `${runtimesRoot}/${nameEntry.name}`;
+    for (const verEntry of fs.readdir(nameDir)) {
+      if (verEntry.type !== 'directory') continue;
+      const verDir = `${nameDir}/${verEntry.name}`;
+      const manifestPath = `${verDir}/manifest.json`;
+      if (!fs.exists(manifestPath)) continue;
+      try {
+        const manifest = parseRuntimeManifest(JSON.parse(fs.readFileString(manifestPath)));
+        out.push({ root: verDir, manifest });
+      } catch {
+        // Malformed manifest — skip silently. Surfacing via stderr
+        // would require a ctx we don't have at boot-time rehydration.
+      }
+    }
+  }
+  return out;
+}
+
+/**
+ * Re-register every installed runtime's entrypoints in the shell
+ * registry. Call once at session-init time after all runner factories
+ * are registered (init.ts:registerRunnerFactory blocks).
+ */
+export function rehydrateInstalledRuntimes(
+  vfs: SqliteVFS,
+  registry: MinShellRegistry,
+  homeDir: string,
+): { count: number; bins: string[] } {
+  return rehydrateInstalledRuntimesView(vfs.as(CRED_KERNEL), registry, homeDir, runnerFactoryFor);
+}
+
+export function rehydrateInstalledRuntimesView(
+  vfs: CredentialedVfs,
+  registry: MinShellRegistry,
+  homeDir: string,
+  runnerFor: RunnerLookup,
+): { count: number; bins: string[] } {
+  const bins: string[] = [];
+  for (const { root, manifest } of listInstalledManifestsView(vfs, homeDir)) {
+    for (const ep of runtimeEntrypoints(manifest)) {
+      const factory = runnerFor(ep.runner);
+      if (!factory) continue; // runner not registered yet — skip
+      const handler = factory(manifest, root, ep.binName, ep.kind);
+      registry.register(ep.binName, handler);
+      bins.push(ep.binName);
+    }
+  }
+  return { count: bins.length, bins };
+}
+
+export function listInstalledRuntimes(
+  vfs: SqliteVFS,
+  homeDir: string,
+): RuntimeSummary[] {
+  return listInstalledManifests(vfs, homeDir).map(({ root, manifest }) => ({
+    name: manifest.name,
+    version: manifest.version,
+    root,
+    abi: runtimeAbiForManifest(manifest),
+    bins: runtimeEntrypoints(manifest).map((e) => e.binName),
+    sizeBytes: manifest.files.reduce((a, f) => a + f.size, 0),
+    license: manifest.license,
+  }));
+}

package/src/runtime/local-facet-host.ts

@@ -0,0 +1,205 @@
+/**
+ * local-facet-host.ts — a facet that runs in the caller's own isolate.
+ *
+ * The {@link FacetHost} for every embedder that is not workerd. There is no
+ * dynamic-worker substrate to reach for and no CSP forbidding a compile, so the
+ * scope a facet needs is built where it is asked for: compile the wasm table,
+ * evaluate the preamble once, and evaluate each submitted function inside it.
+ *
+ * The function is still SERIALIZED rather than called in place, and that is the
+ * point rather than an accident of symmetry. A runner's facet function reads
+ * names the preamble declares — `__wasiMakeImports`, `__bashBoot` — which exist
+ * only in the scope the preamble was evaluated in; calling the original closure
+ * would resolve them against this module instead and fail. Serializing also
+ * keeps the contract honest: a closure reference that would break on workerd
+ * breaks here too, in a unit test, rather than in production.
+ *
+ * `globalThis` inside a facet is the facet's own scope object, not the process
+ * global. Preambles publish their entry points on it and runners read them back
+ * from it, so two facets in one process must not see each other's — and the
+ * process must not see either.
+ */
+
+import type {
+  Facet,
+  FacetBindings,
+  FacetFilesystemOptions,
+  FacetFilesystemSeed,
+  FacetFn,
+  FacetHost,
+  FacetSpec,
+  FacetSubmitOptions,
+} from './facet-host.js';
+import type { CredentialedVfs } from '../vfs/sqlite-vfs.js';
+import { snapshotVfs } from './vfs-snapshot.js';
+import { vfsSupervisor } from './vfs-supervisor.js';
+
+/** A submitted function after it has been re-created inside the facet's scope. */
+type ScopedFacetFn = (args: unknown, bindings: FacetBindings) => unknown;
+
+/**
+ * `new Function`, but for a body that may `await` at its top level.
+ *
+ * A facet preamble is written as a MODULE body, and the WASI shim uses that:
+ * it resolves `cloudflare:sockets` with a top-level `await import(...)` inside a
+ * try/catch, so a host that does not have the module gets a shim without
+ * sockets instead of a shim that fails to parse. `new Function` cannot hold
+ * that; an async function body can, and the rejected import lands in the same
+ * catch it was written for.
+ */
+const AsyncFunction = Object.getPrototypeOf(async (): Promise<void> => {}).constructor as
+  new (...args: string[]) => (this: object, ...args: unknown[]) => Promise<unknown>;
+
+type WasmCompiler = (bytes: BufferSource) => Promise<WebAssembly.Module>;
+
+/**
+ * The standard `WebAssembly.compile`, checked for rather than assumed.
+ *
+ * `@cloudflare/workers-types` declares no compiler and an abstract `Module`,
+ * which is not an oversight: workerd forbids compiling at request time, and
+ * that prohibition is the entire reason facets exist. Core is typed against
+ * that surface, so the one host that DOES compile asks for the capability by
+ * name and says so plainly when it is absent.
+ */
+function wasmCompiler(): WasmCompiler {
+  const compile: unknown = Reflect.get(WebAssembly, 'compile');
+  if (typeof compile !== 'function') {
+    throw new Error(
+      'Nimbus: this host cannot compile WebAssembly in place (no WebAssembly.compile), '
+      + 'so it needs a facet host with its own isolates rather than this one',
+    );
+  }
+  return compile as WasmCompiler;
+}
+
+/**
+ * The bound on a complete seed: a limit on this process's own heap.
+ *
+ * Deliberately far above `snapshotVfs`'s 32 MiB default, which is a TRANSPORT
+ * limit — the ceiling on one workerd RPC payload. Nothing is transported here,
+ * so keeping that number would refuse a program over a filesystem the host is
+ * already holding. Exceeding this is still an error rather than a truncation:
+ * a seed that silently omitted a file would make it absent to the guest.
+ */
+const LOCAL_SEED_MAX_BYTES = 512 * 1024 * 1024;
+const LOCAL_SEED_MAX_FILES = 200_000;
+
+/**
+ * Run facets in this isolate.
+ *
+ * `parking: 'none'` is the whole character of this host, and everything else
+ * follows from it: the guest is entered on an ordinary stack, so no syscall may
+ * suspend it, so {@link FacetHost.seedFilesystem} hands over the bytes rather
+ * than a manifest to fetch them with, and the supervisor it mints serves only
+ * the writes — which drain after the program returns, where a promise is free.
+ *
+ * The one thing a substrate with its own isolates gives that this cannot:
+ * {@link FacetSubmitOptions.timeoutMs} is not honoured. A guest spinning
+ * synchronously holds the only thread, so no timer fires until it is already
+ * finished; racing one would return while the program ran on.
+ */
+export function localFacetHost(): FacetHost {
+  return {
+    parking: 'none',
+    /**
+     * By value, and exhaustively: no skip list, because a directory hidden
+     * from the seed is a directory the guest cannot see at all — there is no
+     * second chance to fetch it. That completeness is what
+     * `snapshotVfs` records as `enumeratedRoots`, and it is what keeps CPython's
+     * thousands of startup probes for absent paths from each trying to suspend.
+     */
+    seedFilesystem(
+      vfs: CredentialedVfs,
+      root: string,
+      options?: FacetFilesystemOptions,
+    ): FacetFilesystemSeed | { error: string } {
+      return snapshotVfs(vfs, root, {
+        extraRoots: options?.extraRoots,
+        skipSubdirs: [],
+        maxBytes: LOCAL_SEED_MAX_BYTES,
+        maxFiles: LOCAL_SEED_MAX_FILES,
+      });
+    },
+    open: (spec) => new LocalFacet(spec),
+  };
+}
+
+class LocalFacet implements Facet {
+  /** Evaluates a source string in the preamble's own scope. Built on first use. */
+  private evaluate: ((source: string) => unknown) | null = null;
+  private readonly wasmTable: Record<string, WebAssembly.Module> = {};
+  /** Keyed by the bytes themselves: the same image is compiled once per facet. */
+  private readonly modules = new WeakMap<ArrayBuffer, WebAssembly.Module>();
+  private readonly scoped = new Map<FacetFn<never, unknown>, ScopedFacetFn>();
+  /** Submits are serialized: one scope, and a facet's calls are ordered. */
+  private queue: Promise<unknown> = Promise.resolve();
+  private disposed = false;
+
+  private readonly bindings: FacetBindings;
+
+  constructor(private readonly spec: FacetSpec) {
+    this.bindings = spec.syscalls
+      ? { SUPERVISOR: vfsSupervisor(spec.syscalls.vfs) }
+      : {};
+  }
+
+  submit<A, R>(fn: FacetFn<A, R>, args: A, options?: FacetSubmitOptions): Promise<Awaited<R>> {
+    const run = this.queue.then(() => this.call(fn, args, options));
+    // The chain must survive a rejected call, or one failure poisons the facet.
+    this.queue = run.catch(() => undefined);
+    return run as Promise<Awaited<R>>;
+  }
+
+  private async call<A, R>(fn: FacetFn<A, R>, args: A, options?: FacetSubmitOptions): Promise<R> {
+    if (this.disposed) throw new Error(`Nimbus: facet '${this.spec.tag}' is disposed`);
+    const scope = await this.scope();
+    await this.addModules(options?.wasmModules);
+    let scoped = this.scoped.get(fn as FacetFn<never, unknown>);
+    if (!scoped) {
+      const value = scope(`(${fn.toString()})`);
+      if (typeof value !== 'function') {
+        throw new Error(`Nimbus: facet '${this.spec.tag}' was submitted a value that is not a function`);
+      }
+      scoped = value as ScopedFacetFn;
+      this.scoped.set(fn as FacetFn<never, unknown>, scoped);
+    }
+    return await scoped(args, this.bindings) as R;
+  }
+
+  /**
+   * The facet's scope, built once.
+   *
+   * The returned closure's `eval` is a DIRECT eval inside the body the preamble
+   * was evaluated in, which is what puts the preamble's top-level declarations
+   * in scope for every function submitted afterwards.
+   */
+  private async scope(): Promise<(source: string) => unknown> {
+    if (this.evaluate) return this.evaluate;
+    await this.addModules(this.spec.wasmModules);
+    const globals = { __NIMBUS_WASM: this.wasmTable };
+    const build = new AsyncFunction(
+      'globalThis',
+      `${this.spec.preamble ?? ''}\nreturn (source) => eval(source);`,
+    );
+    this.evaluate = await build.call(globals, globals) as (source: string) => unknown;
+    return this.evaluate;
+  }
+
+  private async addModules(modules: Record<string, ArrayBuffer> | undefined): Promise<void> {
+    for (const [name, bytes] of Object.entries(modules ?? {})) {
+      let module = this.modules.get(bytes);
+      if (!module) {
+        module = await wasmCompiler()(bytes);
+        this.modules.set(bytes, module);
+      }
+      this.wasmTable[name] = module;
+    }
+  }
+
+  dispose(): void {
+    this.disposed = true;
+    this.evaluate = null;
+    this.scoped.clear();
+    for (const name of Object.keys(this.wasmTable)) delete this.wasmTable[name];
+  }
+}