@nimbus-sh/core 0.1.0

package/src/_shared/byte-stream.ts

@@ -0,0 +1,97 @@
+/**
+ * _shared/byte-stream.ts — bounded byte streaming between the VFS and a
+ * command's output.
+ *
+ * Every coreutil that copies file content (`cat`, `head -c`, `dd`) needs the
+ * same three properties, and getting any of them wrong produces silently
+ * wrong bytes:
+ *
+ *   1. Bounded memory. Content moves one chunk at a time, so a 25 MB file
+ *      never becomes a 25 MB buffer — which matters because facet↔supervisor
+ *      RPC payloads are capped (MAX_RPC_SAFE_PAYLOAD_BYTES).
+ *   2. A real write offset. The sink advances; it does not restate the file.
+ *   3. Chunk boundaries that do not corrupt text. A UTF-8 sequence split
+ *      across two chunks must decode as one character, not two replacements,
+ *      so text sinks get a streaming decoder rather than per-chunk decodes.
+ */
+
+/**
+ * A command's output destination.
+ *
+ * `writeBytes` is present on sinks that store bytes verbatim (files). Sinks
+ * that are inherently textual — the terminal, shell pipes — omit it and
+ * receive decoded text instead. This is a real capability difference, not a
+ * fallback: a terminal has no way to hold a byte that is not text.
+ */
+export interface ByteSink {
+  write(text: string): void;
+  writeBytes?(bytes: Uint8Array): void;
+}
+
+/** Reads at most `length` bytes at `offset`; an empty result means EOF. */
+export type RangeReader = (offset: number, length: number) => Uint8Array;
+
+/** Bytes moved per read. Comfortably under any RPC payload ceiling. */
+export const STREAM_CHUNK_BYTES = 64 * 1024;
+
+/**
+ * Feeds bytes to a {@link ByteSink}, decoding across chunk boundaries when the
+ * sink is textual. Callers must `end()` so a trailing partial UTF-8 sequence
+ * is flushed.
+ */
+export class SinkWriter {
+  private readonly decoder: TextDecoder | null;
+  private written = 0;
+
+  constructor(private readonly sink: ByteSink) {
+    this.decoder = sink.writeBytes ? null : new TextDecoder('utf-8');
+  }
+
+  get bytesWritten(): number {
+    return this.written;
+  }
+
+  write(bytes: Uint8Array): void {
+    if (bytes.length === 0) return;
+    this.written += bytes.length;
+    if (this.decoder) this.sink.write(this.decoder.decode(bytes, { stream: true }));
+    else this.sink.writeBytes!(bytes);
+  }
+
+  end(): void {
+    if (!this.decoder) return;
+    const tail = this.decoder.decode();
+    if (tail) this.sink.write(tail);
+  }
+}
+
+/**
+ * Copies bytes from `read` into `sink` in {@link STREAM_CHUNK_BYTES} chunks.
+ *
+ * Stops at `length` bytes when given one, otherwise at the first short read
+ * (EOF). Returns the number of bytes copied. Does not call `sink.end()` —
+ * callers that write more than one range share a single writer.
+ */
+export function streamRange(
+  read: RangeReader,
+  writer: SinkWriter,
+  options: { offset?: number; length?: number; signal?: AbortSignal } = {},
+): number {
+  const { offset = 0, length, signal } = options;
+  let position = offset;
+  let copied = 0;
+
+  while (length === undefined || copied < length) {
+    if (signal?.aborted) break;
+    const want = length === undefined
+      ? STREAM_CHUNK_BYTES
+      : Math.min(STREAM_CHUNK_BYTES, length - copied);
+    const chunk = read(position, want);
+    if (chunk.length === 0) break;
+    writer.write(chunk);
+    position += chunk.length;
+    copied += chunk.length;
+  }
+
+  return copied;
+}

package/src/_shared/bytes.ts

@@ -0,0 +1,19 @@
+/**
+ * _shared/bytes.ts — Singleton TextEncoder / TextDecoder.
+ *
+ * TextEncoder and TextDecoder for UTF-8 are stateless per the WHATWG
+ * Encoding spec, so a module-scope singleton is safe and saves the
+ * repeated allocation (which shows up in flame graphs of hot paths
+ * like sqlite-vfs writeFile and the WebSocket terminal frame decoder).
+ *
+ * Use these everywhere instead of `new TextEncoder()` / `new TextDecoder()`
+ * in the supervisor isolate. Facet-isolate code-strings (e.g. inside
+ * generateGitNetworkFacetCode) cannot import this module and must keep
+ * their inline allocations — those copies are justified.
+ */
+
+/** Shared UTF-8 encoder. Stateless; safe to share across all callers. */
+export const enc = new TextEncoder();
+
+/** Shared UTF-8 decoder. Stateless; safe to share across all callers. */
+export const dec = new TextDecoder();

package/src/_shared/cache-stats.ts

@@ -0,0 +1,231 @@
+/**
+ * cache-stats.ts — Per-tier hit/miss/bytes counters for the npm cache stack.
+ *
+ * Purpose
+ * ───────
+ * The supervisor (and its facets) make cache lookups at four tiers:
+ *
+ *   L1 — per-DO SQLite (NpmCache)                    ~1 ms / file
+ *   L2 — caches.default (per-colo)                   ~50-500 µs hit
+ *   L3 — R2 (cross-tenant global)                    ~30-100 ms regional
+ *   L4 — registry.npmjs.org origin                   ~100-300 ms cross-region
+ *
+ * Pre-cache metrics support: hit/miss counters lived in
+ * src/observability/diag-counters.ts but only tracked the RPC-call
+ * boundary, not per-tier. Those flat counters have since been removed.
+ *
+ * This module adds per-tier × per-kind counters. SAME singleton-per-
+ * isolate pattern as diag-counters.ts so any supervisor code path can
+ * write to it and `/api/_diag/cache` reads it back.
+ *
+ * Why a separate module instead of extending diag-counters.ts:
+ *   - Avoids touching diag-counters.ts (other waves may extend that
+ *     file's R2 race counters; keeping these orthogonal makes rebases
+ *     trivial).
+ *   - The new shape (Tier × Kind grid) is structurally different from
+ *     diag-counters' flat r2.* shape. Mixing them in one type would
+ *     create awkward conditional types.
+ *   - The new counters are exposed at /api/_diag/cache; diag-counters
+ *     is exposed at /api/_diag/memory. Separate endpoints, separate
+ *     modules.
+ *
+ * Lives in src/_shared/ so anyone (supervisor + future facet code) can
+ * import it without crossing the npm/, runtime/, facets/ boundaries
+ * that have wave-owners.
+ */
+
+/**
+ * Tier in the cache stack. Each tier is checked in order from L1 (warmest,
+ * fastest) down to L4 (coldest, slowest origin). Cache writes typically
+ * "fill" lower numbers from higher numbers — an L4 fetch may writeback
+ * to L3 and L2, an L3 hit may writeback to L2, etc.
+ */
+export type CacheTier = 'L1' | 'L2' | 'L3' | 'L4';
+
+/**
+ * Kind of content cached. Each kind has its own size profile and eviction
+ * policy:
+ *
+ *   tarball   — gzipped tar bytes, immutable forever, 30 MiB cap on L3
+ *   packument — registry metadata JSON, TTL-bounded (60 min post-P6),
+ *               unbounded size in practice (~10 KB - 1 MB typical)
+ *   asset     — non-package binaries cached cross-tenant (esbuild.wasm,
+ *               rolldown.wasm, etc.). Immutable forever, low churn.
+ */
+export type CacheKind = 'tarball' | 'packument' | 'asset';
+
+/**
+ * Per-(tier, kind) cell. A hit happened when we asked the tier "do you
+ * have <key>?" and it said yes (and returned bytes). A miss happened when
+ * we asked and it said no.
+ *
+ * `bytes` accumulates the size of HIT payloads only — misses have no
+ * bytes by definition. This means:
+ *
+ *   bytes / hits = average hit payload size
+ *
+ * which is the metric operators care about for capacity planning.
+ */
+export interface CacheCell {
+  hits: number;
+  misses: number;
+  /** Cumulative bytes returned on hits. Does NOT include miss "size" (0). */
+  bytes: number;
+}
+
+export interface CacheStatsSnapshot {
+  /** Tier × Kind grid. Each cell tracks hits, misses, bytes-on-hit. */
+  byTier: Record<CacheTier, Record<CacheKind, CacheCell>>;
+  /**
+   * Wall-clock ms at module load (when this isolate first received a
+   * request that imported this module). Stable across requests within
+   * the same isolate; resets on DO reboot (itself a useful signal).
+   */
+  startedAt: number;
+  /**
+   * Wall-clock ms at the most recent `reset()` call. Equals startedAt
+   * if reset() has never been called.
+   */
+  lastResetAt: number;
+  /**
+   * Derived hit-rate per (tier, kind), 0..1. Computed as
+   * hits / (hits + misses). NaN-safe: returns 0 when (hits + misses)
+   * is 0 so JSON serialisation stays clean. Mirrors the byTier shape
+   * exactly so consumers can ZIP the two structures with the same key
+   * paths.
+   */
+  hitRate: Record<CacheTier, Record<CacheKind, number>>;
+}
+
+const TIERS: CacheTier[] = ['L1', 'L2', 'L3', 'L4'];
+const KINDS: CacheKind[] = ['tarball', 'packument', 'asset'];
+
+function makeEmptyGrid(): Record<CacheTier, Record<CacheKind, CacheCell>> {
+  const grid = {} as Record<CacheTier, Record<CacheKind, CacheCell>>;
+  for (const tier of TIERS) {
+    grid[tier] = {} as Record<CacheKind, CacheCell>;
+    for (const kind of KINDS) {
+      grid[tier][kind] = { hits: 0, misses: 0, bytes: 0 };
+    }
+  }
+  return grid;
+}
+
+/**
+ * Module-scoped singleton. Lives for the lifetime of the isolate.
+ *
+ * Workerd may evict the isolate at any time (memory pressure, idle
+ * shutdown, code update); when that happens, counters reset to zero
+ * on the next request — itself diagnostic signal: counters all-zero
+ * immediately after a request means we just woke up from cold.
+ *
+ * startedAt lazy-init: workerd returns 0 for `Date.now()` at
+ * module-evaluation time because IO is gated until the first request.
+ * Initialize lazily on the first hit/miss/snapshot so the timestamp
+ * reflects actual module-first-touched-by-request time, not the
+ * sentinel 0 value.
+ */
+const _grid = makeEmptyGrid();
+let _startedAt = 0;
+let _lastResetAt = 0;
+function _ensureStartedAt(): void {
+  if (_startedAt === 0) {
+    const now = Date.now();
+    _startedAt = now;
+    _lastResetAt = now;
+  }
+}
+
+/**
+ * Record a cache HIT at the given tier for the given kind.
+ *
+ * `bytes` is the size of the payload returned by the hit, in bytes.
+ * For packuments this is the JSON text length; for tarballs it's the
+ * gzipped tar byte length; for assets it's the asset bytes. The caller
+ * already has the size (post-fetch); passing it here is cheap.
+ *
+ * Pass 0 if the hit returned an empty payload (shouldn't happen in
+ * practice but the API tolerates it).
+ */
+export function recordHit(tier: CacheTier, kind: CacheKind, bytes: number): void {
+  _ensureStartedAt();
+  const cell = _grid[tier][kind];
+  cell.hits++;
+  // Negative bytes is meaningless; coerce to 0. We trust the caller for
+  // accuracy but defend against accidental -1 sentinels.
+  cell.bytes += bytes > 0 ? bytes : 0;
+}
+
+/**
+ * Record a cache MISS at the given tier for the given kind.
+ *
+ * Misses don't carry payload bytes. A miss at tier N typically means
+ * the caller will fall through to tier N+1; that downstream call will
+ * record its OWN hit or miss. So a single fetch flow naturally records
+ * a chain of misses ending in one hit at the tier that served the data.
+ *
+ * Example: cold fetch of `react@18.3.1` tarball records:
+ *
+ *   recordMiss('L1', 'tarball');
+ *   recordMiss('L2', 'tarball');
+ *   recordMiss('L3', 'tarball');
+ *   recordHit('L4', 'tarball', 12345);  // bytes from registry response
+ *
+ * After the writeback path runs, the same key from a subsequent fetch
+ * would record:
+ *
+ *   recordMiss('L1', 'tarball');
+ *   recordHit('L2', 'tarball', 12345);
+ */
+export function recordMiss(tier: CacheTier, kind: CacheKind): void {
+  _ensureStartedAt();
+  _grid[tier][kind].misses++;
+}
+
+/**
+ * Return a snapshot of current counters + derived hit-rates.
+ *
+ * The returned object is a fresh copy — caller-side mutations don't
+ * affect the singleton. Mirrors readDiagCounters() in diag-counters.ts.
+ */
+export function snapshot(): CacheStatsSnapshot {
+  _ensureStartedAt();
+  // Deep copy so caller mutations don't leak back into the singleton.
+  const byTier = {} as Record<CacheTier, Record<CacheKind, CacheCell>>;
+  const hitRate = {} as Record<CacheTier, Record<CacheKind, number>>;
+  for (const tier of TIERS) {
+    byTier[tier] = {} as Record<CacheKind, CacheCell>;
+    hitRate[tier] = {} as Record<CacheKind, number>;
+    for (const kind of KINDS) {
+      const c = _grid[tier][kind];
+      byTier[tier][kind] = { hits: c.hits, misses: c.misses, bytes: c.bytes };
+      const lookups = c.hits + c.misses;
+      hitRate[tier][kind] = lookups === 0 ? 0 : c.hits / lookups;
+    }
+  }
+  return {
+    byTier,
+    startedAt: _startedAt,
+    lastResetAt: _lastResetAt,
+    hitRate,
+  };
+}
+
+/**
+ * Zero all counters. Sets lastResetAt to now. Used by the
+ * `/api/_diag/cache/reset` endpoint to start a fresh measurement window.
+ *
+ * Does NOT clear startedAt — that remains the original module-load time
+ * so operators can tell "this isolate has been up for X but counters
+ * were reset at Y" (the gap is intentional vs an isolate reboot).
+ */
+export function reset(): void {
+  for (const tier of TIERS) {
+    for (const kind of KINDS) {
+      _grid[tier][kind].hits = 0;
+      _grid[tier][kind].misses = 0;
+      _grid[tier][kind].bytes = 0;
+    }
+  }
+  _lastResetAt = Date.now();
+}

package/src/_shared/crypto.ts

@@ -0,0 +1,184 @@
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+const SEALED_JSON_V2 = 'v2.';
+const SEALED_JSON_V1 = 'v1.';
+const HKDF_SALT = textEncoder.encode('nimbus-sh sealed-json v2');
+const BASE64URL_RE = /^[A-Za-z0-9_-]*$/;
+
+export interface SealedJsonOptions {
+  purpose?: string;
+  minSecretLength?: number;
+}
+
+export function randomBase64Url(byteLength: number): string {
+  const bytes = new Uint8Array(byteLength);
+  crypto.getRandomValues(bytes);
+  return base64Url(bytes);
+}
+
+export async function sha256Base64Url(input: string): Promise<string> {
+  const digest = await crypto.subtle.digest('SHA-256', textEncoder.encode(input));
+  return base64Url(new Uint8Array(digest));
+}
+
+export async function pkceChallenge(verifier: string): Promise<string> {
+  return sha256Base64Url(verifier);
+}
+
+/** Lowercase hex SHA-256 — the digest form the staged-artifact integrity checks pin. */
+export async function sha256Hex(input: BufferSource | string): Promise<string> {
+  const digest = await crypto.subtle.digest(
+    'SHA-256',
+    typeof input === 'string' ? textEncoder.encode(input) : input,
+  );
+  const view = new Uint8Array(digest);
+  let hex = '';
+  for (let i = 0; i < view.length; i++) hex += view[i].toString(16).padStart(2, '0');
+  return hex;
+}
+
+export async function sealJson(
+  value: unknown,
+  secret: string,
+  options: SealedJsonOptions = {},
+): Promise<string> {
+  const purpose = normalizePurpose(options.purpose);
+  const key = await hkdfAesGcmKey(secret, purpose, options.minSecretLength);
+  const iv = new Uint8Array(12);
+  crypto.getRandomValues(iv);
+  const plaintext = textEncoder.encode(JSON.stringify(value));
+  const ciphertext = new Uint8Array(await crypto.subtle.encrypt(
+    { name: 'AES-GCM', iv, additionalData: aad(purpose) },
+    key,
+    plaintext,
+  ));
+  const packed = new Uint8Array(iv.length + ciphertext.length);
+  packed.set(iv, 0);
+  packed.set(ciphertext, iv.length);
+  return SEALED_JSON_V2 + base64Url(packed);
+}
+
+export async function unsealJson<T>(
+  value: string,
+  secret: string,
+  options: SealedJsonOptions = {},
+): Promise<T | null> {
+  if (value.startsWith(SEALED_JSON_V2)) {
+    const purpose = normalizePurpose(options.purpose);
+    const packed = base64UrlDecode(value.slice(SEALED_JSON_V2.length));
+    if (packed.length <= 12) return null;
+    const iv = packed.slice(0, 12);
+    const ciphertext = packed.slice(12);
+    const key = await hkdfAesGcmKey(secret, purpose, options.minSecretLength);
+    const plaintext = await crypto.subtle.decrypt(
+      { name: 'AES-GCM', iv, additionalData: aad(purpose) },
+      key,
+      ciphertext,
+    );
+    return JSON.parse(textDecoder.decode(plaintext)) as T;
+  }
+
+  // Backward compatibility for the original agent OAuth cookie format.
+  // New cookies are always v2 and purpose-bound through AES-GCM AAD.
+  if (value.startsWith(SEALED_JSON_V1)) {
+    const packed = base64UrlDecode(value.slice(SEALED_JSON_V1.length));
+    if (packed.length <= 12) return null;
+    const iv = packed.slice(0, 12);
+    const ciphertext = packed.slice(12);
+    const key = await legacyShaAesGcmKey(secret, options.minSecretLength);
+    const plaintext = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, key, ciphertext);
+    return JSON.parse(textDecoder.decode(plaintext)) as T;
+  }
+
+  return null;
+}
+
+export function encodeJsonBase64Url(value: unknown): string {
+  return base64Url(textEncoder.encode(JSON.stringify(value)));
+}
+
+export function decodeJsonBase64Url<T>(value: string): T {
+  return JSON.parse(textDecoder.decode(base64UrlDecode(value))) as T;
+}
+
+export function base64Url(bytes: Uint8Array): string {
+  let binary = '';
+  const chunk = 0x8000;
+  for (let i = 0; i < bytes.length; i += chunk) {
+    binary += String.fromCharCode(...bytes.subarray(i, i + chunk));
+  }
+  return btoa(binary)
+    .replace(/\+/g, '-')
+    .replace(/\//g, '_')
+    .replace(/=+$/g, '');
+}
+
+export function base64UrlDecode(value: string): Uint8Array {
+  if (!BASE64URL_RE.test(value) || value.length % 4 === 1) {
+    throw new Error('Invalid base64url input');
+  }
+  const padded = value.replace(/-/g, '+').replace(/_/g, '/').padEnd(Math.ceil(value.length / 4) * 4, '=');
+  const binary = atob(padded);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+  return bytes;
+}
+
+export function base64Utf8(value: string): string {
+  const bytes = textEncoder.encode(value);
+  let binary = '';
+  const chunk = 0x8000;
+  for (let i = 0; i < bytes.length; i += chunk) {
+    binary += String.fromCharCode(...bytes.subarray(i, i + chunk));
+  }
+  return btoa(binary);
+}
+
+async function hkdfAesGcmKey(
+  secret: string,
+  purpose: string,
+  minSecretLength = 32,
+): Promise<CryptoKey> {
+  assertSecret(secret, minSecretLength);
+  const material = await crypto.subtle.importKey(
+    'raw',
+    textEncoder.encode(secret),
+    'HKDF',
+    false,
+    ['deriveKey'],
+  );
+  return crypto.subtle.deriveKey(
+    {
+      name: 'HKDF',
+      hash: 'SHA-256',
+      salt: HKDF_SALT,
+      info: textEncoder.encode(purpose),
+    },
+    material,
+    { name: 'AES-GCM', length: 256 },
+    false,
+    ['encrypt', 'decrypt'],
+  );
+}
+
+async function legacyShaAesGcmKey(secret: string, minSecretLength = 32): Promise<CryptoKey> {
+  assertSecret(secret, minSecretLength);
+  const digest = await crypto.subtle.digest('SHA-256', textEncoder.encode(secret));
+  return crypto.subtle.importKey('raw', digest, 'AES-GCM', false, ['decrypt']);
+}
+
+function assertSecret(secret: string, minSecretLength: number): void {
+  if (!secret || secret.length < minSecretLength) {
+    throw new Error(`Sealed JSON secret must be ${minSecretLength}+ characters`);
+  }
+}
+
+function normalizePurpose(purpose: string | undefined): string {
+  const value = (purpose || 'default').trim();
+  if (!value) return 'default';
+  return value.slice(0, 128);
+}
+
+function aad(purpose: string): Uint8Array {
+  return textEncoder.encode(`nimbus-sh:${purpose}:sealed-json:v2`);
+}

package/src/_shared/esbuild-facet-shim.ts

@@ -0,0 +1,22 @@
+/**
+ * esbuild's `__name` helper, for source injected into a facet isolate.
+ *
+ * wrangler bundles the supervisor with esbuild `keepNames`, which wraps every
+ * named function and named arrow as `__name(fn, "fn")`. Code we ship into a
+ * facet by `.toString()` therefore carries bare `__name` references, and the
+ * supervisor's module-local binding does NOT cross the isolate boundary — the
+ * facet must provide one or the embed dies with "__name is not defined".
+ *
+ * Assigned on `globalThis` rather than declared with `const` because these
+ * injection sites do not own their scope: the pyodide asm.js inlined next to
+ * it already declares `__name`/`__defProp` (a `const` would be "already
+ * declared"), and the W7 facet frame evaluates the embed elsewhere. A bare
+ * `__name` with no lexical binding resolves to the global either way, and the
+ * `typeof` guard keeps it idempotent when several embeds share one isolate.
+ *
+ * Unit tests import the un-bundled TS source, so only the deployed,
+ * esbuild-bundled worker ever needs this.
+ */
+export const ESBUILD_NAME_GLOBAL_SHIM = `if (typeof globalThis.__name !== "function") {
+  globalThis.__name = (target, value) => Object.defineProperty(target, "name", { value, configurable: true });
+}`;