@nimbus-sh/core 0.1.0

package/dist/observability/diag-counters.d.ts

@@ -0,0 +1,236 @@
+/**
+ * diag-counters.ts — application-level memory + phase observability.
+ *
+ * Why: workerd's `process.memoryUsage()` returns 0 for all fields inside
+ * a Durable Object class context (only dynamic-worker isolates under
+ * nodejs_compat get the real implementation). Without a working heap
+ * probe we have no way to confirm OOM hypotheses or verify a fix.
+ *
+ * Replacement: deterministic counters bumped at known allocation sites.
+ * Cumulative bytes decoded, in-flight stub counts, and phase markers
+ * give us the same operational signal as a heap probe — and they're
+ * exact, not estimated.
+ *
+ * Singleton-per-isolate. Lives at module scope so any code path in the
+ * supervisor bundle can write to it (installer and RPC handlers) and
+ * the request handler in nimbus-session.ts:/api/_diag/memory can read
+ * it. Survives across requests within the same isolate; resets on DO
+ * reboot — itself a useful signal (counters at 0 immediately after the
+ * banner reprinted = the killed isolate took its state with it).
+ */
+/**
+ * Phase tags surfaced via /api/_diag/memory. Strings are ASCII so
+ * humans can grep for them in logs.
+ *
+ * installPhase tracks the high-level npm-install state machine:
+ *   idle → resolve → hoist → diff → fetch → write → link-bins → bundle → done
+ *
+ */
+import type { InstallPhase } from '../_shared/install-phase.js';
+export interface DiagCounters {
+    /** Top-level install phase. */
+    installPhase: InstallPhase;
+    /**
+     * Sum of payload bytes claimed by RPCs currently in flight on the
+     * SUPERVISOR boundary [Phase 2 A'.2].
+     *
+     * Bumped at RPC entry by `rpcPayloadStart(bytes)` in the DO-side
+     * byte-returning handlers in src/session/rpc.ts; debited at exit by
+     * `rpcPayloadEnd(bytes)`. Goes back to zero after every RPC settles.
+     *
+     * The C'.1 heap estimator surfaces this as
+     * `breakdown.streamingBuffersBytes` so any RPC path that buffers
+     * MORE than the W7 streaming guarantee shows up as supervisor heap
+     * pressure rather than disappearing into the bundle baseline.
+     *
+     * Streamed payloads (writeBatchStream over W7 frames) report -1
+     * up-front; we substitute 0 here because the bytes flow with
+     * backpressure and the supervisor-resident buffer is bounded by
+     * the W7 chunk size (a few KiB), not by the total payload.
+     */
+    inFlightRpcPayloadBytes: number;
+    /**
+     * Bytes reserved by prefetch-bundle builds currently in flight.
+     *
+     * `buildPrefetchBundle` accumulates raw VFS file contents into an in-heap
+     * bundle object, and used to do so with nothing observing it: the estimator
+     * read 9.4 MiB while those bytes were resetting the DO. Each build now takes
+     * an `acquireSupervisorAllocation` lease for the budget it may use and bumps
+     * this alongside it, so the reservation is ATTRIBUTED to the bundle path
+     * rather than landing in `unattributedReservationBytes`.
+     *
+     * Surfaced as `breakdown.prefetchBundleBytes`. Back to zero between builds.
+     */
+    prefetchBundleBytes: number;
+    /**
+     * Bytes retained by the FacetManager's prefetch-bundle LRU right now.
+     *
+     * Unlike the counter above these bytes persist ACROSS execs — each entry
+     * holds the raw bundle plus its serialized source, manifest and metadata.
+     * The LRU used to be bounded by entry count alone, which bounded nothing:
+     * 16 pi-sized entries exceed the supervisor ceiling several times over. It
+     * is now bounded by PREFETCH_CACHE_MAX_BYTES and reports its live total
+     * here, so the retained cost is a measurement rather than a cap.
+     *
+     * Surfaced as `breakdown.prefetchCacheBytes`. A gauge, not a delta.
+     */
+    prefetchCacheBytes: number;
+    /** Install-facet counters. Populated by npm-installer after a
+     *  successful batch-facet dispatch returns. Confirms the install ran
+     *  in the facet (tarballsCompleted > 0) and surfaces the
+     *  facet-internal byte count. */
+    installFacet: {
+        /** Number of tarballs the install-batch-facet successfully streamed. */
+        tarballsCompleted: number;
+        /** Cumulative tarball body bytes the facet decoded (gunzip input). */
+        cumulativeBytesDecoded: number;
+        /** Peak in-flight tarball pipelines inside the facet at any one moment. */
+        peakInFlight: number;
+    };
+    /** Pre-bundle facet counters. Tracks the fire-and-forget pre-bundle
+     *  phase. Aggregates across all dispatches in this DO's lifetime. */
+    preBundleFacet: {
+        /** Specifiers attempted (queue depth at start of phase). */
+        attempted: number;
+        /** Bundles that completed successfully (esmCode produced). */
+        bundlesCompleted: number;
+        /** Bundles that failed (errorText set on PrebundleResult). */
+        errors: number;
+        /** Bundles skipped before facet dispatch (e.g. slice cap exceeded). */
+        skipped: number;
+        /** First-call wasm-fetch payload size. 0 until first successful dispatch
+         *  records it; stays at first-seen value (RPC returns the same Module
+         *  every time, but the cumulativeBytes the FACET reports may grow on
+         *  retries). */
+        wasmBootBytes: number;
+        /** Per-module errors from the MOST RECENT pre-bundle batch, keyed by
+         *  specifier (e.g. "lucide-react", "framer-motion"). REPLACED on each
+         *  recordPreBundleSummary() call so the map is bounded by the batch
+         *  size (≤ pending.length, typically <20).
+         *
+         *  Example: { "lucide-react": "Worker exceeded memory limit." }
+         *
+         *  Empty when the most recent batch had zero errors. */
+        errorsByModule: Record<string, string>;
+    };
+    /** Pipelined R2 race outcomes. All counts are cumulative since
+     *  DO-isolate start. Per-tier cache statistics live in cache-stats. */
+    r2: {
+        /** Pipelined-RPC race wins for tarballs (R2 came back first; network
+         *  was cancelled). */
+        pipelinedTarballRaceWins: number;
+        /** Pipelined-RPC race losses for tarballs (R2 came back too slow or
+         *  empty; network response used). */
+        pipelinedTarballRaceLosses: number;
+        /** Pipelined-RPC race wins for packuments. */
+        pipelinedPackumentRaceWins: number;
+        /** Pipelined-RPC race losses for packuments. */
+        pipelinedPackumentRaceLosses: number;
+    };
+}
+/** Read a snapshot — caller-side mutations don't affect the singleton. */
+export declare function readDiagCounters(): DiagCounters;
+/** Set the install phase. */
+export declare function setInstallPhase(p: InstallPhase): void;
+/**
+ * Track an in-flight supervisor RPC payload [Phase 2 A'.2].
+ *
+ * Call at RPC entry, BEFORE awaiting any work that depends on
+ * `payload`. Pair with `rpcPayloadEnd(bytes)` in the matching `finally`
+ * so the counter goes back to zero on both success and failure.
+ *
+ * `bytes` is the supervisor-resident byte cost of the RPC's argument
+ * (or return value, whichever is bigger). For structured-clone RPCs
+ * this is the size of the cloned payload in bytes.
+ *
+ * Streamed payloads (ReadableStream-over-RPC) flow with backpressure
+ * and the supervisor-resident bound is the chunk size, not the total
+ * payload. Pass the chunk-size estimate (typically ≤ 1 MiB) — never
+ * the unknown total. -1 is silently coerced to 0 to keep the counter
+ * non-negative; callers that don't know the size should call this
+ * with 0 explicitly rather than relying on coercion.
+ */
+export declare function rpcPayloadStart(bytes: number): void;
+/**
+ * Release an in-flight supervisor RPC payload [Phase 2 A'.2].
+ * Pass the same byte count given to the matching `rpcPayloadStart`.
+ *
+ * Floors at 0 to absorb arithmetic drift (e.g. if a payload-byte
+ * counter rounding gave a slightly different number on entry vs.
+ * exit). Drift in routine paths should be zero — a non-zero floor
+ * hit is a bug worth investigating.
+ */
+export declare function rpcPayloadEnd(bytes: number): void;
+/**
+ * Track an in-flight prefetch-bundle build. Call with the byte reservation the
+ * build took from the supervisor allocation budget, and pair with
+ * `prefetchBundleEnd` in the matching `finally` so it returns to zero whether
+ * the build succeeded or threw.
+ */
+export declare function prefetchBundleStart(bytes: number): void;
+/** Release an in-flight prefetch-bundle reservation. Floors at 0. */
+export declare function prefetchBundleEnd(bytes: number): void;
+/**
+ * Publish the prefetch-bundle LRU's live retained total. A gauge: the cache
+ * owns the number and reports it whenever it admits or evicts an entry.
+ */
+export declare function setPrefetchCacheBytes(bytes: number): void;
+/** Fold facet-returned counters into the supervisor's diag state.
+ *  Called by npm-installer after the batch-facet returns; aggregates
+ *  rather than replaces so multiple install runs in the same DO
+ *  lifetime accumulate in cumulativeBytesDecoded. */
+export declare function recordInstallFacetCounters(c: {
+    tarballsCompleted: number;
+    cumulativeBytesDecoded: number;
+    peakInFlight: number;
+}): void;
+/** Record pre-bundle phase summary. Aggregates lifetime totals across
+ *  all phases run in this DO's lifetime, but REPLACES the
+ *  errorsByModule map every call so it reflects only the most recent
+ *  batch. Losing prior-batch errors is fine — they're already aggregated
+ *  into `errors` (count); the map is for "which modules failed this
+ *  time." */
+export declare function recordPreBundleSummary(s: {
+    attempted: number;
+    bundlesCompleted: number;
+    errors: number;
+    skipped: number;
+    wasmBootBytes?: number;
+    errorsByModule?: Record<string, string>;
+}): void;
+/** Bump pipelined-RPC race outcome counters. The facet returns these
+ *  in its result counters; the supervisor folds them in alongside the
+ *  existing installFacet counters. */
+export declare function recordR2RaceCounters(c: {
+    pipelinedTarballRaceWins: number;
+    pipelinedTarballRaceLosses: number;
+    pipelinedPackumentRaceWins: number;
+    pipelinedPackumentRaceLosses: number;
+}): void;
+/**
+ * cache-obs-2: fold facet-collected per-tier cache events into the
+ * DO-side cache-stats singleton. Called from installer.ts after a
+ * batch-facet / resolve-facet returns — mirrors recordR2RaceCounters
+ * (the facet collects metrics in its result and the supervisor folds
+ * them into the DO isolate).
+ *
+ * Each event has shape:
+ *   { kind: 'hit', tier: 'L2'|'L3'|'L4', cacheKind: 'tarball'|'packument'|'asset', bytes: number }
+ *   { kind: 'miss', tier: ..., cacheKind: ... }
+ *
+ * The recordHit/recordMiss surface is defined in
+ * src/_shared/cache-stats.ts.
+ */
+import { type CacheTier, type CacheKind } from '../_shared/cache-stats.js';
+export type CacheStatEvent = {
+    kind: 'hit';
+    tier: CacheTier;
+    cacheKind: CacheKind;
+    bytes: number;
+} | {
+    kind: 'miss';
+    tier: CacheTier;
+    cacheKind: CacheKind;
+};
+export declare function recordCacheStatEvents(events: readonly CacheStatEvent[] | undefined): void;
+//# sourceMappingURL=diag-counters.d.ts.map

package/dist/observability/diag-counters.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diag-counters.d.ts","sourceRoot":"","sources":["../../src/observability/diag-counters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;;;GAOG;AAMH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAEhE,MAAM,WAAW,YAAY;IAC3B,+BAA+B;IAC/B,YAAY,EAAE,YAAY,CAAC;IAC3B;;;;;;;;;;;;;;;;;OAiBG;IACH,uBAAuB,EAAE,MAAM,CAAC;IAChC;;;;;;;;;;;OAWG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;;;;;;;OAWG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;qCAGiC;IACjC,YAAY,EAAE;QACZ,wEAAwE;QACxE,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sEAAsE;QACtE,sBAAsB,EAAE,MAAM,CAAC;QAC/B,2EAA2E;QAC3E,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,cAAc,EAAE;QACd,4DAA4D;QAC5D,SAAS,EAAE,MAAM,CAAC;QAClB,8DAA8D;QAC9D,gBAAgB,EAAE,MAAM,CAAC;QACzB,8DAA8D;QAC9D,MAAM,EAAE,MAAM,CAAC;QACf,uEAAuE;QACvE,OAAO,EAAE,MAAM,CAAC;QAChB;;;wBAGgB;QAChB,aAAa,EAAE,MAAM,CAAC;QACtB;;;;;;;gEAOwD;QACxD,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACxC,CAAC;IACF;2EACuE;IACvE,EAAE,EAAE;QACF;8BACsB;QACtB,wBAAwB,EAAE,MAAM,CAAC;QACjC;6CACqC;QACrC,0BAA0B,EAAE,MAAM,CAAC;QACnC,8CAA8C;QAC9C,0BAA0B,EAAE,MAAM,CAAC;QACnC,gDAAgD;QAChD,4BAA4B,EAAE,MAAM,CAAC;KACtC,CAAC;CACH;AA4BD,0EAA0E;AAC1E,wBAAgB,gBAAgB,IAAI,YAAY,CAE/C;AAED,6BAA6B;AAC7B,wBAAgB,eAAe,CAAC,CAAC,EAAE,YAAY,GAAG,IAAI,CAErD;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAInD;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAOjD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAIvD;AAED,qEAAqE;AACrE,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAKrD;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAGzD;AAED;;;qDAGqD;AACrD,wBAAgB,0BAA0B,CAAC,CAAC,EAAE;IAC5C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,IAAI,CAMP;AAED;;;;;aAKa;AACb,wBAAgB,sBAAsB,CAAC,CAAC,EAAE;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACzC,GAAG,IAAI,CAiBP;AAED;;sCAEsC;AACtC,wBAAgB,oBAAoB,CAAC,CAAC,EAAE;IACtC,wBAAwB,EAAE,MAAM,CAAC;IACjC,0BAA0B,EAAE,MAAM,CAAC;IACnC,0BAA0B,EAAE,MAAM,CAAC;IACnC,4BAA4B,EAAE,MAAM,CAAC;CACtC,GAAG,IAAI,CAKP;AAED;;;;;;;;;;;;;GAaG;AACH,OAAO,EAGL,KAAK,SAAS,EACd,KAAK,SAAS,EACf,MAAM,2BAA2B,CAAC;AAEnC,MAAM,MAAM,cAAc,GACtB;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC;IAAC,SAAS,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,CAAC;AAE5D,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,SAAS,cAAc,EAAE,GAAG,SAAS,GAAG,IAAI,CASzF"}

package/dist/observability/diag-counters.js

@@ -0,0 +1,195 @@
+/**
+ * diag-counters.ts — application-level memory + phase observability.
+ *
+ * Why: workerd's `process.memoryUsage()` returns 0 for all fields inside
+ * a Durable Object class context (only dynamic-worker isolates under
+ * nodejs_compat get the real implementation). Without a working heap
+ * probe we have no way to confirm OOM hypotheses or verify a fix.
+ *
+ * Replacement: deterministic counters bumped at known allocation sites.
+ * Cumulative bytes decoded, in-flight stub counts, and phase markers
+ * give us the same operational signal as a heap probe — and they're
+ * exact, not estimated.
+ *
+ * Singleton-per-isolate. Lives at module scope so any code path in the
+ * supervisor bundle can write to it (installer and RPC handlers) and
+ * the request handler in nimbus-session.ts:/api/_diag/memory can read
+ * it. Survives across requests within the same isolate; resets on DO
+ * reboot — itself a useful signal (counters at 0 immediately after the
+ * banner reprinted = the killed isolate took its state with it).
+ */
+const _counters = {
+    installPhase: 'idle',
+    inFlightRpcPayloadBytes: 0,
+    prefetchBundleBytes: 0,
+    prefetchCacheBytes: 0,
+    installFacet: {
+        tarballsCompleted: 0,
+        cumulativeBytesDecoded: 0,
+        peakInFlight: 0,
+    },
+    preBundleFacet: {
+        attempted: 0,
+        bundlesCompleted: 0,
+        errors: 0,
+        skipped: 0,
+        wasmBootBytes: 0,
+        errorsByModule: {},
+    },
+    r2: {
+        pipelinedTarballRaceWins: 0,
+        pipelinedTarballRaceLosses: 0,
+        pipelinedPackumentRaceWins: 0,
+        pipelinedPackumentRaceLosses: 0,
+    },
+};
+/** Read a snapshot — caller-side mutations don't affect the singleton. */
+export function readDiagCounters() {
+    return { ..._counters };
+}
+/** Set the install phase. */
+export function setInstallPhase(p) {
+    _counters.installPhase = p;
+}
+/**
+ * Track an in-flight supervisor RPC payload [Phase 2 A'.2].
+ *
+ * Call at RPC entry, BEFORE awaiting any work that depends on
+ * `payload`. Pair with `rpcPayloadEnd(bytes)` in the matching `finally`
+ * so the counter goes back to zero on both success and failure.
+ *
+ * `bytes` is the supervisor-resident byte cost of the RPC's argument
+ * (or return value, whichever is bigger). For structured-clone RPCs
+ * this is the size of the cloned payload in bytes.
+ *
+ * Streamed payloads (ReadableStream-over-RPC) flow with backpressure
+ * and the supervisor-resident bound is the chunk size, not the total
+ * payload. Pass the chunk-size estimate (typically ≤ 1 MiB) — never
+ * the unknown total. -1 is silently coerced to 0 to keep the counter
+ * non-negative; callers that don't know the size should call this
+ * with 0 explicitly rather than relying on coercion.
+ */
+export function rpcPayloadStart(bytes) {
+    const n = Number(bytes);
+    if (!Number.isFinite(n) || n <= 0)
+        return;
+    _counters.inFlightRpcPayloadBytes += n;
+}
+/**
+ * Release an in-flight supervisor RPC payload [Phase 2 A'.2].
+ * Pass the same byte count given to the matching `rpcPayloadStart`.
+ *
+ * Floors at 0 to absorb arithmetic drift (e.g. if a payload-byte
+ * counter rounding gave a slightly different number on entry vs.
+ * exit). Drift in routine paths should be zero — a non-zero floor
+ * hit is a bug worth investigating.
+ */
+export function rpcPayloadEnd(bytes) {
+    const n = Number(bytes);
+    if (!Number.isFinite(n) || n <= 0)
+        return;
+    _counters.inFlightRpcPayloadBytes -= n;
+    if (_counters.inFlightRpcPayloadBytes < 0) {
+        _counters.inFlightRpcPayloadBytes = 0;
+    }
+}
+/**
+ * Track an in-flight prefetch-bundle build. Call with the byte reservation the
+ * build took from the supervisor allocation budget, and pair with
+ * `prefetchBundleEnd` in the matching `finally` so it returns to zero whether
+ * the build succeeded or threw.
+ */
+export function prefetchBundleStart(bytes) {
+    const n = Number(bytes);
+    if (!Number.isFinite(n) || n <= 0)
+        return;
+    _counters.prefetchBundleBytes += n;
+}
+/** Release an in-flight prefetch-bundle reservation. Floors at 0. */
+export function prefetchBundleEnd(bytes) {
+    const n = Number(bytes);
+    if (!Number.isFinite(n) || n <= 0)
+        return;
+    _counters.prefetchBundleBytes -= n;
+    if (_counters.prefetchBundleBytes < 0)
+        _counters.prefetchBundleBytes = 0;
+}
+/**
+ * Publish the prefetch-bundle LRU's live retained total. A gauge: the cache
+ * owns the number and reports it whenever it admits or evicts an entry.
+ */
+export function setPrefetchCacheBytes(bytes) {
+    const n = Number(bytes);
+    _counters.prefetchCacheBytes = Number.isFinite(n) && n > 0 ? n : 0;
+}
+/** Fold facet-returned counters into the supervisor's diag state.
+ *  Called by npm-installer after the batch-facet returns; aggregates
+ *  rather than replaces so multiple install runs in the same DO
+ *  lifetime accumulate in cumulativeBytesDecoded. */
+export function recordInstallFacetCounters(c) {
+    _counters.installFacet.tarballsCompleted += c.tarballsCompleted;
+    _counters.installFacet.cumulativeBytesDecoded += c.cumulativeBytesDecoded;
+    if (c.peakInFlight > _counters.installFacet.peakInFlight) {
+        _counters.installFacet.peakInFlight = c.peakInFlight;
+    }
+}
+/** Record pre-bundle phase summary. Aggregates lifetime totals across
+ *  all phases run in this DO's lifetime, but REPLACES the
+ *  errorsByModule map every call so it reflects only the most recent
+ *  batch. Losing prior-batch errors is fine — they're already aggregated
+ *  into `errors` (count); the map is for "which modules failed this
+ *  time." */
+export function recordPreBundleSummary(s) {
+    _counters.preBundleFacet.attempted += s.attempted;
+    _counters.preBundleFacet.bundlesCompleted += s.bundlesCompleted;
+    _counters.preBundleFacet.errors += s.errors;
+    _counters.preBundleFacet.skipped += s.skipped;
+    if (s.wasmBootBytes && _counters.preBundleFacet.wasmBootBytes === 0) {
+        _counters.preBundleFacet.wasmBootBytes = s.wasmBootBytes;
+    }
+    // Replace (not merge) so we capture only the most recent batch.
+    // Bounded by batch size; truncate each value to 200 chars.
+    if (s.errorsByModule) {
+        const trimmed = {};
+        for (const [name, msg] of Object.entries(s.errorsByModule)) {
+            trimmed[name] = String(msg).slice(0, 200);
+        }
+        _counters.preBundleFacet.errorsByModule = trimmed;
+    }
+}
+/** Bump pipelined-RPC race outcome counters. The facet returns these
+ *  in its result counters; the supervisor folds them in alongside the
+ *  existing installFacet counters. */
+export function recordR2RaceCounters(c) {
+    _counters.r2.pipelinedTarballRaceWins += c.pipelinedTarballRaceWins;
+    _counters.r2.pipelinedTarballRaceLosses += c.pipelinedTarballRaceLosses;
+    _counters.r2.pipelinedPackumentRaceWins += c.pipelinedPackumentRaceWins;
+    _counters.r2.pipelinedPackumentRaceLosses += c.pipelinedPackumentRaceLosses;
+}
+/**
+ * cache-obs-2: fold facet-collected per-tier cache events into the
+ * DO-side cache-stats singleton. Called from installer.ts after a
+ * batch-facet / resolve-facet returns — mirrors recordR2RaceCounters
+ * (the facet collects metrics in its result and the supervisor folds
+ * them into the DO isolate).
+ *
+ * Each event has shape:
+ *   { kind: 'hit', tier: 'L2'|'L3'|'L4', cacheKind: 'tarball'|'packument'|'asset', bytes: number }
+ *   { kind: 'miss', tier: ..., cacheKind: ... }
+ *
+ * The recordHit/recordMiss surface is defined in
+ * src/_shared/cache-stats.ts.
+ */
+import { recordHit as _cacheRecordHit, recordMiss as _cacheRecordMiss, } from '../_shared/cache-stats.js';
+export function recordCacheStatEvents(events) {
+    if (!events || events.length === 0)
+        return;
+    for (const e of events) {
+        if (e.kind === 'hit') {
+            _cacheRecordHit(e.tier, e.cacheKind, e.bytes);
+        }
+        else {
+            _cacheRecordMiss(e.tier, e.cacheKind);
+        }
+    }
+}

package/dist/observability/heap-estimate.d.ts

@@ -0,0 +1,225 @@
+/**
+ * heap-estimate.ts — deterministic supervisor-heap estimator [C'.1]
+ *
+ * Why this module exists
+ * ──────────────────────
+ * `process.memoryUsage()` returns 0 for every field inside a Durable
+ * Object class context (only dynamic-worker isolates under nodejs_compat
+ * get the real implementation). The previous `readNodeMem` /
+ * `sampleMemory` helpers in nimbus-session-diag.ts called it anyway and
+ * therefore reported zero forever — useless for verifying memory-
+ * containment work in plan §3 Track A'.
+ *
+ * Replacement: a deterministic estimator that sums the INSTRUMENTED
+ * supervisor heap allocation sources — runtime counters that ARE accurate
+ * (diag-counters.ts singleton + SqliteVFS.getStats()). Every byte it
+ * reports has a named contributor, so a regression in any one component
+ * is locatable.
+ *
+ * `estimatedBytes` is a LOWER BOUND, not a total
+ * ─────────────────────────────────────────────
+ * Within each instrumented component the estimator over-reports on
+ * purpose: components carry a peak-or-current value, in-flight bytes are
+ * counted before they are freed, and LRU bytes are the cap rather than the
+ * live footprint.
+ *
+ * Across components it can only sum what is instrumented. Allocation sites
+ * that neither bump a diag counter nor take a lease via
+ * `acquireSupervisorAllocation` are invisible to it, and it has no way to
+ * infer their size — so any site known to be uninstrumented is named in
+ * HEAP_BLIND_SPOTS rather than left silently absent.
+ *
+ * The prefetch-bundle path used to be exactly that gap, and a large one: the
+ * supervisor DO was reset three times under bundle construction while this
+ * estimator reported a 9.4 MiB baseline and 35 KB of LRU. Both of its sites
+ * are now instrumented where they allocate — the per-exec build leases the
+ * budget it spends (`prefetchBundleBytes`) and the cross-exec cache is bounded
+ * by bytes and reports its retained total (`prefetchCacheBytes`) — so they are
+ * components of the breakdown rather than caveats beside it.
+ *
+ * `blindSpots` is still returned alongside `estimatedBytes` because a list
+ * being empty is a claim that has to be re-earned as the system grows, not a
+ * property of the estimator. Closing a blind spot always means instrumenting
+ * it at its own allocation site and adding it to the breakdown — never
+ * estimating it from here.
+ *
+ * Eviction-label taxonomy
+ * ───────────────────────
+ * Workerd distinguishes five labelled eviction reasons:
+ *   - lru                       → memory pressure on the runtime process
+ *   - condemned                 → kill (operator / abuse pipeline)
+ *   - inactive                  → idle eviction (70-140 s of no traffic)
+ *   - dynamic_worker            → per-owner LRU cap (default 50)
+ *   - dynamic_worker_banned     → Dice abuse-detection ban
+ *
+ * Nimbus surfaces the labels here so any tool that reads
+ * /api/_diag/memory has a fixed, well-known taxonomy to count against.
+ * The actual count of evictions Nimbus has observed lives in the C'.2
+ * recovery_event ring, separate from this module.
+ */
+import type { DiagCounters } from './diag-counters.js';
+import { type SupervisorAllocationBudgetStats } from './heavy-alloc-coord.js';
+/**
+ * Five labelled workerd eviction reasons. Surfaced as a constant
+ * taxonomy in /api/_diag/memory so any consumer can count observed
+ * events against the well-known set.
+ *
+ */
+export declare const WORKERD_EVICTION_LABELS: readonly ["lru", "condemned", "inactive", "dynamic_worker", "dynamic_worker_banned"];
+/**
+ * A supervisor allocation site the estimator cannot see.
+ *
+ * Listing one here is a statement of fact about instrumentation, not a
+ * measurement: the site allocates on the supervisor heap without bumping a
+ * diag counter or taking an `acquireSupervisorAllocation` lease, so its
+ * live size is unknown to this module. Only `capBytes` — the bound the
+ * site enforces on itself — is knowable from here.
+ */
+export interface HeapBlindSpot {
+    /** Module-qualified allocation site. */
+    source: string;
+    /**
+     * Upper bound the site enforces per retained item, or null when nothing
+     * bounds its total bytes. A null here is the important case: it means no
+     * finite worst case can be stated for the supervisor at all.
+     */
+    capBytes: number | null;
+    /** What allocates, and why the estimator cannot observe it. */
+    reason: string;
+}
+/**
+ * Supervisor allocation sites known to be unaccounted for.
+ *
+ * Empty: both former entries were the prefetch-bundle path in
+ * facets/manager.ts, and both are now instrumented at their own allocation
+ * sites rather than described from here. `buildPrefetchBundle` takes an
+ * `acquireSupervisorAllocation` lease and reports
+ * `breakdown.prefetchBundleBytes`; `prefetchBundleCache` is bounded by
+ * PREFETCH_CACHE_MAX_BYTES instead of by entry count and reports its live
+ * retained total as `breakdown.prefetchCacheBytes`.
+ *
+ * An empty list says nothing is CURRENTLY known to be missing — it is not a
+ * proof of completeness, which no list can be. Adding an entry is how a newly
+ * discovered gap stays visible in /api/_diag/memory until it is closed the
+ * same way: instrumented where it allocates, then folded into HeapBreakdown.
+ */
+export declare const HEAP_BLIND_SPOTS: readonly HeapBlindSpot[];
+/**
+ * Supervisor heap snapshot. Components are PEAK-OR-CURRENT bytes from
+ * runtime counters; the estimator never calls process.memoryUsage().
+ */
+export interface HeapEstimate {
+    /**
+     * Sum of the instrumented components below. Always equals the breakdown
+     * sum, and is a LOWER BOUND on supervisor heap — see `blindSpots`.
+     */
+    estimatedBytes: number;
+    /** SUPERVISOR_HEAP_CEILING_BYTES (constant — 64 MiB by design). */
+    ceilingBytes: number;
+    /**
+     * estimatedBytes / ceilingBytes × 100, one decimal place. Shares
+     * estimatedBytes' lower-bound caveat: a low value does NOT mean the
+     * supervisor has headroom, only that instrumented sources are small.
+     */
+    percentOfCeiling: number;
+    /** Per-source byte attribution for the instrumented sources. */
+    breakdown: HeapBreakdown;
+    /**
+     * Allocation sites this estimate does not cover. Non-empty means
+     * `estimatedBytes` cannot be read as a total. See HEAP_BLIND_SPOTS.
+     */
+    blindSpots: readonly HeapBlindSpot[];
+    /**
+     * Worst-case bytes the blind spots could add, or null when any of them
+     * is unbounded — in which case no finite worst case exists and only
+     * direct instrumentation can answer the headroom question.
+     */
+    blindSpotCeilingBytes: number | null;
+    /**
+     * Shared transient-allocation budget occupancy. Named contributors overlap
+     * this total; any otherwise-unattributed occupancy is included in
+     * breakdown.unattributedReservationBytes.
+     */
+    allocationBudget: SupervisorAllocationBudgetStats;
+}
+export interface HeapBreakdown {
+    /** Static module bundle + runtime baseline. Constant per build. */
+    supervisorBaselineBytes: number;
+    /** SqliteVFS LRU cache hot-byte count (sqlite-vfs.ts cache.hotBytes). */
+    vfsLruBytes: number;
+    /** SqliteVFS current logical retained write payload bytes. */
+    vfsInFlightBytes: number;
+    /** Pre-bundle slice bytes resident in supervisor heap (peak across
+     *  in-flight pool slots × SLICE_CAP_BYTES). Drops to 0 once
+     *  A'.2/A'.3 stream the slice through ReadableStream-over-RPC. */
+    preBundleSliceBytes: number;
+    /**
+     * In-flight supervisor RPC payload bytes (Phase 2 A'.2).
+     *
+     * Sum of bytes claimed by DO-side byte-returning filesystem RPC handlers
+     * between payload allocation and exit. Tracked by
+     * `inFlightRpcPayloadBytes` in src/observability/diag-counters.ts; bumped
+     * after shared-budget admission and debited in `finally`.
+     *
+     * At idle this is 0. Under load it should stay bounded by the
+     * shared allocation budget.
+     * Persistent non-zero readings here mean an RPC handler isn't
+     * decrementing on its failure path — a leak worth fixing.
+     */
+    streamingBuffersBytes: number;
+    /**
+     * Bytes reserved by prefetch-bundle builds in flight (diag-counters
+     * `prefetchBundleBytes`). Transient: each build leases the enrichment budget
+     * for its duration and releases it on the way out, success or failure.
+     */
+    prefetchBundleBytes: number;
+    /**
+     * Bytes retained by the FacetManager's prefetch-bundle LRU (diag-counters
+     * `prefetchCacheBytes`). Unlike every other component here these persist
+     * across execs, and unlike the transient ones they are a measurement of what
+     * the cache holds rather than a reservation it might use.
+     */
+    prefetchCacheBytes: number;
+    /**
+     * Shared-budget occupancy not already represented by the named transient
+     * counters. This covers full-budget owners and keeps cross-DO module-local
+     * reservations visible without double-counting read/write payloads.
+     */
+    unattributedReservationBytes: number;
+}
+/**
+ * Inputs the estimator needs from the SqliteVFS layer. Kept narrow
+ * (just the two fields we actually consume) to avoid a circular import
+ * between observability/ and the VFS module.
+ */
+export interface VfsHeapInputs {
+    /** SqliteVFS.getStats().cache.hotBytes — actual LRU memory in use. */
+    cacheHotBytes: number;
+    /** Current sum of in-flight write payloads (or 0 if none). */
+    inFlightWriteBytes: number;
+}
+/**
+ * Estimate pre-bundle slice bytes resident in the supervisor heap.
+ *
+ * Pre A'.2: the supervisor builds the slice in heap (up to
+ * SLICE_CAP_BYTES = 28 MiB per concurrent slot) before passing to the
+ * pre-bundle facet. With PRE_BUNDLE_CONCURRENCY = 1 the cap is 28 MiB.
+ *
+ * Diag counters track pre-bundle attempts/completions but not
+ * in-flight slice bytes specifically. Until A'.2 streams the slice,
+ * we treat the slice as PRESENT in supervisor heap iff the pre-bundle
+ * phase is active AND a slot is in flight (no completion yet for the
+ * current attempted count).
+ *
+ * After A'.2 lands the slice flows via ReadableStream-over-RPC and
+ * supervisor never holds the bytes — this contribution drops to 0.
+ */
+export declare function estimatePreBundleSliceBytes(c: DiagCounters): number;
+/**
+ * Build a heap estimate from runtime counters + VFS inputs.
+ *
+ * Deterministic and I/O-free. Called from the /api/_diag/memory request
+ * handler; the allocation-budget snapshot is process-local state.
+ */
+export declare function estimateSupervisorHeap(c: DiagCounters, vfs: VfsHeapInputs): HeapEstimate;
+//# sourceMappingURL=heap-estimate.d.ts.map

package/dist/observability/heap-estimate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"heap-estimate.d.ts","sourceRoot":"","sources":["../../src/observability/heap-estimate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AAOH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,wBAAwB,CAAC;AAEhC;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,sFAM1B,CAAC;AAEX;;;;;;;;GAQG;AACH,MAAM,WAAW,aAAa;IAC5B,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,aAAa,EAAO,CAAC;AAE7D;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB,mEAAmE;IACnE,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,gEAAgE;IAChE,SAAS,EAAE,aAAa,CAAC;IACzB;;;OAGG;IACH,UAAU,EAAE,SAAS,aAAa,EAAE,CAAC;IACrC;;;;OAIG;IACH,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC;;;;OAIG;IACH,gBAAgB,EAAE,+BAA+B,CAAC;CACnD;AAED,MAAM,WAAW,aAAa;IAC5B,mEAAmE;IACnE,uBAAuB,EAAE,MAAM,CAAC;IAChC,yEAAyE;IACzE,WAAW,EAAE,MAAM,CAAC;IACpB,8DAA8D;IAC9D,gBAAgB,EAAE,MAAM,CAAC;IACzB;;sEAEkE;IAClE,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;;;;;;;;OAYG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;OAIG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,4BAA4B,EAAE,MAAM,CAAC;CACtC;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,8DAA8D;IAC9D,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAyBD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,2BAA2B,CAAC,CAAC,EAAE,YAAY,GAAG,MAAM,CAQnE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,CAAC,EAAE,YAAY,EACf,GAAG,EAAE,aAAa,GACjB,YAAY,CAkDd"}