@prisma/streams-server 0.1.1 → 0.1.3

package/src/index/secondary_schema.ts

@@ -0,0 +1,105 @@
+import { createHash } from "node:crypto";
+import { Result } from "better-result";
+import type { SchemaRegistry, SearchFieldConfig } from "../schema/registry";
+import { canonicalizeExactValue, extractSearchExactTermsResult, extractSearchExactValuesResult, getSearchFieldBinding } from "../search/schema";
+import { schemaVersionForOffset } from "../schema/read_json";
+import { resolvePointerResult } from "../util/json_pointer";
+
+export type SecondaryIndexField = {
+  name: string;
+  config: SearchFieldConfig;
+};
+
+export type SecondaryIndexTerm = {
+  index: SecondaryIndexField;
+  canonical: string;
+  bytes: Uint8Array;
+};
+
+function addRawValues(out: unknown[], value: unknown): void {
+  if (Array.isArray(value)) {
+    for (const item of value) addRawValues(out, item);
+    return;
+  }
+  out.push(value);
+}
+
+export function getConfiguredSecondaryIndexes(registry: SchemaRegistry): SecondaryIndexField[] {
+  const search = registry.search;
+  if (!search) return [];
+  return Object.entries(search.fields)
+    .filter(([, config]) => config.exact === true)
+    .map(([name, config]) => ({ name, config }));
+}
+
+export function hashSecondaryIndexField(index: SecondaryIndexField): string {
+  return createHash("sha256")
+    .update(
+      JSON.stringify({
+        name: index.name,
+        kind: index.config.kind,
+        bindings: index.config.bindings,
+        normalizer: index.config.normalizer ?? null,
+        analyzer: index.config.analyzer ?? null,
+        exact: index.config.exact === true,
+      })
+    )
+    .digest("hex");
+}
+
+export function canonicalizeSecondaryIndexValue(config: SearchFieldConfig, value: unknown): string | null {
+  return canonicalizeExactValue(config, value);
+}
+
+export function extractSecondaryIndexTermsResult(
+  registry: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<SecondaryIndexTerm[], { message: string }> {
+  const termsRes = extractSearchExactTermsResult(registry, offset, value);
+  if (Result.isError(termsRes)) return termsRes;
+  return Result.ok(
+    termsRes.value.map((term) => ({
+      index: { name: term.field, config: term.config },
+      canonical: term.canonical,
+      bytes: term.bytes,
+    }))
+  );
+}
+
+export function extractSecondaryIndexValuesResult(
+  registry: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<Map<string, string[]>, { message: string }> {
+  return extractSearchExactValuesResult(registry, offset, value);
+}
+
+export function extractSecondaryIndexValuesForFieldResult(
+  registry: SchemaRegistry,
+  offset: bigint,
+  value: unknown,
+  index: SecondaryIndexField
+): Result<string[], { message: string }> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return Result.err({ message: "search fields require JSON object records" });
+  }
+  const version = schemaVersionForOffset(registry, offset);
+  const binding = getSearchFieldBinding(index.config, version);
+  if (!binding) return Result.ok([]);
+  const resolvedRes = resolvePointerResult(value, binding.jsonPointer);
+  if (Result.isError(resolvedRes)) return Result.err({ message: resolvedRes.error.message });
+  if (!resolvedRes.value.exists) return Result.ok([]);
+
+  const rawValues: unknown[] = [];
+  addRawValues(rawValues, resolvedRes.value.value);
+  const out: string[] = [];
+  const seen = new Set<string>();
+  for (const rawValue of rawValues) {
+    const canonical = canonicalizeExactValue(index.config, rawValue);
+    if (canonical == null || seen.has(canonical)) continue;
+    seen.add(canonical);
+    out.push(canonical);
+  }
+  return Result.ok(out);
+}

package/src/ingest.ts

@@ -3,7 +3,6 @@ import type { SqliteDurableStore } from "./db/db";
 import { STREAM_FLAG_DELETED } from "./db/db";
 import type { StatsCollector } from "./stats";
 import type { BackpressureGate } from "./backpressure";
-import type { MemoryGuard } from "./memory";
 import type { Metrics } from "./metrics";
 import { Result } from "better-result";
 
@@ -55,7 +54,6 @@ export class IngestQueue {
   private readonly db: SqliteDurableStore;
   private readonly stats?: StatsCollector;
   private readonly gate?: BackpressureGate;
-  private readonly memory?: MemoryGuard;
   private readonly metrics?: Metrics;
   private readonly q: AppendTask[] = [];
   private timer: any | null = null;
@@ -73,12 +71,11 @@ export class IngestQueue {
     upsertProducerState: any;
   };
 
-  constructor(cfg: Config, db: SqliteDurableStore, stats?: StatsCollector, gate?: BackpressureGate, memory?: MemoryGuard, metrics?: Metrics) {
+  constructor(cfg: Config, db: SqliteDurableStore, stats?: StatsCollector, gate?: BackpressureGate, metrics?: Metrics) {
     this.cfg = cfg;
     this.db = db;
     this.stats = stats;
     this.gate = gate;
-    this.memory = memory;
     this.metrics = metrics;
 
     this.stmts = {
@@ -95,6 +92,7 @@ export class IngestQueue {
         `UPDATE streams
          SET next_offset=?, updated_at_ms=?, last_append_ms=?,
              pending_rows=pending_rows+?, pending_bytes=pending_bytes+?,
+             logical_size_bytes=logical_size_bytes+?,
              wal_rows=wal_rows+?, wal_bytes=wal_bytes+?,
              stream_seq=?,
              closed=CASE WHEN ? THEN 1 ELSE closed END,
@@ -149,14 +147,6 @@ export class IngestQueue {
     close?: boolean;
   }, opts?: { bypassBackpressure?: boolean; priority?: "high" | "normal" }): Promise<AppendResult> {
     const bytes = args.rows.reduce((acc, r) => acc + r.payload.byteLength, 0);
-    if (this.memory && !this.memory.shouldAllow()) {
-      this.memory.maybeGc("memory limit");
-      if (!opts?.bypassBackpressure) {
-        this.memory.maybeHeapSnapshot("memory limit");
-        if (this.metrics) this.metrics.record("tieredstore.backpressure.over_limit", 1, "count", { reason: "memory" });
-        return Promise.resolve(Result.err({ kind: "overloaded" }));
-      }
-    }
     if (!opts?.bypassBackpressure) {
       if (this.q.length >= this.cfg.ingestMaxQueueRequests || this.queuedBytes + bytes > this.cfg.ingestMaxQueueBytes) {
         if (this.metrics) this.metrics.record("tieredstore.backpressure.over_limit", 1, "count", { reason: "queue" });
@@ -208,6 +198,13 @@ export class IngestQueue {
     return { requests: this.q.length, bytes: this.queuedBytes };
   }
 
+  getMemoryStats(): { queuedPayloadBytes: number; queuedRequests: number } {
+    return {
+      queuedPayloadBytes: this.queuedBytes,
+      queuedRequests: this.q.length,
+    };
+  }
+
   isQueueFull(): boolean {
     return this.q.length >= this.cfg.ingestMaxQueueRequests || this.queuedBytes >= this.cfg.ingestMaxQueueBytes;
   }
@@ -560,6 +557,7 @@ export class IngestQueue {
           st.lastAppendMs,
           BigInt(task.rows.length),
           totalBytes,
+          totalBytes,
           BigInt(task.rows.length),
           totalBytes,
           st.streamSeq,

package/src/manifest.ts

@@ -1,6 +1,17 @@
 import { zstdCompressSync } from "node:zlib";
 import { Result } from "better-result";
-import type { IndexRunRow, IndexStateRow, SegmentMetaRow, StreamRow } from "./db/db";
+import type {
+  IndexRunRow,
+  IndexStateRow,
+  LexiconIndexRunRow,
+  LexiconIndexStateRow,
+  SearchCompanionPlanRow,
+  SearchSegmentCompanionRow,
+  SecondaryIndexRunRow,
+  SecondaryIndexStateRow,
+  SegmentMetaRow,
+  StreamRow,
+} from "./db/db";
 import { encodeOffsetResult } from "./offset";
 import { dsError } from "./util/ds_error.ts";
 
@@ -12,6 +23,15 @@ function compressB64(bytes: Uint8Array): string {
   return b64(new Uint8Array(zstdCompressSync(bytes)));
 }
 
+function parseSectionsJson(raw: string): string[] {
+  try {
+    const parsed = JSON.parse(raw);
+    return Array.isArray(parsed) ? parsed.filter((value): value is string => typeof value === "string") : [];
+  } catch {
+    return [];
+  }
+}
+
 export type ManifestJson = Record<string, any>;
 export type ManifestBuildError = { kind: "invalid_manifest"; message: string };
 
@@ -22,16 +42,37 @@ function invalidManifest<T = never>(message: string): Result<T, ManifestBuildErr
 type BuildManifestArgs = {
   streamName: string;
   streamRow: StreamRow;
+  publishedLogicalSizeBytes: bigint;
+  profileJson?: Record<string, any> | null;
   segmentMeta: SegmentMetaRow;
   uploadedPrefixCount: number;
   generation: number;
   indexState?: IndexStateRow | null;
   indexRuns?: IndexRunRow[];
   retiredRuns?: IndexRunRow[];
+  secondaryIndexStates?: SecondaryIndexStateRow[];
+  secondaryIndexRuns?: SecondaryIndexRunRow[];
+  retiredSecondaryIndexRuns?: SecondaryIndexRunRow[];
+  lexiconIndexStates?: LexiconIndexStateRow[];
+  lexiconIndexRuns?: LexiconIndexRunRow[];
+  retiredLexiconIndexRuns?: LexiconIndexRunRow[];
+  searchCompanionPlan?: SearchCompanionPlanRow | null;
+  searchSegmentCompanions?: SearchSegmentCompanionRow[];
 };
 
 export function buildManifestResult(args: BuildManifestArgs): Result<ManifestJson, ManifestBuildError> {
-  const { streamName, streamRow, segmentMeta, uploadedPrefixCount, generation, indexState, indexRuns, retiredRuns } = args;
+  const {
+    streamName,
+    streamRow,
+    publishedLogicalSizeBytes,
+    profileJson,
+    segmentMeta,
+    uploadedPrefixCount,
+    generation,
+    indexState,
+    indexRuns,
+    retiredRuns,
+  } = args;
 
   const createdAt = new Date(Number(streamRow.created_at_ms)).toISOString();
   const expiresAt = streamRow.expires_at_ms == null ? null : new Date(Number(streamRow.expires_at_ms)).toISOString();
@@ -59,28 +100,111 @@ export function buildManifestResult(args: BuildManifestArgs): Result<ManifestJso
       start_segment: r.start_segment,
       end_segment: r.end_segment,
       object_key: r.object_key,
+      size_bytes: r.size_bytes,
       filter_len: r.filter_len,
       record_count: r.record_count,
     })) ?? [];
   const retired = retiredRuns?.map((r) => ({
     run_id: r.run_id,
     level: r.level,
-    start_segment: r.start_segment,
-    end_segment: r.end_segment,
-    object_key: r.object_key,
-    filter_len: r.filter_len,
-    record_count: r.record_count,
-    retired_gen: r.retired_gen ?? undefined,
+      start_segment: r.start_segment,
+      end_segment: r.end_segment,
+      object_key: r.object_key,
+      size_bytes: r.size_bytes,
+      filter_len: r.filter_len,
+      record_count: r.record_count,
+      retired_gen: r.retired_gen ?? undefined,
     retired_at_unix: r.retired_at_ms != null ? Number(r.retired_at_ms / 1000n) : undefined,
   })) ?? [];
   const indexSecret = indexState?.index_secret ? b64(indexState.index_secret) : "";
   const indexedThrough = indexState?.indexed_through ?? 0;
+  const secondaryIndexes: Record<string, any> = {};
+  const secondaryStates = args.secondaryIndexStates ?? [];
+  const secondaryRuns = args.secondaryIndexRuns ?? [];
+  const retiredSecondaryRuns = args.retiredSecondaryIndexRuns ?? [];
+  for (const state of secondaryStates) {
+    secondaryIndexes[state.index_name] = {
+      index_secret: b64(state.index_secret),
+      config_hash: state.config_hash,
+      indexed_through: state.indexed_through,
+      active_runs: secondaryRuns
+        .filter((run) => run.index_name === state.index_name)
+        .map((run) => ({
+          run_id: run.run_id,
+          level: run.level,
+          start_segment: run.start_segment,
+          end_segment: run.end_segment,
+          object_key: run.object_key,
+          size_bytes: run.size_bytes,
+          filter_len: run.filter_len,
+          record_count: run.record_count,
+        })),
+      retired_runs: retiredSecondaryRuns
+        .filter((run) => run.index_name === state.index_name)
+        .map((run) => ({
+          run_id: run.run_id,
+          level: run.level,
+          start_segment: run.start_segment,
+          end_segment: run.end_segment,
+          object_key: run.object_key,
+          size_bytes: run.size_bytes,
+          filter_len: run.filter_len,
+          record_count: run.record_count,
+          retired_gen: run.retired_gen ?? undefined,
+          retired_at_unix: run.retired_at_ms != null ? Number(run.retired_at_ms / 1000n) : undefined,
+        })),
+      };
+  }
+  const lexiconIndexes = (args.lexiconIndexStates ?? []).map((state) => ({
+    source_kind: state.source_kind,
+    source_name: state.source_name,
+    indexed_through: state.indexed_through,
+    active_runs: (args.lexiconIndexRuns ?? [])
+      .filter((run) => run.source_kind === state.source_kind && run.source_name === state.source_name)
+      .map((run) => ({
+        run_id: run.run_id,
+        level: run.level,
+        start_segment: run.start_segment,
+        end_segment: run.end_segment,
+        object_key: run.object_key,
+        size_bytes: run.size_bytes,
+        record_count: run.record_count,
+      })),
+    retired_runs: (args.retiredLexiconIndexRuns ?? [])
+      .filter((run) => run.source_kind === state.source_kind && run.source_name === state.source_name)
+      .map((run) => ({
+        run_id: run.run_id,
+        level: run.level,
+        start_segment: run.start_segment,
+        end_segment: run.end_segment,
+        object_key: run.object_key,
+        size_bytes: run.size_bytes,
+        record_count: run.record_count,
+        retired_gen: run.retired_gen ?? undefined,
+        retired_at_unix: run.retired_at_ms != null ? Number(run.retired_at_ms / 1000n) : undefined,
+      })),
+  }));
+  const searchCompanionPlan = args.searchCompanionPlan ?? null;
+  const searchCompanionSegments = (args.searchSegmentCompanions ?? [])
+    .filter((segment) => segment.segment_index < prefix)
+    .map((segment) => ({
+      segment_index: segment.segment_index,
+      object_key: segment.object_key,
+      size_bytes: segment.size_bytes,
+      plan_generation: segment.plan_generation,
+      primary_timestamp_min_ms: segment.primary_timestamp_min_ms?.toString() ?? undefined,
+      primary_timestamp_max_ms: segment.primary_timestamp_max_ms?.toString() ?? undefined,
+      sections: parseSectionsJson(segment.sections_json),
+      section_sizes: JSON.parse(segment.section_sizes_json || "{}"),
+    }));
 
   return Result.ok({
     name: streamName,
     created_at: createdAt,
     expires_at: expiresAt,
     content_type: streamRow.content_type,
+    profile: streamRow.profile ?? "generic",
+    profile_json: profileJson ?? null,
     stream_seq: streamRow.stream_seq ?? null,
     closed: streamRow.closed,
     closed_producer_id: streamRow.closed_producer_id ?? null,
@@ -92,6 +216,7 @@ export function buildManifestResult(args: BuildManifestArgs): Result<ManifestJso
     epoch: streamRow.epoch,
     next_offset: nextOffsetNum,
     next_offset_encoded: nextOffsetEncoded,
+    logical_size_bytes: publishedLogicalSizeBytes.toString(),
     segment_count: prefix,
     uploaded_through: prefix,
     active_file_offset: nextOffsetNum,
@@ -104,6 +229,16 @@ export function buildManifestResult(args: BuildManifestArgs): Result<ManifestJso
     index_secret: indexSecret,
     active_runs: activeRuns,
     retired_runs: retired,
+    secondary_indexes: secondaryIndexes,
+    lexicon_indexes: lexiconIndexes,
+    search_companions: searchCompanionPlan
+      ? {
+          generation: searchCompanionPlan.generation,
+          plan_hash: searchCompanionPlan.plan_hash,
+          plan_json: JSON.parse(searchCompanionPlan.plan_json),
+          segments: searchCompanionSegments,
+        }
+      : null,
   });
 }
 

package/src/memory.ts

@@ -1,6 +1,36 @@
-export class MemoryGuard {
+import { execFileSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import os from "node:os";
+
+const HOST_MEMORY_GUARD_FRACTION = 0.7;
+const HOST_MEMORY_HEADROOM_FRACTION = 0.15;
+const HOST_MEMORY_HEADROOM_MIN_BYTES = 512 * 1024 * 1024;
+const HOST_MEMORY_HEADROOM_MAX_BYTES = 2 * 1024 * 1024 * 1024;
+
+export function deriveMemoryPressureLimitBytes(requestedLimitBytes: number, hostTotalBytes = os.totalmem()): number {
+  const requested = Math.max(0, Math.floor(requestedLimitBytes));
+  if (requested <= 0) return 0;
+  if (!Number.isFinite(hostTotalBytes) || hostTotalBytes <= 0) return requested;
+  const safeHostCap = Math.max(256 * 1024 * 1024, Math.floor(hostTotalBytes * HOST_MEMORY_GUARD_FRACTION));
+  return Math.min(requested, safeHostCap);
+}
+
+export function deriveMemoryPressureHeadroomBytes(limitBytes: number, hostTotalBytes = os.totalmem()): number {
+  const limit = Math.max(0, Math.floor(limitBytes));
+  if (limit <= 0) return 0;
+  if (!Number.isFinite(hostTotalBytes) || hostTotalBytes <= 0) {
+    return Math.min(limit, HOST_MEMORY_HEADROOM_MIN_BYTES);
+  }
+  const headroomFromHost = Math.floor(hostTotalBytes * HOST_MEMORY_HEADROOM_FRACTION);
+  const headroom = Math.max(HOST_MEMORY_HEADROOM_MIN_BYTES, headroomFromHost);
+  return Math.min(limit, Math.min(HOST_MEMORY_HEADROOM_MAX_BYTES, headroom));
+}
+
+export class MemoryPressureMonitor {
   private readonly limitBytes: number;
   private readonly resumeBytes: number;
+  private readonly hostHeadroomBytes: number;
+  private readonly hostResumeHeadroomBytes: number;
   private readonly intervalMs: number;
   private readonly onSample?: (rssBytes: number, overLimit: boolean, limitBytes: number) => void;
   private readonly heapSnapshotPath?: string;
@@ -10,7 +40,16 @@ export class MemoryGuard {
   private maxRssBytes = 0;
   private lastRssBytes = 0;
   private lastGcMs = 0;
+  private forcedGcCount = 0;
+  private forcedGcReclaimedBytesTotal = 0;
+  private lastForcedGcAtMs = 0;
+  private lastForcedGcBeforeBytes = 0;
+  private lastForcedGcAfterBytes = 0;
+  private lastForcedGcReclaimedBytes = 0;
   private lastSnapshotMs = 0;
+  private heapSnapshotsWritten = 0;
+  private lastDarwinPhysicalBytes = 0;
+  private lastDarwinPhysicalAtMs = 0;
 
   constructor(
     limitBytes: number,
@@ -22,11 +61,20 @@ export class MemoryGuard {
       heapSnapshotMinIntervalMs?: number;
     } = {}
   ) {
-    this.limitBytes = Math.max(0, limitBytes);
+    const requestedLimitBytes = Math.max(0, Math.floor(limitBytes));
+    this.limitBytes = deriveMemoryPressureLimitBytes(requestedLimitBytes);
+    if (requestedLimitBytes > 0 && this.limitBytes < requestedLimitBytes) {
+      // eslint-disable-next-line no-console
+      console.warn(
+        `[memory] clamped limit from ${formatBytes(requestedLimitBytes)} to ${formatBytes(this.limitBytes)} based on host memory`
+      );
+    }
     // Resume as soon as RSS drops back below the limit by default (no hysteresis),
     // so the server doesn't "deadlock" itself under a stable high-water mark.
     const resumeFraction = Math.min(1.0, Math.max(0.5, opts.resumeFraction ?? 1.0));
     this.resumeBytes = Math.floor(this.limitBytes * resumeFraction);
+    this.hostHeadroomBytes = deriveMemoryPressureHeadroomBytes(this.limitBytes);
+    this.hostResumeHeadroomBytes = Math.floor(this.hostHeadroomBytes * 1.25);
     this.intervalMs = Math.max(50, opts.intervalMs ?? 1000);
     this.onSample = opts.onSample;
     this.heapSnapshotPath = opts.heapSnapshotPath;
@@ -46,10 +94,13 @@ export class MemoryGuard {
 
   private sample(): void {
     const rss = process.memoryUsage().rss;
+    const effectiveBytes = this.effectiveBytesForGuard(rss);
+    const hostAvailableBytes = readHostAvailableMemoryBytes();
     this.lastRssBytes = rss;
     if (rss > this.maxRssBytes) this.maxRssBytes = rss;
+    const hostLowMemory = this.hostHeadroomBytes > 0 && hostAvailableBytes <= this.hostHeadroomBytes;
+    const overLimit = this.limitBytes > 0 && (effectiveBytes > this.limitBytes || hostLowMemory);
     if (this.onSample) {
-      const overLimit = this.limitBytes > 0 && rss > this.limitBytes;
       try {
         this.onSample(rss, overLimit, this.limitBytes);
       } catch {
@@ -57,16 +108,35 @@ export class MemoryGuard {
       }
     }
     if (this.limitBytes <= 0) return;
+    if (overLimit) {
+      this.maybeGc(hostLowMemory ? "host memory headroom" : "memory sample");
+      this.maybeHeapSnapshot(hostLowMemory ? "host memory headroom" : "memory sample");
+    }
     if (this.overLimit) {
-      if (rss <= this.resumeBytes) this.overLimit = false;
-    } else if (rss > this.limitBytes) {
+      if (effectiveBytes <= this.resumeBytes && hostAvailableBytes > this.hostResumeHeadroomBytes) this.overLimit = false;
+    } else if (effectiveBytes > this.limitBytes) {
+      this.overLimit = true;
+    } else if (hostLowMemory) {
       this.overLimit = true;
     }
   }
 
-  shouldAllow(): boolean {
-    if (this.limitBytes <= 0) return true;
-    return !this.overLimit;
+  private effectiveBytesForGuard(rssBytes: number): number {
+    if (this.limitBytes <= 0 || rssBytes <= this.limitBytes) return rssBytes;
+    if (process.platform !== "darwin") return rssBytes;
+    const now = Date.now();
+    if (this.lastDarwinPhysicalAtMs !== 0 && now - this.lastDarwinPhysicalAtMs < 5_000) {
+      return this.lastDarwinPhysicalBytes > 0 ? this.lastDarwinPhysicalBytes : this.limitBytes;
+    }
+    this.lastDarwinPhysicalAtMs = now;
+    const physicalBytes = readDarwinTopMemBytes(process.pid);
+    if (physicalBytes != null) {
+      this.lastDarwinPhysicalBytes = physicalBytes;
+      return physicalBytes;
+    }
+    if (this.lastDarwinPhysicalBytes > 0) return this.lastDarwinPhysicalBytes;
+    this.lastDarwinPhysicalBytes = this.limitBytes;
+    return this.lastDarwinPhysicalBytes;
   }
 
   isOverLimit(): boolean {
@@ -91,6 +161,28 @@ export class MemoryGuard {
     return this.limitBytes;
   }
 
+  getGcStats(): {
+    forced_gc_count: number;
+    forced_gc_reclaimed_bytes_total: number;
+    last_forced_gc_at_ms: number | null;
+    last_forced_gc_before_bytes: number | null;
+    last_forced_gc_after_bytes: number | null;
+    last_forced_gc_reclaimed_bytes: number | null;
+    heap_snapshots_written: number;
+    last_heap_snapshot_at_ms: number | null;
+  } {
+    return {
+      forced_gc_count: this.forcedGcCount,
+      forced_gc_reclaimed_bytes_total: this.forcedGcReclaimedBytesTotal,
+      last_forced_gc_at_ms: this.lastForcedGcAtMs > 0 ? this.lastForcedGcAtMs : null,
+      last_forced_gc_before_bytes: this.lastForcedGcAtMs > 0 ? this.lastForcedGcBeforeBytes : null,
+      last_forced_gc_after_bytes: this.lastForcedGcAtMs > 0 ? this.lastForcedGcAfterBytes : null,
+      last_forced_gc_reclaimed_bytes: this.lastForcedGcAtMs > 0 ? this.lastForcedGcReclaimedBytes : null,
+      heap_snapshots_written: this.heapSnapshotsWritten,
+      last_heap_snapshot_at_ms: this.lastSnapshotMs > 0 ? this.lastSnapshotMs : null,
+    };
+  }
+
   maybeGc(reason: string): void {
     const gcFn = (globalThis as any)?.Bun?.gc;
     if (typeof gcFn !== "function") return;
@@ -108,6 +200,13 @@ export class MemoryGuard {
       }
     }
     const after = process.memoryUsage().rss;
+    const reclaimed = Math.max(0, before - after);
+    this.forcedGcCount += 1;
+    this.forcedGcReclaimedBytesTotal += reclaimed;
+    this.lastForcedGcAtMs = now;
+    this.lastForcedGcBeforeBytes = before;
+    this.lastForcedGcAfterBytes = after;
+    this.lastForcedGcReclaimedBytes = reclaimed;
     // eslint-disable-next-line no-console
     console.warn(`[gc] forced GC (${reason}) rss ${formatBytes(before)} -> ${formatBytes(after)}`);
   }
@@ -133,6 +232,7 @@ export class MemoryGuard {
       const before = process.memoryUsage().rss;
       v8.writeHeapSnapshot(this.heapSnapshotPath);
       const after = process.memoryUsage().rss;
+      this.heapSnapshotsWritten += 1;
       // eslint-disable-next-line no-console
       console.warn(`[heap] snapshot (${reason}) rss ${formatBytes(before)} -> ${formatBytes(after)} path=${this.heapSnapshotPath}`);
     } catch (err) {
@@ -142,6 +242,81 @@ export class MemoryGuard {
   }
 }
 
+export function parseDarwinTopMemBytes(output: string, pid: number): number | null {
+  const line = output
+    .split(/\r?\n/)
+    .map((entry) => entry.trim())
+    .find((entry) => new RegExp(`^${pid}\\s+`).test(entry));
+  if (!line) return null;
+  const match = line.match(new RegExp(`^${pid}\\s+([0-9]+(?:\\.[0-9]+)?)([BKMGTP])\\+?\\b`, "i"));
+  if (!match) return null;
+  const value = Number(match[1]);
+  if (!Number.isFinite(value)) return null;
+  const unit = match[2]!.toUpperCase();
+  const power =
+    unit === "B"
+      ? 0
+      : unit === "K"
+        ? 1
+        : unit === "M"
+          ? 2
+          : unit === "G"
+            ? 3
+            : unit === "T"
+              ? 4
+              : unit === "P"
+                ? 5
+                : -1;
+  if (power < 0) return null;
+  return Math.round(value * 1024 ** power);
+}
+
+export function darwinTopMemArgs(pid: number): string[] {
+  return ["-l", "1", "-pid", String(pid), "-stats", "pid,mem"];
+}
+
+export function parseLinuxMemAvailableBytes(meminfo: string): number | null {
+  for (const line of meminfo.split(/\r?\n/)) {
+    const match = line.match(/^MemAvailable:\s+([0-9]+)\s+kB$/i);
+    if (!match) continue;
+    const kb = Number(match[1]);
+    if (!Number.isFinite(kb) || kb < 0) return null;
+    return kb * 1024;
+  }
+  return null;
+}
+
+function readLinuxMemAvailableBytes(): number | null {
+  try {
+    const meminfo = readFileSync("/proc/meminfo", "utf8");
+    return parseLinuxMemAvailableBytes(meminfo);
+  } catch {
+    return null;
+  }
+}
+
+function readHostAvailableMemoryBytes(): number {
+  if (process.platform === "linux") {
+    const available = readLinuxMemAvailableBytes();
+    if (available != null) return available;
+  }
+  return os.freemem();
+}
+
+function readDarwinTopMemBytes(pid: number): number | null {
+  try {
+    const output = execFileSync("/usr/bin/top", darwinTopMemArgs(pid), {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+      timeout: 5_000,
+      maxBuffer: 256 * 1024,
+    });
+    return parseDarwinTopMemBytes(output, pid);
+  } catch {
+    return null;
+  }
+}
+
 export function formatBytes(bytes: number): string {
   const units = ["b", "kb", "mb", "gb"];
   let value = bytes;