@prisma/streams-server 0.0.1 → 0.1.1

package/src/app_local.ts

@@ -0,0 +1,46 @@
+import type { Config } from "./config";
+import { createAppCore, type App } from "./app_core";
+import type { ObjectStore } from "./objectstore/interface";
+import { NullObjectStore } from "./objectstore/null";
+import { StreamReader } from "./reader";
+import type { StatsCollector } from "./stats";
+import type { UploaderController } from "./uploader";
+import type { SegmenterController } from "./segment/segmenter_workers";
+
+class NoopUploader implements UploaderController {
+  start(): void {}
+  stop(_hard?: boolean): void {}
+  countSegmentsWaiting(): number {
+    return 0;
+  }
+  setHooks(_hooks: { onSegmentsUploaded?: (stream: string) => void } | undefined): void {}
+  async publishManifest(_stream: string): Promise<void> {}
+}
+
+const noopSegmenter: SegmenterController = {
+  start(): void {},
+  stop(_hard?: boolean): void {},
+};
+
+export type CreateLocalAppOptions = {
+  stats?: StatsCollector;
+};
+
+export function createLocalApp(cfg: Config, os?: ObjectStore, opts: CreateLocalAppOptions = {}): App {
+  return createAppCore(cfg, {
+    stats: opts.stats,
+    createRuntime: ({ config, db }) => {
+      const store = os ?? new NullObjectStore();
+      const reader = new StreamReader(config, db, store);
+
+      return {
+        store,
+        reader,
+        segmenter: noopSegmenter,
+        uploader: new NoopUploader(),
+        uploadSchemaRegistry: async (): Promise<void> => {},
+        start: (): void => {},
+      };
+    },
+  });
+}

package/src/backpressure.ts

@@ -0,0 +1,66 @@
+export class BackpressureGate {
+  private readonly maxBytes: number;
+  private currentBytes: number;
+  private reservedBytes: number;
+
+  constructor(maxBytes: number, initialBytes: number) {
+    this.maxBytes = maxBytes;
+    this.currentBytes = Math.max(0, initialBytes);
+    this.reservedBytes = 0;
+  }
+
+  enabled(): boolean {
+    return this.maxBytes > 0;
+  }
+
+  reserve(bytes: number): boolean {
+    if (this.maxBytes <= 0) return true;
+    if (bytes <= 0) return true;
+    if (this.currentBytes + this.reservedBytes + bytes > this.maxBytes) return false;
+    this.reservedBytes += bytes;
+    return true;
+  }
+
+  commit(bytes: number, reservedBytes: number = bytes): void {
+    if (this.maxBytes <= 0) return;
+    if (bytes <= 0) return;
+    if (reservedBytes > 0) this.reservedBytes = Math.max(0, this.reservedBytes - reservedBytes);
+    this.currentBytes += bytes;
+  }
+
+  release(bytes: number): void {
+    if (this.maxBytes <= 0) return;
+    if (bytes <= 0) return;
+    this.reservedBytes = Math.max(0, this.reservedBytes - bytes);
+  }
+
+  adjustOnSeal(payloadBytes: number, segmentBytes: number): void {
+    if (this.maxBytes <= 0) return;
+    const delta = segmentBytes - payloadBytes;
+    this.currentBytes = Math.max(0, this.currentBytes + delta);
+  }
+
+  adjustOnUpload(segmentBytes: number): void {
+    if (this.maxBytes <= 0) return;
+    this.currentBytes = Math.max(0, this.currentBytes - segmentBytes);
+  }
+
+  adjustOnWalTrim(payloadBytes: number): void {
+    if (this.maxBytes <= 0) return;
+    if (payloadBytes <= 0) return;
+    this.currentBytes = Math.max(0, this.currentBytes - payloadBytes);
+  }
+
+  getCurrentBytes(): number {
+    return this.currentBytes;
+  }
+
+  getMaxBytes(): number {
+    return this.maxBytes;
+  }
+
+  isOverLimit(): boolean {
+    if (this.maxBytes <= 0) return false;
+    return this.currentBytes + this.reservedBytes >= this.maxBytes;
+  }
+}

package/src/bootstrap.ts

@@ -0,0 +1,239 @@
+import { mkdirSync, rmSync } from "node:fs";
+import { dirname } from "node:path";
+import { zstdDecompressSync } from "node:zlib";
+import type { Config } from "./config";
+import { SqliteDurableStore } from "./db/db";
+import type { ObjectStore } from "./objectstore/interface";
+import { localSegmentPath, schemaObjectKey, segmentObjectKey, streamHash16Hex } from "./util/stream_paths";
+import { retry } from "./util/retry";
+import { dsError } from "./util/ds_error.ts";
+
+type Manifest = Record<string, any>;
+
+export async function bootstrapFromR2(cfg: Config, store: ObjectStore, opts: { clearLocal?: boolean } = {}): Promise<void> {
+  if (opts.clearLocal !== false) {
+    try {
+      rmSync(cfg.dbPath, { force: true });
+    } catch {
+      // ignore
+    }
+    try {
+      rmSync(`${cfg.rootDir}/local`, { recursive: true, force: true });
+    } catch {
+      // ignore
+    }
+    try {
+      rmSync(`${cfg.rootDir}/cache`, { recursive: true, force: true });
+    } catch {
+      // ignore
+    }
+  }
+
+  mkdirSync(cfg.rootDir, { recursive: true });
+
+  const db = new SqliteDurableStore(cfg.dbPath, { cacheBytes: cfg.sqliteCacheBytes });
+  try {
+    const retryOpts = {
+      retries: cfg.objectStoreRetries,
+      baseDelayMs: cfg.objectStoreBaseDelayMs,
+      maxDelayMs: cfg.objectStoreMaxDelayMs,
+      timeoutMs: cfg.objectStoreTimeoutMs,
+    };
+    const keys = await retry(() => store.list("streams/"), retryOpts);
+    const manifestKeys = keys.filter((k) => k.endsWith("/manifest.json"));
+    for (const mkey of manifestKeys) {
+      const mbytes = await retry(async () => {
+        const data = await store.get(mkey);
+        if (!data) throw dsError(`missing manifest ${mkey}`);
+        return data;
+      }, retryOpts);
+      const manifest = JSON.parse(new TextDecoder().decode(mbytes)) as Manifest;
+      const stream = String(manifest.name ?? "");
+      if (!stream) continue;
+
+      const shash = streamHash16Hex(stream);
+      const nowMs = db.nowMs();
+
+      const createdAtMs = parseIsoMs(manifest.created_at) ?? nowMs;
+      const expiresAtMs = parseIsoMs(manifest.expires_at);
+      const epoch = typeof manifest.epoch === "number" ? manifest.epoch : 0;
+      const nextOffsetNum = typeof manifest.next_offset === "number" ? manifest.next_offset : 0;
+      const nextOffset = BigInt(nextOffsetNum);
+
+      const contentType = typeof manifest.content_type === "string" ? manifest.content_type : "application/octet-stream";
+      const streamSeq = typeof manifest.stream_seq === "string" ? manifest.stream_seq : null;
+      const closed = typeof manifest.closed === "number" ? manifest.closed : 0;
+      const closedProducerId = typeof manifest.closed_producer_id === "string" ? manifest.closed_producer_id : null;
+      const closedProducerEpoch = typeof manifest.closed_producer_epoch === "number" ? manifest.closed_producer_epoch : null;
+      const closedProducerSeq = typeof manifest.closed_producer_seq === "number" ? manifest.closed_producer_seq : null;
+      const ttlSeconds = typeof manifest.ttl_seconds === "number" ? manifest.ttl_seconds : null;
+      const streamFlags = typeof manifest.stream_flags === "number" ? manifest.stream_flags : 0;
+
+      const segmentOffsetsBytes = decodeZstdBase64(manifest.segment_offsets ?? "");
+      const segmentBlocksBytes = decodeZstdBase64(manifest.segment_blocks ?? "");
+      const segmentLastTsBytes = decodeZstdBase64(manifest.segment_last_ts ?? "");
+      const segmentOffsets = decodeU64LeArray(segmentOffsetsBytes);
+      const segmentBlocks = decodeU32LeArray(segmentBlocksBytes);
+      const segmentLastTs = decodeU64LeArray(segmentLastTsBytes);
+      const segmentCount = typeof manifest.segment_count === "number" ? manifest.segment_count : segmentOffsets.length;
+
+      if (segmentOffsets.length !== segmentCount || segmentBlocks.length !== segmentCount || segmentLastTs.length !== segmentCount) {
+        throw dsError(`manifest array length mismatch for ${stream}`);
+      }
+
+      const lastEndOffset = segmentCount > 0 ? segmentOffsets[segmentCount - 1] - 1n : -1n;
+      const uploadedPrefix = typeof manifest.uploaded_through === "number" ? manifest.uploaded_through : segmentCount;
+      const uploadedThrough =
+        uploadedPrefix > 0 && uploadedPrefix <= segmentOffsets.length ? segmentOffsets[uploadedPrefix - 1] - 1n : -1n;
+      const lastAppendMs = segmentCount > 0 ? segmentLastTs[segmentCount - 1] / 1_000_000n : nowMs;
+
+      db.restoreStreamRow({
+        stream,
+        created_at_ms: createdAtMs,
+        updated_at_ms: nowMs,
+        content_type: contentType,
+        stream_seq: streamSeq,
+        closed,
+        closed_producer_id: closedProducerId,
+        closed_producer_epoch: closedProducerEpoch,
+        closed_producer_seq: closedProducerSeq,
+        ttl_seconds: ttlSeconds,
+        epoch,
+        next_offset: nextOffset,
+        sealed_through: lastEndOffset,
+        uploaded_through: uploadedThrough,
+        uploaded_segment_count: uploadedPrefix,
+        pending_rows: 0n,
+        pending_bytes: 0n,
+        wal_rows: 0n,
+        wal_bytes: 0n,
+        last_append_ms: lastAppendMs,
+        last_segment_cut_ms: lastAppendMs,
+        segment_in_progress: 0,
+        expires_at_ms: expiresAtMs,
+        stream_flags: streamFlags,
+      });
+
+      db.upsertSegmentMeta(stream, segmentCount, segmentOffsetsBytes, segmentBlocksBytes, segmentLastTsBytes);
+
+      const manifestHead = await retry(async () => {
+        const head = await store.head(mkey);
+        if (!head) throw dsError(`missing manifest head ${mkey}`);
+        return head;
+      }, retryOpts);
+      db.upsertManifestRow(stream, Number(manifest.generation ?? 0), Number(manifest.generation ?? 0), nowMs, manifestHead?.etag ?? null);
+
+      for (let i = 0; i < segmentCount; i++) {
+        const startOffset = i === 0 ? 0n : segmentOffsets[i - 1];
+        const endOffset = segmentOffsets[i] - 1n;
+        const lastTsMs = segmentLastTs[i] / 1_000_000n;
+        const localPath = localSegmentPath(cfg.rootDir, shash, i);
+        const segmentId = `${shash}-${i}-${startOffset.toString()}-${endOffset.toString()}`;
+        mkdirSync(dirname(localPath), { recursive: true });
+        const objectKey = segmentObjectKey(shash, i);
+        const head = await retry(async () => {
+          const h = await store.head(objectKey);
+          if (!h) throw dsError(`missing segment ${objectKey}`);
+          return h;
+        }, retryOpts);
+        if (!head) throw dsError(`missing segment ${objectKey}`);
+        db.createSegmentRow({
+          segmentId,
+          stream,
+          segmentIndex: i,
+          startOffset,
+          endOffset,
+          blockCount: segmentBlocks[i],
+          lastAppendMs: lastTsMs,
+          sizeBytes: head.size,
+          localPath,
+        });
+        db.markSegmentUploaded(segmentId, head.etag, nowMs);
+      }
+
+      const indexSecretB64 = typeof manifest.index_secret === "string" ? manifest.index_secret : "";
+      if (indexSecretB64) {
+        const secret = new Uint8Array(Buffer.from(indexSecretB64, "base64"));
+        const indexedThrough = typeof manifest.indexed_through === "number" ? manifest.indexed_through : 0;
+        db.upsertIndexState(stream, secret, indexedThrough);
+      }
+
+      const activeRuns = Array.isArray(manifest.active_runs) ? manifest.active_runs : [];
+      const retiredRuns = Array.isArray(manifest.retired_runs) ? manifest.retired_runs : [];
+      for (const r of activeRuns) {
+        db.insertIndexRun({
+          run_id: String(r.run_id),
+          stream,
+          level: Number(r.level),
+          start_segment: Number(r.start_segment),
+          end_segment: Number(r.end_segment),
+          object_key: String(r.object_key),
+          filter_len: Number(r.filter_len ?? 0),
+          record_count: Number(r.record_count ?? 0),
+        });
+      }
+      for (const r of retiredRuns) {
+        const runId = String(r.run_id);
+        db.insertIndexRun({
+          run_id: runId,
+          stream,
+          level: Number(r.level),
+          start_segment: Number(r.start_segment),
+          end_segment: Number(r.end_segment),
+          object_key: String(r.object_key),
+          filter_len: Number(r.filter_len ?? 0),
+          record_count: Number(r.record_count ?? 0),
+        });
+        const retiredGen = typeof r.retired_gen === "number" ? r.retired_gen : Number(manifest.generation ?? 0);
+        const retiredAtUnix = typeof r.retired_at_unix === "number" ? r.retired_at_unix : Math.floor(Number(nowMs) / 1000);
+        db.retireIndexRuns([runId], retiredGen, BigInt(retiredAtUnix) * 1000n);
+      }
+
+      const schemaKey = schemaObjectKey(shash);
+      const schemaBytes = await retry(async () => {
+        const data = await store.get(schemaKey);
+        if (!data) return null;
+        return data;
+      }, retryOpts);
+      if (schemaBytes) {
+        db.upsertSchemaRegistry(stream, new TextDecoder().decode(schemaBytes));
+      }
+    }
+  } finally {
+    db.close();
+  }
+}
+
+function decodeZstdBase64(value: string): Uint8Array {
+  if (!value) return new Uint8Array(0);
+  const raw = Buffer.from(value, "base64");
+  if (raw.byteLength === 0) return new Uint8Array(0);
+  return new Uint8Array(zstdDecompressSync(raw));
+}
+
+function decodeU64LeArray(bytes: Uint8Array): bigint[] {
+  if (bytes.byteLength === 0) return [];
+  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const out: bigint[] = [];
+  for (let off = 0; off + 8 <= bytes.byteLength; off += 8) {
+    out.push(dv.getBigUint64(off, true));
+  }
+  return out;
+}
+
+function decodeU32LeArray(bytes: Uint8Array): number[] {
+  if (bytes.byteLength === 0) return [];
+  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const out: number[] = [];
+  for (let off = 0; off + 4 <= bytes.byteLength; off += 4) {
+    out.push(dv.getUint32(off, true));
+  }
+  return out;
+}
+
+function parseIsoMs(value: any): bigint | null {
+  if (!value || typeof value !== "string") return null;
+  const ms = Date.parse(value);
+  if (!Number.isFinite(ms)) return null;
+  return BigInt(ms);
+}

package/src/config.ts

@@ -0,0 +1,251 @@
+import { dsError } from "./util/ds_error.ts";
+export type Config = {
+  host: string;
+  rootDir: string;
+  dbPath: string;
+  segmentMaxBytes: number;
+  blockMaxBytes: number;
+  segmentTargetRows: number;
+  segmentMaxIntervalMs: number;
+  segmentCheckIntervalMs: number;
+  segmenterWorkers: number;
+  uploadIntervalMs: number;
+  uploadConcurrency: number;
+  segmentCacheMaxBytes: number;
+  segmentFooterCacheEntries: number;
+  indexRunCacheMaxBytes: number;
+  indexRunMemoryCacheBytes: number;
+  indexL0SpanSegments: number;
+  indexBuildConcurrency: number;
+  indexCheckIntervalMs: number;
+  indexCompactionFanout: number;
+  indexMaxLevel: number;
+  indexCompactionConcurrency: number;
+  indexRetireGenWindow: number;
+  indexRetireMinMs: number;
+  readMaxBytes: number;
+  readMaxRecords: number;
+  appendMaxBodyBytes: number;
+  ingestFlushIntervalMs: number;
+  ingestMaxBatchRequests: number;
+  ingestMaxBatchBytes: number;
+  ingestMaxQueueRequests: number;
+  ingestMaxQueueBytes: number;
+  ingestBusyTimeoutMs: number;
+  localBacklogMaxBytes: number;
+  memoryLimitBytes: number;
+  sqliteCacheBytes: number;
+  objectStoreTimeoutMs: number;
+  objectStoreRetries: number;
+  objectStoreBaseDelayMs: number;
+  objectStoreMaxDelayMs: number;
+  expirySweepIntervalMs: number;
+  expirySweepBatchLimit: number;
+  metricsFlushIntervalMs: number;
+  interpreterWorkers: number;
+  interpreterCheckIntervalMs: number;
+  interpreterMaxBatchRows: number;
+  interpreterMaxBatchBytes: number;
+  port: number;
+};
+
+const KNOWN_DS_ENVS = new Set<string>([
+  "DS_ROOT",
+  "DS_HOST",
+  "DS_DB_PATH",
+  "DS_SEGMENT_MAX_BYTES",
+  "DS_BLOCK_MAX_BYTES",
+  "DS_SEGMENT_TARGET_ROWS",
+  "DS_SEGMENT_MAX_INTERVAL_MS",
+  "DS_SEGMENT_CHECK_MS",
+  "DS_SEGMENTER_WORKERS",
+  "DS_UPLOAD_CHECK_MS",
+  "DS_UPLOAD_CONCURRENCY",
+  "DS_SEGMENT_CACHE_MAX_BYTES",
+  "DS_SEGMENT_FOOTER_CACHE_ENTRIES",
+  "DS_INDEX_RUN_CACHE_MAX_BYTES",
+  "DS_INDEX_RUN_MEM_CACHE_BYTES",
+  "DS_INDEX_L0_SPAN",
+  "DS_INDEX_BUILD_CONCURRENCY",
+  "DS_INDEX_CHECK_MS",
+  "DS_INDEX_COMPACTION_FANOUT",
+  "DS_INDEX_MAX_LEVEL",
+  "DS_INDEX_COMPACT_CONCURRENCY",
+  "DS_INDEX_RETIRE_GEN_WINDOW",
+  "DS_INDEX_RETIRE_MIN_MS",
+  "DS_READ_MAX_BYTES",
+  "DS_READ_MAX_RECORDS",
+  "DS_APPEND_MAX_BODY_BYTES",
+  "DS_INGEST_FLUSH_MS",
+  "DS_INGEST_MAX_BATCH_REQS",
+  "DS_INGEST_MAX_BATCH_BYTES",
+  "DS_INGEST_MAX_QUEUE_REQS",
+  "DS_INGEST_MAX_QUEUE_BYTES",
+  "DS_INGEST_BUSY_MS",
+  "DS_LOCAL_BACKLOG_MAX_BYTES",
+  "DS_MEMORY_LIMIT_BYTES",
+  "DS_MEMORY_LIMIT_MB",
+  "DS_SQLITE_CACHE_BYTES",
+  "DS_SQLITE_CACHE_MB",
+  "DS_OBJECTSTORE_TIMEOUT_MS",
+  "DS_OBJECTSTORE_RETRIES",
+  "DS_OBJECTSTORE_RETRY_BASE_MS",
+  "DS_OBJECTSTORE_RETRY_MAX_MS",
+  "DS_LOCAL_DATA_ROOT",
+  "DS_EXPIRY_SWEEP_MS",
+  "DS_EXPIRY_SWEEP_LIMIT",
+  "DS_METRICS_FLUSH_MS",
+  "DS_INTERPRETER_WORKERS",
+  "DS_INTERPRETER_CHECK_MS",
+  "DS_INTERPRETER_MAX_BATCH_ROWS",
+  "DS_INTERPRETER_MAX_BATCH_BYTES",
+  "DS_STATS_INTERVAL_MS",
+  "DS_BACKPRESSURE_BUDGET_MS",
+  "DS_MOCK_R2_MAX_INMEM_BYTES",
+  "DS_MOCK_R2_MAX_INMEM_MB",
+  "DS_MOCK_R2_SPILL_DIR",
+  "DS_BENCH_URL",
+  "DS_BENCH_DURATION_MS",
+  "DS_BENCH_INTERVAL_MS",
+  "DS_BENCH_PAYLOAD_BYTES",
+  "DS_BENCH_CONCURRENCY",
+  "DS_BENCH_REQUEST_TIMEOUT_MS",
+  "DS_BENCH_DRAIN_TIMEOUT_MS",
+  "DS_BENCH_PAUSE_BACKGROUND",
+  "DS_BENCH_YIELD_EVERY",
+  "DS_BENCH_DEBUG",
+  "DS_BENCH_SCENARIOS",
+  "DS_MEMORY_STRESS_LIMITS_MB",
+  "DS_MEMORY_STRESS_STATS_MS",
+  "DS_MEMORY_STRESS_PORT_BASE",
+  "DS_RK_EVENTS_MAX",
+  "DS_RK_EVENTS_STEP",
+  "DS_RK_PAYLOAD_BYTES",
+  "DS_RK_APPEND_BATCH",
+  "DS_RK_KEYS",
+  "DS_RK_HOT_KEYS",
+  "DS_RK_HOT_PCT",
+  "DS_RK_PAYLOAD_POOL",
+  "DS_RK_READ_ENTRIES",
+  "DS_RK_WARM_READS",
+  "DS_RK_SEGMENT_BYTES",
+  "DS_RK_BLOCK_BYTES",
+  "DS_RK_SEED",
+  "DS_RK_R2_GET_DELAY_MS",
+]);
+
+let warnedUnknownEnv = false;
+
+function warnUnknownEnv(): void {
+  if (warnedUnknownEnv) return;
+  warnedUnknownEnv = true;
+  const unknown: string[] = [];
+  for (const key of Object.keys(process.env)) {
+    if (!key.startsWith("DS_")) continue;
+    if (KNOWN_DS_ENVS.has(key)) continue;
+    unknown.push(key);
+  }
+  if (unknown.length > 0) {
+    unknown.sort();
+    console.warn(`[config] unknown DS_* environment variables: ${unknown.join(", ")}`);
+  }
+}
+
+function envNum(name: string, def: number): number {
+  const v = process.env[name];
+  if (!v) return def;
+  const n = Number(v);
+  if (!Number.isFinite(n)) throw dsError(`invalid ${name}: ${v}`);
+  return n;
+}
+
+function envBytes(name: string): number | null {
+  const v = process.env[name];
+  if (!v) return null;
+  const n = Number(v);
+  if (!Number.isFinite(n)) throw dsError(`invalid ${name}: ${v}`);
+  return Math.max(0, Math.floor(n));
+}
+
+function clampBytes(value: number, min: number, max: number): number {
+  if (!Number.isFinite(value)) return min;
+  if (value < min) return min;
+  if (value > max) return max;
+  return value;
+}
+
+export function loadConfig(): Config {
+  warnUnknownEnv();
+  const rootDir = process.env.DS_ROOT ?? "./ds-data";
+  const host = process.env.DS_HOST?.trim() || "127.0.0.1";
+  const bytesOverride = envBytes("DS_MEMORY_LIMIT_BYTES");
+  const mbOverride = envBytes("DS_MEMORY_LIMIT_MB");
+  const memoryLimitBytes = bytesOverride ?? (mbOverride != null ? mbOverride * 1024 * 1024 : 0);
+  const backlogOverride = envBytes("DS_LOCAL_BACKLOG_MAX_BYTES");
+  const sqliteCacheBytesOverride = envBytes("DS_SQLITE_CACHE_BYTES");
+  const sqliteCacheMbOverride = envBytes("DS_SQLITE_CACHE_MB");
+  const indexMemOverride = envBytes("DS_INDEX_RUN_MEM_CACHE_BYTES");
+  const indexDiskOverride = envBytes("DS_INDEX_RUN_CACHE_MAX_BYTES");
+  const localBacklogMaxBytes = backlogOverride ?? 10 * 1024 * 1024 * 1024;
+  const sqliteCacheBytes =
+    sqliteCacheBytesOverride ??
+    (sqliteCacheMbOverride != null
+      ? sqliteCacheMbOverride * 1024 * 1024
+      : memoryLimitBytes > 0
+        ? Math.floor(memoryLimitBytes * 0.25)
+        : 0);
+  const tunedIndexMem =
+    indexMemOverride ??
+    (memoryLimitBytes > 0
+      ? clampBytes(Math.floor(memoryLimitBytes * 0.05), 8 * 1024 * 1024, 128 * 1024 * 1024)
+      : 64 * 1024 * 1024);
+  return {
+    host,
+    rootDir,
+    dbPath: process.env.DS_DB_PATH ?? `${rootDir}/wal.sqlite`,
+    segmentMaxBytes: envNum("DS_SEGMENT_MAX_BYTES", 16 * 1024 * 1024),
+    blockMaxBytes: envNum("DS_BLOCK_MAX_BYTES", 256 * 1024),
+    segmentTargetRows: envNum("DS_SEGMENT_TARGET_ROWS", 50_000),
+    segmentMaxIntervalMs: envNum("DS_SEGMENT_MAX_INTERVAL_MS", 0),
+    segmentCheckIntervalMs: envNum("DS_SEGMENT_CHECK_MS", 250),
+    segmenterWorkers: envNum("DS_SEGMENTER_WORKERS", 0),
+    uploadIntervalMs: envNum("DS_UPLOAD_CHECK_MS", 250),
+    uploadConcurrency: envNum("DS_UPLOAD_CONCURRENCY", 4),
+    segmentCacheMaxBytes: envNum("DS_SEGMENT_CACHE_MAX_BYTES", 256 * 1024 * 1024),
+    segmentFooterCacheEntries: envNum("DS_SEGMENT_FOOTER_CACHE_ENTRIES", 2048),
+    indexRunCacheMaxBytes: indexDiskOverride ?? 256 * 1024 * 1024,
+    indexRunMemoryCacheBytes: tunedIndexMem,
+    indexL0SpanSegments: envNum("DS_INDEX_L0_SPAN", 16),
+    indexBuildConcurrency: envNum("DS_INDEX_BUILD_CONCURRENCY", 4),
+    indexCheckIntervalMs: envNum("DS_INDEX_CHECK_MS", 1000),
+    indexCompactionFanout: envNum("DS_INDEX_COMPACTION_FANOUT", 16),
+    indexMaxLevel: envNum("DS_INDEX_MAX_LEVEL", 4),
+    indexCompactionConcurrency: envNum("DS_INDEX_COMPACT_CONCURRENCY", 4),
+    indexRetireGenWindow: envNum("DS_INDEX_RETIRE_GEN_WINDOW", 2),
+    indexRetireMinMs: envNum("DS_INDEX_RETIRE_MIN_MS", 5 * 60 * 1000),
+    readMaxBytes: envNum("DS_READ_MAX_BYTES", 1 * 1024 * 1024),
+    readMaxRecords: envNum("DS_READ_MAX_RECORDS", 1000),
+    appendMaxBodyBytes: envNum("DS_APPEND_MAX_BODY_BYTES", 10 * 1024 * 1024),
+    ingestFlushIntervalMs: envNum("DS_INGEST_FLUSH_MS", 10),
+    ingestMaxBatchRequests: envNum("DS_INGEST_MAX_BATCH_REQS", 200),
+    ingestMaxBatchBytes: envNum("DS_INGEST_MAX_BATCH_BYTES", 8 * 1024 * 1024),
+    ingestMaxQueueRequests: envNum("DS_INGEST_MAX_QUEUE_REQS", 50_000),
+    ingestMaxQueueBytes: envNum("DS_INGEST_MAX_QUEUE_BYTES", 64 * 1024 * 1024),
+    ingestBusyTimeoutMs: envNum("DS_INGEST_BUSY_MS", 5000),
+    localBacklogMaxBytes,
+    memoryLimitBytes,
+    sqliteCacheBytes,
+    objectStoreTimeoutMs: envNum("DS_OBJECTSTORE_TIMEOUT_MS", 5000),
+    objectStoreRetries: envNum("DS_OBJECTSTORE_RETRIES", 3),
+    objectStoreBaseDelayMs: envNum("DS_OBJECTSTORE_RETRY_BASE_MS", 50),
+    objectStoreMaxDelayMs: envNum("DS_OBJECTSTORE_RETRY_MAX_MS", 2000),
+    expirySweepIntervalMs: envNum("DS_EXPIRY_SWEEP_MS", 60_000),
+    expirySweepBatchLimit: envNum("DS_EXPIRY_SWEEP_LIMIT", 100),
+    metricsFlushIntervalMs: envNum("DS_METRICS_FLUSH_MS", 10_000),
+    interpreterWorkers: envNum("DS_INTERPRETER_WORKERS", 1),
+    interpreterCheckIntervalMs: envNum("DS_INTERPRETER_CHECK_MS", 250),
+    interpreterMaxBatchRows: envNum("DS_INTERPRETER_MAX_BATCH_ROWS", 500),
+    interpreterMaxBatchBytes: envNum("DS_INTERPRETER_MAX_BATCH_BYTES", 4 * 1024 * 1024),
+    port: envNum("PORT", 8080),
+  };
+}