@tungthedev/streams-server 0.2.0

package/src/compute/entry.ts

@@ -0,0 +1,19 @@
+function hasFlag(argv: string[], flag: string): boolean {
+  return argv.includes(flag) || argv.some((arg) => arg.startsWith(`${flag}=`));
+}
+
+export function ensureComputeArgv(argv: string[], env: NodeJS.ProcessEnv = process.env): string[] {
+  const next = [...argv];
+  if (!hasFlag(next, "--object-store")) {
+    next.push("--object-store", "r2");
+  }
+  if (env.DS_MEMORY_LIMIT_MB != null && !hasFlag(next, "--auto-tune")) {
+    next.push("--auto-tune");
+  }
+  return next;
+}
+
+if (import.meta.main) {
+  process.argv = ensureComputeArgv(process.argv);
+  await import("../server");
+}

package/src/compute/package_entry.ts

@@ -0,0 +1,4 @@
+import { ensureComputeArgv } from "./entry";
+
+process.argv = ensureComputeArgv(process.argv);
+await import("../server");

package/src/compute/virtual-modules.d.ts

@@ -0,0 +1,15 @@
+declare module "virtual:prebuilt-studio-assets" {
+  const mod: {
+    appScript: string;
+    appStyles: string;
+    builtAssets: Map<
+      string,
+      {
+        bytes: Uint8Array;
+        contentType: string;
+      }
+    >;
+  };
+
+  export = mod;
+}

package/src/compute/worker_module_url.ts

@@ -0,0 +1,9 @@
+export function resolveWorkerModuleUrl(
+  currentModuleUrl: string,
+  sourceRelativePath: string,
+  builtRelativePath = sourceRelativePath.endsWith(".ts")
+    ? `${sourceRelativePath.slice(0, -3)}.js`
+    : sourceRelativePath
+): URL {
+  return new URL(currentModuleUrl.endsWith(".js") ? builtRelativePath : sourceRelativePath, currentModuleUrl);
+}

package/src/concurrency_gate.ts

@@ -0,0 +1,108 @@
+export type GateRelease = () => void;
+
+type Waiter = {
+  resolve: (release: GateRelease) => void;
+  reject: (error: unknown) => void;
+  signal: AbortSignal | null;
+  onAbort: (() => void) | null;
+};
+
+function abortError(): Error {
+  const err = new Error("operation aborted");
+  err.name = "AbortError";
+  return err;
+}
+
+export class ConcurrencyGate {
+  private limit: number;
+  private active = 0;
+  private readonly waiters: Waiter[] = [];
+
+  constructor(limit: number) {
+    this.limit = Math.max(1, Math.floor(limit));
+  }
+
+  getLimit(): number {
+    return this.limit;
+  }
+
+  getActive(): number {
+    return this.active;
+  }
+
+  getQueued(): number {
+    return this.waiters.length;
+  }
+
+  setLimit(nextLimit: number): void {
+    this.limit = Math.max(1, Math.floor(nextLimit));
+    this.drain();
+  }
+
+  async acquire(signal?: AbortSignal | null): Promise<GateRelease> {
+    if (signal?.aborted) throw abortError();
+    if (this.active < this.limit) {
+      this.active += 1;
+      return this.releaseFactory();
+    }
+    return await new Promise<GateRelease>((resolve, reject) => {
+      const waiter: Waiter = {
+        resolve,
+        reject,
+        signal: signal ?? null,
+        onAbort: null,
+      };
+      if (signal) {
+        waiter.onAbort = () => {
+          this.removeWaiter(waiter);
+          reject(abortError());
+        };
+        signal.addEventListener("abort", waiter.onAbort, { once: true });
+      }
+      this.waiters.push(waiter);
+    });
+  }
+
+  async run<T>(fn: () => Promise<T>, signal?: AbortSignal | null): Promise<T> {
+    const release = await this.acquire(signal);
+    try {
+      return await fn();
+    } finally {
+      release();
+    }
+  }
+
+  private releaseFactory(): GateRelease {
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      this.active = Math.max(0, this.active - 1);
+      this.drain();
+    };
+  }
+
+  private removeWaiter(waiter: Waiter): void {
+    const idx = this.waiters.indexOf(waiter);
+    if (idx >= 0) this.waiters.splice(idx, 1);
+    if (waiter.signal && waiter.onAbort) {
+      waiter.signal.removeEventListener("abort", waiter.onAbort);
+      waiter.onAbort = null;
+    }
+  }
+
+  private drain(): void {
+    while (this.active < this.limit && this.waiters.length > 0) {
+      const waiter = this.waiters.shift()!;
+      if (waiter.signal?.aborted) {
+        if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener("abort", waiter.onAbort);
+        waiter.reject(abortError());
+        continue;
+      }
+      if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener("abort", waiter.onAbort);
+      waiter.onAbort = null;
+      this.active += 1;
+      waiter.resolve(this.releaseFactory());
+    }
+  }
+}

package/src/config.ts

@@ -0,0 +1,402 @@
+import { dsError } from "./util/ds_error.ts";
+export type Config = {
+  autoTuneRequestedMemoryMb: number | null;
+  autoTunePresetMb: number | null;
+  autoTuneEffectiveMemoryLimitMb: number | null;
+  storage: "sqlite" | "postgres";
+  postgresMode: "wal" | "full";
+  postgresUrl: string | null;
+  host: string;
+  rootDir: string;
+  dbPath: string;
+  segmentMaxBytes: number;
+  blockMaxBytes: number;
+  segmentTargetRows: number;
+  segmentMaxIntervalMs: number;
+  segmentCheckIntervalMs: number;
+  segmenterWorkers: number;
+  uploadIntervalMs: number;
+  uploadConcurrency: number;
+  segmentCacheMaxBytes: number;
+  segmentFooterCacheEntries: number;
+  indexRunCacheMaxBytes: number;
+  indexRunMemoryCacheBytes: number;
+  lexiconIndexCacheMaxBytes: number;
+  lexiconMappedCacheEntries: number;
+  indexL0SpanSegments: number;
+  indexBuildConcurrency: number;
+  indexCheckIntervalMs: number;
+  searchCompanionBuildBatchSegments: number;
+  searchCompanionYieldBlocks: number;
+  searchCompanionFileCacheMaxBytes: number;
+  searchCompanionFileCacheMaxAgeMs: number;
+  searchCompanionMappedCacheEntries: number;
+  searchCompanionTocCacheBytes: number;
+  searchCompanionSectionCacheBytes: number;
+  searchWalOverlayQuietPeriodMs: number;
+  searchWalOverlayMaxBytes: number;
+  indexCompactionFanout: number;
+  indexMaxLevel: number;
+  indexCompactionConcurrency: number;
+  indexRetireGenWindow: number;
+  indexRetireMinMs: number;
+  readMaxBytes: number;
+  readMaxRecords: number;
+  appendMaxBodyBytes: number;
+  ingestFlushIntervalMs: number;
+  ingestMaxBatchRequests: number;
+  ingestMaxBatchBytes: number;
+  ingestMaxQueueRequests: number;
+  ingestMaxQueueBytes: number;
+  ingestConcurrency: number;
+  ingestBusyTimeoutMs: number;
+  localBacklogMaxBytes: number;
+  memoryLimitBytes: number;
+  sqliteCacheBytes: number;
+  workerSqliteCacheBytes: number;
+  readConcurrency: number;
+  searchConcurrency: number;
+  asyncIndexConcurrency: number;
+  heapSnapshotPath: string | null;
+  memorySamplerPath: string | null;
+  memorySamplerIntervalMs: number;
+  objectStoreTimeoutMs: number;
+  objectStoreRetries: number;
+  objectStoreBaseDelayMs: number;
+  objectStoreMaxDelayMs: number;
+  expirySweepIntervalMs: number;
+  expirySweepBatchLimit: number;
+  metricsFlushIntervalMs: number;
+  touchWorkers: number;
+  touchCheckIntervalMs: number;
+  touchMaxBatchRows: number;
+  touchMaxBatchBytes: number;
+  otlpTracesStream: string | null;
+  otlpAutoCreate: boolean;
+  port: number;
+};
+
+const KNOWN_DS_ENVS = new Set<string>([
+  "DS_STORAGE",
+  "DS_POSTGRES_MODE",
+  "DS_POSTGRES_URL",
+  "DS_TEST_POSTGRES_URL",
+  "DS_ROOT",
+  "DS_HOST",
+  "DS_DB_PATH",
+  "DS_SEGMENT_MAX_BYTES",
+  "DS_BLOCK_MAX_BYTES",
+  "DS_SEGMENT_TARGET_ROWS",
+  "DS_SEGMENT_MAX_INTERVAL_MS",
+  "DS_SEGMENT_CHECK_MS",
+  "DS_SEGMENTER_WORKERS",
+  "DS_UPLOAD_CHECK_MS",
+  "DS_UPLOAD_CONCURRENCY",
+  "DS_BASE_WAL_GC_CHUNK_OFFSETS",
+  "DS_BASE_WAL_GC_INTERVAL_MS",
+  "DS_SEGMENT_CACHE_MAX_BYTES",
+  "DS_SEGMENT_FOOTER_CACHE_ENTRIES",
+  "DS_INDEX_RUN_CACHE_MAX_BYTES",
+  "DS_INDEX_RUN_MEM_CACHE_BYTES",
+  "DS_LEXICON_INDEX_CACHE_MAX_BYTES",
+  "DS_LEXICON_MMAP_CACHE_ENTRIES",
+  "DS_INDEX_L0_SPAN",
+  "DS_INDEX_BUILD_CONCURRENCY",
+  "DS_INDEX_CHECK_MS",
+  "DS_SEARCH_COMPANION_BATCH_SEGMENTS",
+  "DS_SEARCH_COMPANION_YIELD_BLOCKS",
+  "DS_SEARCH_COMPANION_FILE_CACHE_MAX_BYTES",
+  "DS_SEARCH_COMPANION_FILE_CACHE_MAX_AGE_MS",
+  "DS_SEARCH_COMPANION_MMAP_CACHE_ENTRIES",
+  "DS_SEARCH_COMPANION_TOC_CACHE_BYTES",
+  "DS_SEARCH_COMPANION_SECTION_CACHE_BYTES",
+  "DS_SEARCH_WAL_OVERLAY_QUIET_MS",
+  "DS_SEARCH_WAL_OVERLAY_MAX_BYTES",
+  "DS_INDEX_COMPACTION_FANOUT",
+  "DS_INDEX_MAX_LEVEL",
+  "DS_INDEX_COMPACT_CONCURRENCY",
+  "DS_INDEX_RETIRE_GEN_WINDOW",
+  "DS_INDEX_RETIRE_MIN_MS",
+  "DS_READ_MAX_BYTES",
+  "DS_READ_MAX_RECORDS",
+  "DS_APPEND_MAX_BODY_BYTES",
+  "DS_INGEST_FLUSH_MS",
+  "DS_INGEST_MAX_BATCH_REQS",
+  "DS_INGEST_MAX_BATCH_BYTES",
+  "DS_INGEST_MAX_QUEUE_REQS",
+  "DS_INGEST_MAX_QUEUE_BYTES",
+  "DS_INGEST_CONCURRENCY",
+  "DS_INGEST_BUSY_MS",
+  "DS_LOCAL_BACKLOG_MAX_BYTES",
+  "DS_MEMORY_LIMIT_BYTES",
+  "DS_MEMORY_LIMIT_MB",
+  "DS_SQLITE_CACHE_BYTES",
+  "DS_SQLITE_CACHE_MB",
+  "DS_WORKER_SQLITE_CACHE_BYTES",
+  "DS_WORKER_SQLITE_CACHE_MB",
+  "DS_READ_CONCURRENCY",
+  "DS_SEARCH_CONCURRENCY",
+  "DS_ASYNC_INDEX_CONCURRENCY",
+  "DS_HEAP_SNAPSHOT_PATH",
+  "DS_MEMORY_SAMPLER_PATH",
+  "DS_MEMORY_SAMPLER_INTERVAL_MS",
+  "DS_OBJECTSTORE_TIMEOUT_MS",
+  "DS_OBJECTSTORE_RETRIES",
+  "DS_OBJECTSTORE_RETRY_BASE_MS",
+  "DS_OBJECTSTORE_RETRY_MAX_MS",
+  "DS_LOCAL_DATA_ROOT",
+  "DS_EXPIRY_SWEEP_MS",
+  "DS_EXPIRY_SWEEP_LIMIT",
+  "DS_METRICS_FLUSH_MS",
+  "DS_TOUCH_WORKERS",
+  "DS_TOUCH_CHECK_MS",
+  "DS_TOUCH_MAX_BATCH_ROWS",
+  "DS_TOUCH_MAX_BATCH_BYTES",
+  "DS_OTLP_TRACES_STREAM",
+  "DS_OTLP_AUTO_CREATE",
+  "DS_AUTO_TUNE_REQUESTED_MB",
+  "DS_AUTO_TUNE_PRESET_MB",
+  "DS_AUTO_TUNE_EFFECTIVE_MEMORY_LIMIT_MB",
+  "DS_STATS_INTERVAL_MS",
+  "DS_BACKPRESSURE_BUDGET_MS",
+  "DS_MOCK_R2_MAX_INMEM_BYTES",
+  "DS_MOCK_R2_MAX_INMEM_MB",
+  "DS_MOCK_R2_SPILL_DIR",
+  "DS_MOCK_R2_PUT_DELAY_MS",
+  "DS_MOCK_R2_GET_DELAY_MS",
+  "DS_MOCK_R2_HEAD_DELAY_MS",
+  "DS_MOCK_R2_LIST_DELAY_MS",
+  "DS_BENCH_URL",
+  "DS_BENCH_DURATION_MS",
+  "DS_BENCH_INTERVAL_MS",
+  "DS_BENCH_PAYLOAD_BYTES",
+  "DS_BENCH_CONCURRENCY",
+  "DS_BENCH_REQUEST_TIMEOUT_MS",
+  "DS_BENCH_DRAIN_TIMEOUT_MS",
+  "DS_BENCH_PAUSE_BACKGROUND",
+  "DS_BENCH_YIELD_EVERY",
+  "DS_BENCH_DEBUG",
+  "DS_BENCH_SCENARIOS",
+  "DS_MEMORY_STRESS_LIMITS_MB",
+  "DS_MEMORY_STRESS_STATS_MS",
+  "DS_MEMORY_STRESS_PORT_BASE",
+  "DS_RK_EVENTS_MAX",
+  "DS_RK_EVENTS_STEP",
+  "DS_RK_PAYLOAD_BYTES",
+  "DS_RK_APPEND_BATCH",
+  "DS_RK_KEYS",
+  "DS_RK_HOT_KEYS",
+  "DS_RK_HOT_PCT",
+  "DS_RK_PAYLOAD_POOL",
+  "DS_RK_READ_ENTRIES",
+  "DS_RK_WARM_READS",
+  "DS_RK_SEGMENT_BYTES",
+  "DS_RK_BLOCK_BYTES",
+  "DS_RK_SEED",
+  "DS_RK_R2_GET_DELAY_MS",
+  "DS_LARGE_INDEX_FILTER",
+  "DS_LARGE_INDEX_FILTER_TOTAL_BYTES",
+  "DS_LARGE_INDEX_FILTER_PAYLOAD_BYTES",
+  "DS_LARGE_INDEX_FILTER_BATCH_ROWS",
+  "DS_LARGE_INDEX_FILTER_SEGMENT_BYTES",
+  "DS_LARGE_INDEX_FILTER_INDEX_SPAN",
+  "DS_LARGE_INDEX_FILTER_TIMEOUT_MS",
+  "DS_LARGE_INDEX_FILTER_R2_MAX_INMEM_BYTES",
+]);
+
+let warnedUnknownEnv = false;
+
+function warnUnknownEnv(): void {
+  if (warnedUnknownEnv) return;
+  warnedUnknownEnv = true;
+  const unknown: string[] = [];
+  for (const key of Object.keys(process.env)) {
+    if (!key.startsWith("DS_")) continue;
+    if (KNOWN_DS_ENVS.has(key)) continue;
+    unknown.push(key);
+  }
+  if (unknown.length > 0) {
+    unknown.sort();
+    console.warn(`[config] unknown DS_* environment variables: ${unknown.join(", ")}`);
+  }
+}
+
+function envNum(name: string, def: number): number {
+  const v = process.env[name];
+  if (!v) return def;
+  const n = Number(v);
+  if (!Number.isFinite(n)) throw dsError(`invalid ${name}: ${v}`);
+  return n;
+}
+
+function envBool(name: string, def: boolean): boolean {
+  const v = process.env[name];
+  if (v == null || v === "") return def;
+  const normalized = v.trim().toLowerCase();
+  if (normalized === "1" || normalized === "true" || normalized === "yes") return true;
+  if (normalized === "0" || normalized === "false" || normalized === "no") return false;
+  throw dsError(`invalid ${name}: ${v}`);
+}
+
+function envBytes(name: string): number | null {
+  const v = process.env[name];
+  if (!v) return null;
+  const n = Number(v);
+  if (!Number.isFinite(n)) throw dsError(`invalid ${name}: ${v}`);
+  return Math.max(0, Math.floor(n));
+}
+
+function clampBytes(value: number, min: number, max: number): number {
+  if (!Number.isFinite(value)) return min;
+  if (value < min) return min;
+  if (value > max) return max;
+  return value;
+}
+
+function parseStorageMode(): "sqlite" | "postgres" {
+  const raw = process.env.DS_STORAGE?.trim().toLowerCase();
+  if (raw == null || raw === "" || raw === "sqlite") return "sqlite";
+  if (raw === "postgres") return "postgres";
+  throw dsError(`invalid DS_STORAGE: ${process.env.DS_STORAGE}`);
+}
+
+function parsePostgresMode(): "wal" | "full" {
+  const raw = process.env.DS_POSTGRES_MODE?.trim().toLowerCase();
+  if (raw == null || raw === "" || raw === "wal") return "wal";
+  if (raw === "full") return "full";
+  throw dsError(`invalid DS_POSTGRES_MODE: ${process.env.DS_POSTGRES_MODE}`);
+}
+
+export function loadConfig(): Config {
+  warnUnknownEnv();
+  const storage = parseStorageMode();
+  const postgresMode = storage === "postgres" ? parsePostgresMode() : "wal";
+  const postgresUrl = process.env.DS_POSTGRES_URL?.trim() || null;
+  if (storage === "postgres" && postgresUrl == null) throw dsError("DS_POSTGRES_URL is required when DS_STORAGE=postgres");
+  const rootDir = process.env.DS_ROOT ?? "./ds-data";
+  const host = process.env.DS_HOST?.trim() || "127.0.0.1";
+  const autoTuneRequestedMemoryMb = envBytes("DS_AUTO_TUNE_REQUESTED_MB");
+  const autoTunePresetMb = envBytes("DS_AUTO_TUNE_PRESET_MB");
+  const autoTuneEffectiveMemoryLimitMb = envBytes("DS_AUTO_TUNE_EFFECTIVE_MEMORY_LIMIT_MB");
+  const bytesOverride = envBytes("DS_MEMORY_LIMIT_BYTES");
+  const mbOverride = envBytes("DS_MEMORY_LIMIT_MB");
+  const memoryLimitBytes = bytesOverride ?? (mbOverride != null ? mbOverride * 1024 * 1024 : 0);
+  const backlogOverride = envBytes("DS_LOCAL_BACKLOG_MAX_BYTES");
+  const sqliteCacheBytesOverride = envBytes("DS_SQLITE_CACHE_BYTES");
+  const sqliteCacheMbOverride = envBytes("DS_SQLITE_CACHE_MB");
+  const workerSqliteCacheBytesOverride = envBytes("DS_WORKER_SQLITE_CACHE_BYTES");
+  const workerSqliteCacheMbOverride = envBytes("DS_WORKER_SQLITE_CACHE_MB");
+  const indexMemOverride = envBytes("DS_INDEX_RUN_MEM_CACHE_BYTES");
+  const indexDiskOverride = envBytes("DS_INDEX_RUN_CACHE_MAX_BYTES");
+  const lexiconDiskOverride = envBytes("DS_LEXICON_INDEX_CACHE_MAX_BYTES");
+  const localBacklogMaxBytes = backlogOverride ?? 10 * 1024 * 1024 * 1024;
+  const sqliteCacheBytes =
+    sqliteCacheBytesOverride ??
+    (sqliteCacheMbOverride != null
+      ? sqliteCacheMbOverride * 1024 * 1024
+      : memoryLimitBytes > 0
+        ? Math.floor(memoryLimitBytes * 0.25)
+        : 0);
+  const workerSqliteCacheBytes =
+    workerSqliteCacheBytesOverride ??
+    (workerSqliteCacheMbOverride != null
+      ? workerSqliteCacheMbOverride * 1024 * 1024
+      : sqliteCacheBytes > 0
+        ? clampBytes(Math.floor(sqliteCacheBytes / 8), 8 * 1024 * 1024, 32 * 1024 * 1024)
+        : 0);
+  const tunedIndexMem =
+    indexMemOverride ??
+    (memoryLimitBytes > 0
+      ? clampBytes(Math.floor(memoryLimitBytes * 0.05), 8 * 1024 * 1024, 128 * 1024 * 1024)
+      : 64 * 1024 * 1024);
+  const companionSectionCacheBytes =
+    envBytes("DS_SEARCH_COMPANION_SECTION_CACHE_BYTES") ??
+    (memoryLimitBytes > 0
+      ? clampBytes(Math.floor(memoryLimitBytes * 0.02), 8 * 1024 * 1024, 128 * 1024 * 1024)
+      : 32 * 1024 * 1024);
+  const companionFileCacheBytes =
+    envBytes("DS_SEARCH_COMPANION_FILE_CACHE_MAX_BYTES") ??
+    clampBytes(Math.max(512 * 1024 * 1024, Math.floor(localBacklogMaxBytes * 0.1)), 256 * 1024 * 1024, 4 * 1024 * 1024 * 1024);
+  const segmentMaxBytes = envNum("DS_SEGMENT_MAX_BYTES", 16 * 1024 * 1024);
+  const searchWalOverlayMaxBytes = envBytes("DS_SEARCH_WAL_OVERLAY_MAX_BYTES") ?? segmentMaxBytes;
+  return {
+    autoTuneRequestedMemoryMb,
+    autoTunePresetMb,
+    autoTuneEffectiveMemoryLimitMb,
+    storage,
+    postgresMode,
+    postgresUrl,
+    host,
+    rootDir,
+    dbPath: process.env.DS_DB_PATH ?? `${rootDir}/wal.sqlite`,
+    segmentMaxBytes,
+    blockMaxBytes: envNum("DS_BLOCK_MAX_BYTES", 256 * 1024),
+    segmentTargetRows: envNum("DS_SEGMENT_TARGET_ROWS", 100_000),
+    segmentMaxIntervalMs: envNum("DS_SEGMENT_MAX_INTERVAL_MS", 0),
+    segmentCheckIntervalMs: envNum("DS_SEGMENT_CHECK_MS", 250),
+    segmenterWorkers: envNum("DS_SEGMENTER_WORKERS", 0),
+    uploadIntervalMs: envNum("DS_UPLOAD_CHECK_MS", 250),
+    uploadConcurrency: envNum("DS_UPLOAD_CONCURRENCY", 4),
+    segmentCacheMaxBytes: envNum("DS_SEGMENT_CACHE_MAX_BYTES", 256 * 1024 * 1024),
+    segmentFooterCacheEntries: envNum("DS_SEGMENT_FOOTER_CACHE_ENTRIES", 2048),
+    indexRunCacheMaxBytes: indexDiskOverride ?? 256 * 1024 * 1024,
+    indexRunMemoryCacheBytes: tunedIndexMem,
+    lexiconIndexCacheMaxBytes:
+      lexiconDiskOverride ??
+      (memoryLimitBytes > 0
+        ? clampBytes(Math.floor(memoryLimitBytes * 0.03), 8 * 1024 * 1024, 256 * 1024 * 1024)
+        : 64 * 1024 * 1024),
+    lexiconMappedCacheEntries: envNum("DS_LEXICON_MMAP_CACHE_ENTRIES", 64),
+    indexL0SpanSegments: envNum("DS_INDEX_L0_SPAN", 16),
+    indexBuildConcurrency: envNum("DS_INDEX_BUILD_CONCURRENCY", 4),
+    indexCheckIntervalMs: envNum("DS_INDEX_CHECK_MS", 1000),
+    searchCompanionBuildBatchSegments: envNum("DS_SEARCH_COMPANION_BATCH_SEGMENTS", 4),
+    searchCompanionYieldBlocks: envNum("DS_SEARCH_COMPANION_YIELD_BLOCKS", 4),
+    searchCompanionFileCacheMaxBytes: companionFileCacheBytes,
+    searchCompanionFileCacheMaxAgeMs: envNum("DS_SEARCH_COMPANION_FILE_CACHE_MAX_AGE_MS", 24 * 60 * 60 * 1000),
+    searchCompanionMappedCacheEntries: envNum("DS_SEARCH_COMPANION_MMAP_CACHE_ENTRIES", 64),
+    searchCompanionTocCacheBytes: envNum("DS_SEARCH_COMPANION_TOC_CACHE_BYTES", 1 * 1024 * 1024),
+    searchCompanionSectionCacheBytes: companionSectionCacheBytes,
+    searchWalOverlayQuietPeriodMs: envNum("DS_SEARCH_WAL_OVERLAY_QUIET_MS", 5_000),
+    searchWalOverlayMaxBytes,
+    indexCompactionFanout: envNum("DS_INDEX_COMPACTION_FANOUT", 16),
+    indexMaxLevel: envNum("DS_INDEX_MAX_LEVEL", 4),
+    indexCompactionConcurrency: envNum("DS_INDEX_COMPACT_CONCURRENCY", 4),
+    indexRetireGenWindow: envNum("DS_INDEX_RETIRE_GEN_WINDOW", 2),
+    indexRetireMinMs: envNum("DS_INDEX_RETIRE_MIN_MS", 5 * 60 * 1000),
+    readMaxBytes: envNum("DS_READ_MAX_BYTES", 1 * 1024 * 1024),
+    readMaxRecords: envNum("DS_READ_MAX_RECORDS", 1000),
+    appendMaxBodyBytes: envNum("DS_APPEND_MAX_BODY_BYTES", 10 * 1024 * 1024),
+    ingestFlushIntervalMs: envNum("DS_INGEST_FLUSH_MS", 10),
+    ingestMaxBatchRequests: envNum("DS_INGEST_MAX_BATCH_REQS", 200),
+    ingestMaxBatchBytes: envNum("DS_INGEST_MAX_BATCH_BYTES", 8 * 1024 * 1024),
+    ingestMaxQueueRequests: envNum("DS_INGEST_MAX_QUEUE_REQS", 50_000),
+    ingestMaxQueueBytes: envNum("DS_INGEST_MAX_QUEUE_BYTES", 64 * 1024 * 1024),
+    ingestConcurrency: envNum("DS_INGEST_CONCURRENCY", 2),
+    ingestBusyTimeoutMs: envNum("DS_INGEST_BUSY_MS", 5000),
+    localBacklogMaxBytes,
+    memoryLimitBytes,
+    sqliteCacheBytes,
+    workerSqliteCacheBytes,
+    readConcurrency: envNum("DS_READ_CONCURRENCY", 4),
+    searchConcurrency: envNum("DS_SEARCH_CONCURRENCY", 2),
+    asyncIndexConcurrency: envNum("DS_ASYNC_INDEX_CONCURRENCY", 1),
+    heapSnapshotPath: process.env.DS_HEAP_SNAPSHOT_PATH?.trim() || null,
+    memorySamplerPath: process.env.DS_MEMORY_SAMPLER_PATH?.trim() || null,
+    memorySamplerIntervalMs: envNum("DS_MEMORY_SAMPLER_INTERVAL_MS", 1_000),
+    objectStoreTimeoutMs: envNum("DS_OBJECTSTORE_TIMEOUT_MS", 5000),
+    objectStoreRetries: envNum("DS_OBJECTSTORE_RETRIES", 3),
+    objectStoreBaseDelayMs: envNum("DS_OBJECTSTORE_RETRY_BASE_MS", 50),
+    objectStoreMaxDelayMs: envNum("DS_OBJECTSTORE_RETRY_MAX_MS", 2000),
+    expirySweepIntervalMs: envNum("DS_EXPIRY_SWEEP_MS", 60_000),
+    expirySweepBatchLimit: envNum("DS_EXPIRY_SWEEP_LIMIT", 100),
+    metricsFlushIntervalMs: envNum("DS_METRICS_FLUSH_MS", 10_000),
+    touchWorkers: envNum("DS_TOUCH_WORKERS", 1),
+    touchCheckIntervalMs: envNum("DS_TOUCH_CHECK_MS", 250),
+    touchMaxBatchRows: envNum("DS_TOUCH_MAX_BATCH_ROWS", 500),
+    touchMaxBatchBytes: envNum("DS_TOUCH_MAX_BATCH_BYTES", 4 * 1024 * 1024),
+    otlpTracesStream: process.env.DS_OTLP_TRACES_STREAM?.trim() || null,
+    otlpAutoCreate: envBool("DS_OTLP_AUTO_CREATE", false),
+    port: envNum("PORT", 8080),
+  };
+}

package/src/db/bootstrap_store.ts

@@ -0,0 +1,9 @@
+import { SqliteDurableStore } from "./db";
+import type { BootstrapRestoreStore } from "../store/bootstrap_restore_store";
+
+export function createSqliteBootstrapRestoreStore(
+  dbPath: string,
+  opts: { cacheBytes?: number } = {}
+): BootstrapRestoreStore {
+  return new SqliteDurableStore(dbPath, opts);
+}