@prisma/streams-server 0.1.1 → 0.1.3

package/src/segment/segmenter.ts

@@ -6,6 +6,7 @@ import { encodeBlock, encodeFooter, type BlockIndexEntry, type SegmentRecord } f
 import { readU32BE } from "../util/endian";
 import { localSegmentPath, streamHash16Hex } from "../util/stream_paths";
 import { LruCache } from "../util/lru";
+import { RuntimeMemorySampler } from "../runtime_memory_sampler";
 import { yieldToEventLoop } from "../util/yield";
 
 export type SegmenterOptions = {
@@ -17,20 +18,42 @@ export type SegmenterOptions = {
 };
 
 export type SegmenterHooks = {
-  onSegmentSealed?: (payloadBytes: number, segmentBytes: number) => void;
+  onSegmentSealed?: (stream: string, payloadBytes: number, segmentBytes: number) => void;
 };
 
+export type SegmenterMemoryStats = {
+  active_builds: number;
+  active_streams: number;
+  active_payload_bytes: number;
+  active_segment_bytes_estimate: number;
+  active_rows: number;
+};
+
+const SEGMENT_COMPRESSION_WINDOW = 8;
+const MIN_COMPRESSED_FILL_RATIO = 0.5;
+
 export class Segmenter {
   private readonly config: Config;
   private readonly db: SqliteDurableStore;
   private readonly opts: Required<SegmenterOptions>;
   private readonly hooks?: SegmenterHooks;
+  private readonly memorySampler?: RuntimeMemorySampler;
   private timer: any | null = null;
   private running = false;
   private stopping = false;
   private readonly failures = new FailureTracker(1024);
-
-  constructor(config: Config, db: SqliteDurableStore, opts: SegmenterOptions = {}, hooks?: SegmenterHooks) {
+  private activeBuildStream: string | null = null;
+  private activePayloadBytes = 0;
+  private activeSegmentBytesEstimate = 0;
+  private activeRows = 0;
+
+  constructor(
+    config: Config,
+    db: SqliteDurableStore,
+    opts: SegmenterOptions = {},
+    hooks?: SegmenterHooks,
+    memorySampler?: RuntimeMemorySampler
+  ) {
     this.config = config;
     this.db = db;
     this.opts = {
@@ -41,6 +64,7 @@ export class Segmenter {
       maxRowsPerSegment: opts.maxRowsPerSegment ?? 250_000,
     };
     this.hooks = hooks;
+    this.memorySampler = memorySampler;
   }
 
   start(): void {
@@ -58,6 +82,16 @@ export class Segmenter {
     this.timer = null;
   }
 
+  getMemoryStats(): SegmenterMemoryStats {
+    return {
+      active_builds: this.activeBuildStream ? 1 : 0,
+      active_streams: this.activeBuildStream ? 1 : 0,
+      active_payload_bytes: this.activePayloadBytes,
+      active_segment_bytes_estimate: this.activeSegmentBytesEstimate,
+      active_rows: this.activeRows,
+    };
+  }
+
   private async tick(): Promise<void> {
     if (this.stopping) return;
     if (this.running) return;
@@ -129,11 +163,31 @@ export class Segmenter {
     }
   }
 
+  private resolvePayloadSealTargetBytes(stream: string): bigint {
+    const baseTarget = BigInt(this.config.segmentMaxBytes);
+    const ratio = this.db.recentSegmentCompressionRatio(stream, SEGMENT_COMPRESSION_WINDOW);
+    if (ratio == null || !Number.isFinite(ratio) || ratio <= 0 || ratio >= MIN_COMPRESSED_FILL_RATIO) {
+      return baseTarget;
+    }
+    const desiredCompressedBytes = Math.ceil(this.config.segmentMaxBytes * MIN_COMPRESSED_FILL_RATIO);
+    const boosted = BigInt(Math.ceil(desiredCompressedBytes / ratio));
+    return boosted > baseTarget ? boosted : baseTarget;
+  }
+
+  private shouldSealStream(row: { stream: string; pending_bytes: bigint; pending_rows: bigint; last_segment_cut_ms: bigint }): boolean {
+    const payloadSealTargetBytes = this.resolvePayloadSealTargetBytes(row.stream);
+    if (row.pending_bytes >= payloadSealTargetBytes) return true;
+    if (row.pending_rows >= BigInt(this.opts.minCandidateRows)) return true;
+    if (this.opts.maxIntervalMs > 0 && BigInt(Date.now()) - row.last_segment_cut_ms >= BigInt(this.opts.maxIntervalMs)) return true;
+    return false;
+  }
+
   private async buildOne(stream: string): Promise<void> {
     if (this.stopping) return;
     const row = this.db.getStream(stream);
     if (!row || this.db.isDeleted(row)) return;
     if (row.segment_in_progress) return;
+    if (!this.shouldSealStream(row)) return;
 
     const startOffset = row.sealed_through + 1n;
     const maxOffset = row.next_offset - 1n;
@@ -143,10 +197,18 @@ export class Segmenter {
     if (!this.db.tryClaimSegment(stream)) return;
 
     try {
+      this.activeBuildStream = stream;
+      this.activePayloadBytes = 0;
+      this.activeSegmentBytesEstimate = 0;
+      this.activeRows = 0;
       const segmentIndex = this.db.nextSegmentIndexForStream(stream);
       const shash = streamHash16Hex(stream);
       const localPath = localSegmentPath(this.config.rootDir, shash, segmentIndex);
       const tmpPath = `${localPath}.tmp`;
+      const leaveCutPhase = this.memorySampler?.enter("cut", {
+        stream,
+        segment_index: segmentIndex,
+      });
       mkdirSync(dirname(localPath), { recursive: true });
 
       // Build blocks and stream-write to temp file.
@@ -161,6 +223,8 @@ export class Segmenter {
 
         // Decide endOffset by scanning WAL rows until threshold.
         // IMPORTANT: pending_bytes tracks WAL payload bytes only (not record/block overhead).
+        const payloadSealTargetBytes = this.resolvePayloadSealTargetBytes(stream);
+        const rowSealTarget = BigInt(this.opts.minCandidateRows);
         let payloadBytes = 0n;
         let rowsSealed = 0n;
         let endOffset = startOffset - 1n;
@@ -210,6 +274,9 @@ export class Segmenter {
           payloadBytes += BigInt(payload.byteLength);
           rowsSealed += 1n;
           endOffset = offset;
+          this.activePayloadBytes = Number(payloadBytes);
+          this.activeRows = Number(rowsSealed);
+          this.activeSegmentBytesEstimate = fileBytes + blockBytesApprox;
 
           recordsSinceYield += 1;
           if (recordsSinceYield >= 512 || Date.now() - lastYieldMs >= 10) {
@@ -218,7 +285,8 @@ export class Segmenter {
             recordsSinceYield = 0;
           }
 
-          if (payloadBytes >= BigInt(this.config.segmentMaxBytes)) break;
+          if (payloadBytes >= payloadSealTargetBytes) break;
+          if (rowsSealed >= rowSealTarget) break;
           if (rowsSealed >= BigInt(this.opts.maxRowsPerSegment)) break;
         }
 
@@ -244,6 +312,7 @@ export class Segmenter {
         const footer = encodeFooter(blockIndex);
         writeSync(fd, footer);
         fileBytes += footer.byteLength;
+        this.activeSegmentBytesEstimate = fileBytes;
 
         fsyncSync(fd);
 
@@ -267,7 +336,7 @@ export class Segmenter {
                 rowsSealed,
               });
             });
-            if (this.hooks?.onSegmentSealed) this.hooks.onSegmentSealed(Number(payloadBytes), fileBytes);
+            if (this.hooks?.onSegmentSealed) this.hooks.onSegmentSealed(stream, Number(payloadBytes), fileBytes);
           } catch (e) {
             try {
               if (existsSync(localPath)) unlinkSync(localPath);
@@ -280,8 +349,13 @@ export class Segmenter {
       } finally {
         closeSync(fd);
         this.cleanupTmp(tmpPath);
+        leaveCutPhase?.();
       }
     } finally {
+      this.activeBuildStream = null;
+      this.activePayloadBytes = 0;
+      this.activeSegmentBytesEstimate = 0;
+      this.activeRows = 0;
       // Release claim.
       if (!this.stopping) {
         try {

package/src/segment/segmenter_worker.ts

@@ -1,29 +1,53 @@
-import { parentPort, workerData } from "node:worker_threads";
+import { parentPort, workerData, threadId } from "node:worker_threads";
 import type { Config } from "../config.ts";
 import { SqliteDurableStore } from "../db/db.ts";
+import type { HostRuntime } from "../runtime/host_runtime.ts";
+import { RuntimeMemorySampler } from "../runtime_memory_sampler.ts";
+import { setSqliteRuntimeOverride } from "../sqlite/adapter.ts";
 import { Segmenter, type SegmenterHooks, type SegmenterOptions } from "./segmenter.ts";
 import { initConsoleLogging } from "../util/log.ts";
 
 initConsoleLogging();
 
-const data = workerData as { config: Config; opts?: SegmenterOptions };
+const data = workerData as { config: Config; hostRuntime?: HostRuntime; opts?: SegmenterOptions };
 const cfg = data.config;
+setSqliteRuntimeOverride(data.hostRuntime ?? null);
 // The main server process initializes/migrates schema; workers should avoid
 // concurrent migrations on the same sqlite file.
-const db = new SqliteDurableStore(cfg.dbPath, { cacheBytes: cfg.sqliteCacheBytes, skipMigrations: true });
+const db = new SqliteDurableStore(cfg.dbPath, { cacheBytes: cfg.workerSqliteCacheBytes, skipMigrations: true });
+const memorySampler =
+  cfg.memorySamplerPath != null
+    ? new RuntimeMemorySampler(cfg.memorySamplerPath, {
+        intervalMs: cfg.memorySamplerIntervalMs,
+        scope: `segmenter-worker-${threadId}`,
+      })
+    : undefined;
+memorySampler?.start();
 
 const hooks: SegmenterHooks = {
-  onSegmentSealed: (payloadBytes, segmentBytes) => {
-    parentPort?.postMessage({ type: "sealed", payloadBytes, segmentBytes });
+  onSegmentSealed: (stream, payloadBytes, segmentBytes) => {
+    parentPort?.postMessage({ type: "sealed", stream, payloadBytes, segmentBytes });
   },
 };
 
-const segmenter = new Segmenter(cfg, db, data.opts ?? {}, hooks);
+const segmenter = new Segmenter(cfg, db, data.opts ?? {}, hooks, memorySampler);
 segmenter.start();
+const memoryTimer = setInterval(() => {
+  try {
+    parentPort?.postMessage({ type: "memory", workerId: threadId, stats: segmenter.getMemoryStats() });
+  } catch {
+    // ignore
+  }
+}, 1_000);
 
 parentPort?.on("message", (msg: any) => {
   if (!msg || typeof msg !== "object") return;
   if (msg.type === "stop") {
+    try {
+      clearInterval(memoryTimer);
+    } catch {
+      // ignore
+    }
     try {
       segmenter.stop();
     } catch {
@@ -34,6 +58,11 @@ parentPort?.on("message", (msg: any) => {
     } catch {
       // ignore
     }
+    try {
+      memorySampler?.stop();
+    } catch {
+      // ignore
+    }
     try {
       parentPort?.postMessage({ type: "stopped" });
     } catch {

package/src/segment/segmenter_workers.ts

@@ -1,17 +1,18 @@
-import { existsSync } from "node:fs";
-import { resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { Worker } from "node:worker_threads";
 import type { Config } from "../config";
-import type { SegmenterHooks, SegmenterOptions } from "./segmenter";
+import { detectHostRuntime } from "../runtime/host_runtime.ts";
+import type { SegmenterHooks, SegmenterMemoryStats, SegmenterOptions } from "./segmenter";
 
 export type SegmenterController = {
   start: () => void;
   stop: (hard?: boolean) => void;
+  getMemoryStats?: () => SegmenterMemoryStats;
 };
 
 type WorkerMessage =
-  | { type: "sealed"; payloadBytes: number; segmentBytes: number }
+  | { type: "sealed"; stream: string; payloadBytes: number; segmentBytes: number }
+  | { type: "memory"; workerId: number; stats: SegmenterMemoryStats }
   | { type: "stopped" };
 
 export class SegmenterWorkerPool implements SegmenterController {
@@ -20,6 +21,7 @@ export class SegmenterWorkerPool implements SegmenterController {
   private readonly opts: SegmenterOptions;
   private readonly hooks?: SegmenterHooks;
   private readonly workers: Worker[] = [];
+  private readonly workerMemory = new Map<number, { stats: SegmenterMemoryStats; reportedAtMs: number }>();
   private started = false;
 
   constructor(config: Config, workerCount: number, opts: SegmenterOptions = {}, hooks?: SegmenterHooks) {
@@ -49,20 +51,42 @@ export class SegmenterWorkerPool implements SegmenterController {
       void w.terminate();
     }
     this.workers.length = 0;
+    this.workerMemory.clear();
   }
 
-  private spawnWorker(idx: number): void {
-    const workerUrl = new URL("./segmenter_worker.ts", import.meta.url);
-    let workerSpec = fileURLToPath(workerUrl);
-    if (!existsSync(workerSpec)) {
-      const fallback = resolve(process.cwd(), "src/segment/segmenter_worker.ts");
-      if (existsSync(fallback)) {
-        workerSpec = fallback;
+  getMemoryStats(): SegmenterMemoryStats {
+    const now = Date.now();
+    let activeBuilds = 0;
+    let activeStreams = 0;
+    let activePayloadBytes = 0;
+    let activeSegmentBytesEstimate = 0;
+    let activeRows = 0;
+    for (const [workerId, entry] of this.workerMemory) {
+      if (now - entry.reportedAtMs > 5_000) {
+        this.workerMemory.delete(workerId);
+        continue;
       }
+      activeBuilds += Math.max(0, entry.stats.active_builds);
+      activeStreams += Math.max(0, entry.stats.active_streams);
+      activePayloadBytes += Math.max(0, entry.stats.active_payload_bytes);
+      activeSegmentBytesEstimate += Math.max(0, entry.stats.active_segment_bytes_estimate);
+      activeRows += Math.max(0, entry.stats.active_rows);
     }
+    return {
+      active_builds: activeBuilds,
+      active_streams: activeStreams,
+      active_payload_bytes: activePayloadBytes,
+      active_segment_bytes_estimate: activeSegmentBytesEstimate,
+      active_rows: activeRows,
+    };
+  }
+
+  private spawnWorker(idx: number): void {
+    const workerSpec = fileURLToPath(new URL("./segmenter_worker.ts", import.meta.url));
     const worker = new Worker(workerSpec, {
       workerData: {
         config: this.config,
+        hostRuntime: detectHostRuntime(),
         opts: this.opts,
       },
       type: "module",
@@ -71,7 +95,12 @@ export class SegmenterWorkerPool implements SegmenterController {
 
     worker.on("message", (msg: WorkerMessage) => {
       if (msg?.type === "sealed") {
-        this.hooks?.onSegmentSealed?.(msg.payloadBytes, msg.segmentBytes);
+        this.hooks?.onSegmentSealed?.(msg.stream, msg.payloadBytes, msg.segmentBytes);
+      } else if (msg?.type === "memory") {
+        this.workerMemory.set(msg.workerId, {
+          stats: msg.stats,
+          reportedAtMs: Date.now(),
+        });
       }
     });
 
@@ -81,6 +110,7 @@ export class SegmenterWorkerPool implements SegmenterController {
     });
 
     worker.on("exit", (code) => {
+      this.workerMemory.delete(worker.threadId);
       if (!this.started) return;
       if (code !== 0) {
         // eslint-disable-next-line no-console

package/src/server.ts

@@ -6,6 +6,7 @@ import { MockR2Store } from "./objectstore/mock_r2";
 import { R2ObjectStore } from "./objectstore/r2";
 import { bootstrapFromR2 } from "./bootstrap";
 import { initConsoleLogging } from "./util/log";
+import { AUTO_TUNE_PRESETS, memoryLimitForPreset, tuneForPreset, type AutoTuneConfig } from "./auto_tune";
 
 initConsoleLogging();
 
@@ -36,21 +37,6 @@ function formatPresetList<T>(presets: number[], selected: number, map: (preset:
     .join(", ");
 }
 
-type AutoTuneConfig = {
-  sqliteCacheMb: number;
-  indexMemMb: number;
-  ingestBatchMb: number;
-  ingestQueueMb: number;
-  indexBuildConcurrency: number;
-  indexCompactConcurrency: number;
-  segmenterWorkers: number;
-  uploadConcurrency: number;
-};
-
-function memoryLimitForPreset(preset: number): number {
-  return preset === 256 ? 300 : preset;
-}
-
 function applyAutoTune(overrideMb: number | null): void {
   const envMemRaw = process.env.DS_MEMORY_LIMIT_MB;
   if (overrideMb != null) {
@@ -74,11 +60,24 @@ function applyAutoTune(overrideMb: number | null): void {
   }
 
   const conflictVars = [
+    "DS_SEGMENT_MAX_BYTES",
+    "DS_SEGMENT_TARGET_ROWS",
     "DS_SQLITE_CACHE_MB",
     "DS_SQLITE_CACHE_BYTES",
+    "DS_WORKER_SQLITE_CACHE_MB",
+    "DS_WORKER_SQLITE_CACHE_BYTES",
     "DS_INDEX_RUN_MEM_CACHE_BYTES",
+    "DS_LEXICON_INDEX_CACHE_MAX_BYTES",
     "DS_INGEST_MAX_BATCH_BYTES",
     "DS_INGEST_MAX_QUEUE_BYTES",
+    "DS_INGEST_CONCURRENCY",
+    "DS_READ_CONCURRENCY",
+    "DS_SEARCH_CONCURRENCY",
+    "DS_ASYNC_INDEX_CONCURRENCY",
+    "DS_SEARCH_COMPANION_TOC_CACHE_BYTES",
+    "DS_SEARCH_COMPANION_SECTION_CACHE_BYTES",
+    "DS_SEARCH_COMPANION_BATCH_SEGMENTS",
+    "DS_SEARCH_COMPANION_YIELD_BLOCKS",
   ];
   const conflicts = conflictVars.filter((v) => process.env[v] != null);
   if (conflicts.length > 0) {
@@ -86,35 +85,39 @@ function applyAutoTune(overrideMb: number | null): void {
     process.exit(1);
   }
 
-  const presets = [256, 512, 1024, 2048, 4096, 8192];
+  const presets = [...AUTO_TUNE_PRESETS];
   const preset = [...presets].reverse().find((v) => v <= memMb);
   if (!preset) {
     console.error(`DS_MEMORY_LIMIT_MB=${memMb} is below the minimum preset (256)`);
     process.exit(1);
   }
-
-  const tuneFor = (p: number): AutoTuneConfig => ({
-    sqliteCacheMb: Math.max(8, Math.floor(p / 16)),
-    indexMemMb: Math.max(4, Math.floor(p / 64)),
-    ingestBatchMb: Math.max(2, Math.floor(p / 128)),
-    ingestQueueMb: Math.max(8, Math.floor(p / 32)),
-    indexBuildConcurrency: p >= 8192 ? 8 : p >= 4096 ? 4 : p >= 1024 ? 2 : 1,
-    indexCompactConcurrency: p >= 4096 ? 4 : p >= 1024 ? 2 : 1,
-    segmenterWorkers: p >= 8192 ? 8 : p >= 4096 ? 4 : p >= 1024 ? 2 : 1,
-    uploadConcurrency: p >= 8192 ? 16 : p >= 4096 ? 8 : p >= 1024 ? 4 : 2,
-  });
-  const tune = tuneFor(preset);
+  const tune: AutoTuneConfig = tuneForPreset(preset);
 
   const memoryLimitMb = memoryLimitForPreset(preset);
+  process.env.DS_AUTO_TUNE_REQUESTED_MB = String(memMb);
+  process.env.DS_AUTO_TUNE_PRESET_MB = String(preset);
+  process.env.DS_AUTO_TUNE_EFFECTIVE_MEMORY_LIMIT_MB = String(memoryLimitMb);
   process.env.DS_MEMORY_LIMIT_MB = String(memoryLimitMb);
+  process.env.DS_SEGMENT_MAX_BYTES = String(tune.segmentMaxMiB * 1024 * 1024);
+  process.env.DS_SEGMENT_TARGET_ROWS = String(tune.segmentTargetRows);
   process.env.DS_SQLITE_CACHE_MB = String(tune.sqliteCacheMb);
+  process.env.DS_WORKER_SQLITE_CACHE_MB = String(tune.workerSqliteCacheMb);
   process.env.DS_INDEX_RUN_MEM_CACHE_BYTES = String(tune.indexMemMb * 1024 * 1024);
+  process.env.DS_LEXICON_INDEX_CACHE_MAX_BYTES = String(tune.lexiconIndexCacheMb * 1024 * 1024);
+  process.env.DS_SEARCH_COMPANION_TOC_CACHE_BYTES = String(tune.searchCompanionTocCacheMb * 1024 * 1024);
+  process.env.DS_SEARCH_COMPANION_SECTION_CACHE_BYTES = String(tune.searchCompanionSectionCacheMb * 1024 * 1024);
   process.env.DS_INGEST_MAX_BATCH_BYTES = String(tune.ingestBatchMb * 1024 * 1024);
   process.env.DS_INGEST_MAX_QUEUE_BYTES = String(tune.ingestQueueMb * 1024 * 1024);
+  process.env.DS_INGEST_CONCURRENCY = String(tune.ingestConcurrency);
+  process.env.DS_READ_CONCURRENCY = String(tune.readConcurrency);
+  process.env.DS_SEARCH_CONCURRENCY = String(tune.searchConcurrency);
+  process.env.DS_ASYNC_INDEX_CONCURRENCY = String(tune.asyncIndexConcurrency);
   process.env.DS_INDEX_BUILD_CONCURRENCY = String(tune.indexBuildConcurrency);
   process.env.DS_INDEX_COMPACT_CONCURRENCY = String(tune.indexCompactConcurrency);
   process.env.DS_SEGMENTER_WORKERS = String(tune.segmenterWorkers);
   process.env.DS_UPLOAD_CONCURRENCY = String(tune.uploadConcurrency);
+  process.env.DS_SEARCH_COMPANION_BATCH_SEGMENTS = String(tune.searchCompanionBatchSegments);
+  process.env.DS_SEARCH_COMPANION_YIELD_BLOCKS = String(tune.searchCompanionYieldBlocks);
 
   const presetLine = formatPresetList(presets, preset, (v) => v, (v) => String(v));
   console.log(`Auto-tuning for memory preset ${presetLine}`);
@@ -122,22 +125,117 @@ function applyAutoTune(overrideMb: number | null): void {
     `DS_MEMORY_LIMIT_MB presets: ${formatPresetList(presets, preset, (p) => memoryLimitForPreset(p), (v) => String(v))}`
   );
   console.log(
-    `DS_SQLITE_CACHE_MB presets: ${formatPresetList(presets, preset, (p) => tuneFor(p).sqliteCacheMb, (v) => String(v))}`
+    `DS_SEGMENT_MAX_MIB presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).segmentMaxMiB,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_SEGMENT_TARGET_ROWS presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).segmentTargetRows,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_SQLITE_CACHE_MB presets: ${formatPresetList(presets, preset, (p) => tuneForPreset(p).sqliteCacheMb, (v) => String(v))}`
+  );
+  console.log(
+    `DS_WORKER_SQLITE_CACHE_MB presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).workerSqliteCacheMb,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_INDEX_RUN_MEM_CACHE_MB presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).indexMemMb,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_LEXICON_INDEX_CACHE_MB presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).lexiconIndexCacheMb,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_SEARCH_COMPANION_TOC_CACHE_MB presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).searchCompanionTocCacheMb,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_SEARCH_COMPANION_SECTION_CACHE_MB presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).searchCompanionSectionCacheMb,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_INGEST_MAX_BATCH_MB presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).ingestBatchMb,
+      (v) => String(v)
+    )}`
   );
   console.log(
-    `DS_INDEX_RUN_MEM_CACHE_MB presets: ${formatPresetList(presets, preset, (p) => tuneFor(p).indexMemMb, (v) => String(v))}`
+    `DS_INGEST_MAX_QUEUE_MB presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).ingestQueueMb,
+      (v) => String(v)
+    )}`
   );
   console.log(
-    `DS_INGEST_MAX_BATCH_MB presets: ${formatPresetList(presets, preset, (p) => tuneFor(p).ingestBatchMb, (v) => String(v))}`
+    `DS_INGEST_CONCURRENCY presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).ingestConcurrency,
+      (v) => String(v)
+    )}`
   );
   console.log(
-    `DS_INGEST_MAX_QUEUE_MB presets: ${formatPresetList(presets, preset, (p) => tuneFor(p).ingestQueueMb, (v) => String(v))}`
+    `DS_READ_CONCURRENCY presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).readConcurrency,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_SEARCH_CONCURRENCY presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).searchConcurrency,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_ASYNC_INDEX_CONCURRENCY presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).asyncIndexConcurrency,
+      (v) => String(v)
+    )}`
   );
   console.log(
     `DS_INDEX_BUILD_CONCURRENCY presets: ${formatPresetList(
       presets,
       preset,
-      (p) => tuneFor(p).indexBuildConcurrency,
+      (p) => tuneForPreset(p).indexBuildConcurrency,
       (v) => String(v)
     )}`
   );
@@ -145,7 +243,7 @@ function applyAutoTune(overrideMb: number | null): void {
     `DS_INDEX_COMPACT_CONCURRENCY presets: ${formatPresetList(
       presets,
       preset,
-      (p) => tuneFor(p).indexCompactConcurrency,
+      (p) => tuneForPreset(p).indexCompactConcurrency,
       (v) => String(v)
     )}`
   );
@@ -153,7 +251,7 @@ function applyAutoTune(overrideMb: number | null): void {
     `DS_SEGMENTER_WORKERS presets: ${formatPresetList(
       presets,
       preset,
-      (p) => tuneFor(p).segmenterWorkers,
+      (p) => tuneForPreset(p).segmenterWorkers,
       (v) => String(v)
     )}`
   );
@@ -161,7 +259,23 @@ function applyAutoTune(overrideMb: number | null): void {
     `DS_UPLOAD_CONCURRENCY presets: ${formatPresetList(
       presets,
       preset,
-      (p) => tuneFor(p).uploadConcurrency,
+      (p) => tuneForPreset(p).uploadConcurrency,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_SEARCH_COMPANION_BATCH_SEGMENTS presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).searchCompanionBatchSegments,
+      (v) => String(v)
+    )}`
+  );
+  console.log(
+    `DS_SEARCH_COMPANION_YIELD_BLOCKS presets: ${formatPresetList(
+      presets,
+      preset,
+      (p) => tuneForPreset(p).searchCompanionYieldBlocks,
       (v) => String(v)
     )}`
   );