@prisma/streams-server 0.1.1 → 0.1.3

package/src/search/schema.ts

@@ -0,0 +1,245 @@
+import { Result } from "better-result";
+import type { SchemaRegistry, SearchConfig, SearchFieldBinding, SearchFieldConfig } from "../schema/registry";
+import { resolvePointerResult } from "../util/json_pointer";
+import { schemaVersionForOffset } from "../schema/read_json";
+
+export type SearchExactTerm = {
+  field: string;
+  config: SearchFieldConfig;
+  canonical: string;
+  bytes: Uint8Array;
+};
+
+export function resolveSearchAlias(search: SearchConfig | undefined, fieldName: string): string {
+  return search?.aliases?.[fieldName] ?? fieldName;
+}
+
+export function getSearchFieldConfig(search: SearchConfig | undefined, fieldName: string): SearchFieldConfig | null {
+  const resolved = resolveSearchAlias(search, fieldName);
+  return search?.fields?.[resolved] ?? null;
+}
+
+export function getSearchFieldBinding(config: SearchFieldConfig, version: number): SearchFieldBinding | null {
+  let selected: SearchFieldBinding | null = null;
+  for (const binding of config.bindings) {
+    if (binding.version <= version && (!selected || binding.version > selected.version)) {
+      selected = binding;
+    }
+  }
+  return selected;
+}
+
+export function normalizeKeywordValue(value: unknown, normalizer: SearchFieldConfig["normalizer"]): string | null {
+  if (typeof value !== "string") return null;
+  return normalizer === "lowercase_v1" ? value.toLowerCase() : value;
+}
+
+export function canonicalizeExactValue(config: SearchFieldConfig, value: unknown): string | null {
+  switch (config.kind) {
+    case "keyword":
+      return normalizeKeywordValue(value, config.normalizer);
+    case "integer":
+      if (typeof value === "bigint") return value.toString();
+      if (typeof value === "number" && Number.isFinite(value) && Number.isInteger(value)) return String(value);
+      if (typeof value === "string" && /^-?(0|[1-9][0-9]*)$/.test(value.trim())) return String(BigInt(value.trim()));
+      return null;
+    case "float":
+      if (typeof value === "bigint") return value.toString();
+      if (typeof value === "number" && Number.isFinite(value)) return String(value);
+      if (typeof value === "string" && value.trim() !== "") {
+        const n = Number(value);
+        if (Number.isFinite(n)) return String(n);
+      }
+      return null;
+    case "date":
+      if (typeof value === "number" && Number.isFinite(value)) return String(Math.trunc(value));
+      if (typeof value === "bigint") return value.toString();
+      if (typeof value === "string" && value.trim() !== "") {
+        const parsed = Date.parse(value);
+        if (Number.isFinite(parsed)) return String(Math.trunc(parsed));
+        if (/^-?(0|[1-9][0-9]*)$/.test(value.trim())) return String(BigInt(value.trim()));
+      }
+      return null;
+    case "bool":
+      if (typeof value === "boolean") return value ? "true" : "false";
+      if (typeof value === "string") {
+        const lowered = value.trim().toLowerCase();
+        if (lowered === "true" || lowered === "false") return lowered;
+      }
+      return null;
+    default:
+      return null;
+  }
+}
+
+export function canonicalizeColumnValue(config: SearchFieldConfig, value: unknown): bigint | number | boolean | null {
+  switch (config.kind) {
+    case "integer": {
+      const canonical = canonicalizeExactValue(config, value);
+      return canonical == null ? null : BigInt(canonical);
+    }
+    case "date": {
+      const canonical = canonicalizeExactValue(config, value);
+      return canonical == null ? null : BigInt(canonical);
+    }
+    case "float": {
+      const canonical = canonicalizeExactValue(config, value);
+      if (canonical == null) return null;
+      const parsed = Number(canonical);
+      return Number.isFinite(parsed) ? parsed : null;
+    }
+    case "bool":
+      return canonicalizeExactValue(config, value) === "true"
+        ? true
+        : canonicalizeExactValue(config, value) === "false"
+          ? false
+          : null;
+    default:
+      return null;
+  }
+}
+
+export function analyzeTextValue(value: string, analyzer: SearchFieldConfig["analyzer"]): string[] {
+  if (analyzer !== "unicode_word_v1") return [];
+  const matches = value.toLowerCase().match(/[\p{L}\p{N}]+/gu);
+  return matches ? matches.filter((token) => token.length > 0) : [];
+}
+
+function addRawValues(out: unknown[], value: unknown): void {
+  if (Array.isArray(value)) {
+    for (const item of value) addRawValues(out, item);
+    return;
+  }
+  out.push(value);
+}
+
+export function extractRawSearchValuesForFieldsResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown,
+  fieldNames: Iterable<string>
+): Result<Map<string, unknown[]>, { message: string }> {
+  if (!reg.search) return Result.ok(new Map());
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return Result.err({ message: "search fields require JSON object records" });
+  }
+  const version = schemaVersionForOffset(reg, offset);
+  const out = new Map<string, unknown[]>();
+  for (const fieldName of fieldNames) {
+    const config = reg.search.fields[fieldName];
+    if (!config) continue;
+    const binding = getSearchFieldBinding(config, version);
+    if (!binding) continue;
+    const resolvedRes = resolvePointerResult(value, binding.jsonPointer);
+    if (Result.isError(resolvedRes)) return Result.err({ message: resolvedRes.error.message });
+    if (!resolvedRes.value.exists) continue;
+    const values: unknown[] = [];
+    addRawValues(values, resolvedRes.value.value);
+    if (values.length > 0) out.set(fieldName, values);
+  }
+  return Result.ok(out);
+}
+
+export function extractRawSearchValuesResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<Map<string, unknown[]>, { message: string }> {
+  return extractRawSearchValuesForFieldsResult(reg, offset, value, Object.keys(reg.search?.fields ?? {}));
+}
+
+export function extractSearchExactTermsResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<SearchExactTerm[], { message: string }> {
+  const rawValuesRes = extractRawSearchValuesResult(reg, offset, value);
+  if (Result.isError(rawValuesRes)) return rawValuesRes;
+  const out: SearchExactTerm[] = [];
+  const seen = new Set<string>();
+  for (const [fieldName, values] of rawValuesRes.value) {
+    const config = reg.search?.fields[fieldName];
+    if (!config?.exact) continue;
+    for (const rawValue of values) {
+      const canonical = canonicalizeExactValue(config, rawValue);
+      if (canonical == null) continue;
+      const dedupeKey = `${fieldName}\u0000${canonical}`;
+      if (seen.has(dedupeKey)) continue;
+      seen.add(dedupeKey);
+      out.push({
+        field: fieldName,
+        config,
+        canonical,
+        bytes: new TextEncoder().encode(canonical),
+      });
+    }
+  }
+  return Result.ok(out);
+}
+
+export function extractSearchExactValuesResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<Map<string, string[]>, { message: string }> {
+  const rawValuesRes = extractRawSearchValuesResult(reg, offset, value);
+  if (Result.isError(rawValuesRes)) return rawValuesRes;
+  const out = new Map<string, string[]>();
+  for (const [fieldName, values] of rawValuesRes.value) {
+    const config = reg.search?.fields[fieldName];
+    if (!config) continue;
+    const exactValues: string[] = [];
+    for (const rawValue of values) {
+      const canonical = canonicalizeExactValue(config, rawValue);
+      if (canonical != null) exactValues.push(canonical);
+    }
+    if (exactValues.length > 0) out.set(fieldName, exactValues);
+  }
+  return Result.ok(out);
+}
+
+export function extractSearchColumnValuesResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<Map<string, Array<bigint | number | boolean>>, { message: string }> {
+  const rawValuesRes = extractRawSearchValuesResult(reg, offset, value);
+  if (Result.isError(rawValuesRes)) return rawValuesRes;
+  const out = new Map<string, Array<bigint | number | boolean>>();
+  for (const [fieldName, values] of rawValuesRes.value) {
+    const config = reg.search?.fields[fieldName];
+    if (!config?.column) continue;
+    const colValues: Array<bigint | number | boolean> = [];
+    for (const rawValue of values) {
+      const normalized = canonicalizeColumnValue(config, rawValue);
+      if (normalized != null) colValues.push(normalized);
+    }
+    if (colValues.length > 0) out.set(fieldName, colValues);
+  }
+  return Result.ok(out);
+}
+
+export function extractSearchTextValuesResult(
+  reg: SchemaRegistry,
+  offset: bigint,
+  value: unknown
+): Result<Map<string, string[]>, { message: string }> {
+  const rawValuesRes = extractRawSearchValuesResult(reg, offset, value);
+  if (Result.isError(rawValuesRes)) return rawValuesRes;
+  const out = new Map<string, string[]>();
+  for (const [fieldName, values] of rawValuesRes.value) {
+    const config = reg.search?.fields[fieldName];
+    if (!config) continue;
+    const textValues: string[] = [];
+    for (const rawValue of values) {
+      if (config.kind === "keyword") {
+        const normalized = normalizeKeywordValue(rawValue, config.normalizer);
+        if (normalized != null) textValues.push(normalized);
+      } else if (config.kind === "text" && typeof rawValue === "string") {
+        textValues.push(rawValue);
+      }
+    }
+    if (textValues.length > 0) out.set(fieldName, textValues);
+  }
+  return Result.ok(out);
+}

package/src/segment/cache.ts

@@ -1,5 +1,13 @@
 import { mkdirSync, readdirSync, statSync, unlinkSync, renameSync, existsSync, writeFileSync, readFileSync } from "node:fs";
 import { dirname, join, relative } from "node:path";
+import { LruCache } from "../util/lru";
+
+export type MappedSegmentFile = {
+  objectKey: string;
+  path: string;
+  bytes: Uint8Array;
+  sizeBytes: number;
+};
 
 export type SegmentCacheStats = {
   hits: number;
@@ -9,21 +17,27 @@ export type SegmentCacheStats = {
   usedBytes: number;
   maxBytes: number;
   entryCount: number;
+  mappedBytes: number;
+  mappedEntryCount: number;
+  pinnedEntryCount: number;
 };
 
 export class SegmentDiskCache {
   private readonly rootDir: string;
   private readonly maxBytes: number;
   private readonly entries = new Map<string, { path: string; size: number }>();
+  private readonly pinnedKeys = new Set<string>();
+  private readonly mappedFiles: LruCache<string, MappedSegmentFile>;
   private totalBytes = 0;
   private hits = 0;
   private misses = 0;
   private evictions = 0;
   private bytesAdded = 0;
 
-  constructor(rootDir: string, maxBytes: number) {
+  constructor(rootDir: string, maxBytes: number, mappedEntries = 64) {
     this.rootDir = rootDir;
     this.maxBytes = maxBytes;
+    this.mappedFiles = new LruCache(Math.max(1, mappedEntries));
     if (this.maxBytes > 0) {
       mkdirSync(this.rootDir, { recursive: true });
       this.loadIndex();
@@ -86,8 +100,43 @@ export class SegmentDiskCache {
     }
     this.recordHit();
     this.touch(objectKey);
+    const mapped = this.getMapped(objectKey);
+    if (mapped) return mapped.bytes;
+    const path = this.getPath(objectKey);
+    return readFileSync(path);
+  }
+
+  getMapped(objectKey: string): MappedSegmentFile | null {
+    if (!this.has(objectKey)) return null;
+    const cached = this.mappedFiles.get(objectKey);
+    if (cached) {
+      this.pinnedKeys.add(objectKey);
+      this.touch(objectKey);
+      return cached;
+    }
+
     const path = this.getPath(objectKey);
-    return new Uint8Array(readFileSync(path));
+    let sizeBytes: number;
+    try {
+      sizeBytes = statSync(path).size;
+    } catch {
+      this.entries.delete(objectKey);
+      return null;
+    }
+
+    let bytes: Uint8Array;
+    try {
+      bytes = (Bun as any).mmap(path, { shared: true }) as Uint8Array;
+    } catch {
+      return null;
+    }
+    if (bytes.byteLength !== sizeBytes) return null;
+
+    const mapped = { objectKey, path, bytes, sizeBytes };
+    this.mappedFiles.set(objectKey, mapped);
+    this.pinnedKeys.add(objectKey);
+    this.touch(objectKey);
+    return mapped;
   }
 
   put(objectKey: string, bytes: Uint8Array): boolean {
@@ -111,6 +160,7 @@ export class SegmentDiskCache {
     }
     const existing = this.entries.get(objectKey);
     if (existing) this.totalBytes = Math.max(0, this.totalBytes - existing.size);
+    this.mappedFiles.delete(objectKey);
     this.entries.set(objectKey, { path: dest, size: sizeBytes });
     this.totalBytes += sizeBytes;
     this.bytesAdded += sizeBytes;
@@ -130,6 +180,7 @@ export class SegmentDiskCache {
     }
     const existing = this.entries.get(objectKey);
     if (existing) this.totalBytes = Math.max(0, this.totalBytes - existing.size);
+    this.mappedFiles.delete(objectKey);
     this.entries.set(objectKey, { path: dest, size: sizeBytes });
     this.totalBytes += sizeBytes;
     this.bytesAdded += sizeBytes;
@@ -137,6 +188,7 @@ export class SegmentDiskCache {
   }
 
   remove(objectKey: string): void {
+    if (this.pinnedKeys.has(objectKey)) return;
     const entry = this.entries.get(objectKey);
     if (!entry) return;
     try {
@@ -151,6 +203,27 @@ export class SegmentDiskCache {
   private evictIfNeeded(incomingBytes: number): void {
     while (this.totalBytes + incomingBytes > this.maxBytes && this.entries.size > 0) {
       const oldestKey = this.entries.keys().next().value as string;
+      if (this.pinnedKeys.has(oldestKey)) {
+        let removed = false;
+        for (const candidateKey of this.entries.keys()) {
+          if (this.pinnedKeys.has(candidateKey)) continue;
+          const candidate = this.entries.get(candidateKey);
+          if (!candidate) continue;
+          try {
+            unlinkSync(candidate.path);
+          } catch {
+            // ignore
+          }
+          this.totalBytes = Math.max(0, this.totalBytes - candidate.size);
+          this.entries.delete(candidateKey);
+          this.mappedFiles.delete(candidateKey);
+          this.evictions += 1;
+          removed = true;
+          break;
+        }
+        if (!removed) break;
+        continue;
+      }
       const entry = this.entries.get(oldestKey);
       if (entry) {
         try {
@@ -159,6 +232,7 @@ export class SegmentDiskCache {
           // ignore
         }
         this.totalBytes = Math.max(0, this.totalBytes - entry.size);
+        this.mappedFiles.delete(oldestKey);
         this.evictions += 1;
       }
       this.entries.delete(oldestKey);
@@ -166,6 +240,12 @@ export class SegmentDiskCache {
   }
 
   stats(): SegmentCacheStats {
+    let mappedBytes = 0;
+    let mappedEntryCount = 0;
+    for (const mapped of this.mappedFiles.values()) {
+      mappedBytes += mapped.sizeBytes;
+      mappedEntryCount += 1;
+    }
     return {
       hits: this.hits,
       misses: this.misses,
@@ -174,6 +254,17 @@ export class SegmentDiskCache {
       usedBytes: this.totalBytes,
       maxBytes: this.maxBytes,
       entryCount: this.entries.size,
+      mappedBytes,
+      mappedEntryCount,
+      pinnedEntryCount: this.pinnedKeys.size,
     };
   }
+
+  bytesForObjectKeyPrefix(prefix: string): number {
+    let total = 0;
+    for (const [objectKey, entry] of this.entries.entries()) {
+      if (objectKey.startsWith(prefix)) total += entry.size;
+    }
+    return total;
+  }
 }

package/src/segment/cached_segment.ts

@@ -0,0 +1,89 @@
+import { existsSync, readFileSync } from "node:fs";
+import type { SegmentRow } from "../db/db";
+import type { ObjectStore } from "../objectstore/interface";
+import type { SegmentDiskCache } from "./cache";
+import { dsError } from "../util/ds_error";
+import type { RetryOptions } from "../util/retry";
+import { retry } from "../util/retry";
+import { segmentObjectKey, streamHash16Hex } from "../util/stream_paths";
+
+export type SegmentReadSource =
+  | { kind: "mapped"; path: string; bytes: Uint8Array }
+  | { kind: "bytes"; bytes: Uint8Array };
+
+function readRangeFromBytes(bytes: Uint8Array, start: number, end: number): Uint8Array {
+  const boundedStart = Math.max(0, Math.min(start, bytes.byteLength));
+  const boundedEnd = Math.max(boundedStart, Math.min(end + 1, bytes.byteLength));
+  return bytes.subarray(boundedStart, boundedEnd);
+}
+
+export function readRangeFromSource(source: SegmentReadSource, start: number, end: number): Uint8Array {
+  return readRangeFromBytes(source.bytes, start, end);
+}
+
+export async function loadSegmentSource(
+  os: ObjectStore,
+  seg: SegmentRow,
+  diskCache?: SegmentDiskCache,
+  retryOpts?: RetryOptions
+): Promise<SegmentReadSource> {
+  if (seg.local_path && seg.local_path.length > 0 && existsSync(seg.local_path)) {
+    try {
+      const bytes = (Bun as any).mmap(seg.local_path, { shared: true }) as Uint8Array;
+      return { kind: "mapped", path: seg.local_path, bytes };
+    } catch {
+      return { kind: "bytes", bytes: readFileSync(seg.local_path) };
+    }
+  }
+
+  const objectKey = segmentObjectKey(streamHash16Hex(seg.stream), seg.segment_index);
+  if (diskCache && diskCache.has(objectKey)) {
+    diskCache.recordHit();
+    diskCache.touch(objectKey);
+    const mapped = diskCache.getMapped(objectKey);
+    if (mapped) return { kind: "mapped", path: mapped.path, bytes: mapped.bytes };
+    const cachedPath = diskCache.getPath(objectKey);
+    if (existsSync(cachedPath)) return { kind: "bytes", bytes: readFileSync(cachedPath) };
+    diskCache.remove(objectKey);
+  }
+
+  if (diskCache) diskCache.recordMiss();
+
+  const bytes = await retry(
+    async () => {
+      const res = await os.get(objectKey);
+      if (!res) throw dsError(`object store missing segment: ${objectKey}`);
+      return res;
+    },
+    retryOpts ?? { retries: 0, baseDelayMs: 0, maxDelayMs: 0, timeoutMs: 0 }
+  );
+
+  if (diskCache?.put(objectKey, bytes)) {
+    const mapped = diskCache.getMapped(objectKey);
+    if (mapped) return { kind: "mapped", path: mapped.path, bytes: mapped.bytes };
+    return { kind: "bytes", bytes: readFileSync(diskCache.getPath(objectKey)) };
+  }
+  return { kind: "bytes", bytes };
+}
+
+export async function loadSegmentBytesCached(
+  os: ObjectStore,
+  seg: SegmentRow,
+  diskCache?: SegmentDiskCache,
+  retryOpts?: RetryOptions
+): Promise<Uint8Array> {
+  const source = await loadSegmentSource(os, seg, diskCache, retryOpts);
+  return source.bytes;
+}
+
+export async function readSegmentRangeCached(
+  os: ObjectStore,
+  seg: SegmentRow,
+  start: number,
+  end: number,
+  diskCache?: SegmentDiskCache,
+  retryOpts?: RetryOptions
+): Promise<Uint8Array> {
+  const source = await loadSegmentSource(os, seg, diskCache, retryOpts);
+  return readRangeFromSource(source, start, end);
+}

package/src/segment/format.ts

@@ -59,6 +59,14 @@ export type IterateBlockEntry = {
   decoded: DecodedBlock;
 };
 
+export type IterateBlockRecordEntry = {
+  blockOffset: number;
+  recordIndex: number;
+  appendNs: bigint;
+  routingKey: Uint8Array;
+  payload: Uint8Array;
+};
+
 function invalidSegment<T = never>(message: string): Result<T, SegmentFormatError> {
   return Result.err({ kind: "invalid_segment_format", message });
 }
@@ -125,59 +133,39 @@ export function decodeBlock(blockBytes: Uint8Array): DecodedBlock {
 }
 
 export function decodeBlockResult(blockBytes: Uint8Array): Result<DecodedBlock, SegmentFormatError> {
-  if (blockBytes.byteLength < DSB3_HEADER_BYTES) return invalidSegment("block too small");
-  if (
-    blockBytes[0] !== "D".charCodeAt(0) ||
-    blockBytes[1] !== "S".charCodeAt(0) ||
-    blockBytes[2] !== "B".charCodeAt(0) ||
-    blockBytes[3] !== "3".charCodeAt(0)
-  ) {
-    return invalidSegment("bad block magic");
-  }
-
-  const uncompressedLen = readU32BE(blockBytes, 4);
-  const compressedLen = readU32BE(blockBytes, 8);
-  const recordCount = readU32BE(blockBytes, 12);
-  const bloom = blockBytes.slice(16, 48);
-  const firstAppendNs = readU64BE(blockBytes, 48);
-  const lastAppendNs = readU64BE(blockBytes, 56);
-  const expectedCrc = readU32BE(blockBytes, 64);
-
-  const payload = blockBytes.slice(DSB3_HEADER_BYTES, DSB3_HEADER_BYTES + compressedLen);
-  if (payload.byteLength !== compressedLen) return invalidSegment("truncated block");
-  const actualCrc = crc32c(payload);
-  if (actualCrc !== expectedCrc) return invalidSegment("crc mismatch");
-
-  let uncompressed: Uint8Array;
-  try {
-    uncompressed = new Uint8Array(zstdDecompressSync(payload));
-  } catch (e: any) {
-    return invalidSegment(String(e?.message ?? e));
-  }
-  if (uncompressed.byteLength !== uncompressedLen) {
-    return invalidSegment(`bad uncompressed len: got=${uncompressed.byteLength} expected=${uncompressedLen}`);
-  }
+  const headerRes = parseBlockHeaderResult(blockBytes);
+  if (Result.isError(headerRes)) return headerRes;
+  const header = headerRes.value;
+  const uncompressedRes = decompressBlockPayloadResult(blockBytes, header);
+  if (Result.isError(uncompressedRes)) return uncompressedRes;
+  const uncompressed = uncompressedRes.value;
 
   const records: SegmentRecord[] = [];
   let off = 0;
-  for (let i = 0; i < recordCount; i++) {
+  for (let i = 0; i < header.recordCount; i++) {
     if (off + 8 + 4 > uncompressed.byteLength) return invalidSegment("truncated record");
     const appendNs = readU64BE(uncompressed, off);
     off += 8;
     const keyLen = readU32BE(uncompressed, off);
     off += 4;
     if (off + keyLen + 4 > uncompressed.byteLength) return invalidSegment("truncated key");
-    const routingKey = uncompressed.slice(off, off + keyLen);
+    const routingKey = uncompressed.subarray(off, off + keyLen);
     off += keyLen;
     const dataLen = readU32BE(uncompressed, off);
     off += 4;
     if (off + dataLen > uncompressed.byteLength) return invalidSegment("truncated payload");
-    const payload = uncompressed.slice(off, off + dataLen);
+    const payload = uncompressed.subarray(off, off + dataLen);
     off += dataLen;
     records.push({ appendNs, routingKey, payload });
   }
 
-  return Result.ok({ recordCount, firstAppendNs, lastAppendNs, bloom, records });
+  return Result.ok({
+    recordCount: header.recordCount,
+    firstAppendNs: header.firstAppendNs,
+    lastAppendNs: header.lastAppendNs,
+    bloom: header.bloom.slice(),
+    records,
+  });
 }
 
 export function encodeFooter(entries: BlockIndexEntry[]): Uint8Array {
@@ -261,6 +249,65 @@ export function* iterateBlocksResult(
   }
 }
 
+export function* iterateBlockRecordsResult(
+  segmentBytes: Uint8Array
+): Generator<Result<IterateBlockRecordEntry, SegmentFormatError>, void, void> {
+  const parsed = parseFooter(segmentBytes);
+  const limit = parsed ? parsed.footerStart : segmentBytes.byteLength;
+  let off = 0;
+  while (off < limit) {
+    if (off + DSB3_HEADER_BYTES > limit) {
+      yield invalidSegment("truncated segment (block header)");
+      return;
+    }
+    const headerRes = parseBlockHeaderResult(segmentBytes.subarray(off, off + DSB3_HEADER_BYTES));
+    if (Result.isError(headerRes)) {
+      yield headerRes;
+      return;
+    }
+    const header = headerRes.value;
+    const totalLen = DSB3_HEADER_BYTES + header.compressedLen;
+    if (off + totalLen > limit) {
+      yield invalidSegment("truncated segment (block payload)");
+      return;
+    }
+    const blockBytes = segmentBytes.subarray(off, off + totalLen);
+    const uncompressedRes = decompressBlockPayloadResult(blockBytes, header);
+    if (Result.isError(uncompressedRes)) {
+      yield uncompressedRes;
+      return;
+    }
+    const uncompressed = uncompressedRes.value;
+    let recOff = 0;
+    for (let recordIndex = 0; recordIndex < header.recordCount; recordIndex++) {
+      if (recOff + 8 + 4 > uncompressed.byteLength) {
+        yield invalidSegment("truncated record");
+        return;
+      }
+      const appendNs = readU64BE(uncompressed, recOff);
+      recOff += 8;
+      const keyLen = readU32BE(uncompressed, recOff);
+      recOff += 4;
+      if (recOff + keyLen + 4 > uncompressed.byteLength) {
+        yield invalidSegment("truncated key");
+        return;
+      }
+      const routingKey = uncompressed.subarray(recOff, recOff + keyLen);
+      recOff += keyLen;
+      const dataLen = readU32BE(uncompressed, recOff);
+      recOff += 4;
+      if (recOff + dataLen > uncompressed.byteLength) {
+        yield invalidSegment("truncated payload");
+        return;
+      }
+      const payload = uncompressed.subarray(recOff, recOff + dataLen);
+      recOff += dataLen;
+      yield Result.ok({ blockOffset: off, recordIndex, appendNs, routingKey, payload });
+    }
+    off += totalLen;
+  }
+}
+
 export function* iterateBlocks(segmentBytes: Uint8Array): Generator<IterateBlockEntry, void, void> {
   for (const itemRes of iterateBlocksResult(segmentBytes)) {
     if (Result.isError(itemRes)) throw dsError(itemRes.error.message);
@@ -268,6 +315,13 @@ export function* iterateBlocks(segmentBytes: Uint8Array): Generator<IterateBlock
   }
 }
 
+export function* iterateBlockRecords(segmentBytes: Uint8Array): Generator<IterateBlockRecordEntry, void, void> {
+  for (const itemRes of iterateBlockRecordsResult(segmentBytes)) {
+    if (Result.isError(itemRes)) throw dsError(itemRes.error.message);
+    yield itemRes.value;
+  }
+}
+
 export function parseFooterBytes(footer: Uint8Array): SegmentFooter | null {
   if (footer.byteLength < 12) return null;
   const magic = String.fromCharCode(footer[0], footer[1], footer[2], footer[3]);
@@ -329,3 +383,21 @@ export function parseBlockHeaderResult(header: Uint8Array): Result<BlockHeader,
     crc32c: crc32cVal,
   });
 }
+
+function decompressBlockPayloadResult(blockBytes: Uint8Array, header: BlockHeader): Result<Uint8Array, SegmentFormatError> {
+  const payload = blockBytes.subarray(DSB3_HEADER_BYTES, DSB3_HEADER_BYTES + header.compressedLen);
+  if (payload.byteLength !== header.compressedLen) return invalidSegment("truncated block");
+  const actualCrc = crc32c(payload);
+  if (actualCrc !== header.crc32c) return invalidSegment("crc mismatch");
+
+  let uncompressed: Uint8Array;
+  try {
+    uncompressed = new Uint8Array(zstdDecompressSync(payload));
+  } catch (e: any) {
+    return invalidSegment(String(e?.message ?? e));
+  }
+  if (uncompressed.byteLength !== header.uncompressedLen) {
+    return invalidSegment(`bad uncompressed len: got=${uncompressed.byteLength} expected=${header.uncompressedLen}`);
+  }
+  return Result.ok(uncompressed);
+}