@tungthedev/streams-server 0.2.0

package/src/postgres/details.ts

@@ -0,0 +1,189 @@
+import type { Pool } from "pg";
+import type {
+  ExactIndexDetailSnapshot,
+  FullModeDetailsSnapshot,
+  FullModeDetailsSnapshotRequest,
+  FullModeDetailsStore,
+  FullModeLagSnapshotRequest,
+  IndexStorageSummary,
+  LexiconIndexStorageSummary,
+  SecondaryIndexStorageSummary,
+} from "../store/full_mode_details_store";
+import type { SchemaRegistryRow } from "../store/schema_profile_store";
+import type { ManifestRow } from "../store/segment_manifest_store";
+import { readU64LE } from "../util/endian";
+import { getPostgresSearchCompanionPlan, listPostgresSearchSegmentCompanions } from "./companions";
+import { loadPostgresLexiconIndexManifest } from "./lexicon_index";
+import { loadPostgresRoutingIndexManifest } from "./routing_index";
+import { loadPostgresSecondaryIndexManifest } from "./secondary_index";
+import type { PgExecutor } from "./types";
+import { toBigInt, toBytes } from "./rows";
+
+export class PostgresFullModeDetailsStore implements FullModeDetailsStore {
+  constructor(private readonly pool: Pool) {}
+
+  async getFullModeDetailsSnapshot(request: FullModeDetailsSnapshotRequest): Promise<FullModeDetailsSnapshot> {
+    const stream = request.stream;
+    const [segmentCount, uploadedSegmentCount, manifest, schemaRow, routingIndex, secondaryIndex, lexiconIndex, companionPlan, companionRows] =
+      await Promise.all([
+        countSegments(this.pool, stream),
+        countUploadedSegments(this.pool, stream),
+        getManifestRow(this.pool, stream),
+        getSchemaRegistry(this.pool, stream),
+        loadPostgresRoutingIndexManifest(this.pool, stream),
+        loadPostgresSecondaryIndexManifest(this.pool, stream),
+        loadPostgresLexiconIndexManifest(this.pool, stream),
+        getPostgresSearchCompanionPlan(this.pool, stream),
+        listPostgresSearchSegmentCompanions(this.pool, stream),
+      ]);
+    const exactIndexes: ExactIndexDetailSnapshot[] = request.exactIndexNames.map((indexName) => ({
+      indexName,
+      state: secondaryIndex.secondaryIndexStates.find((state) => state.index_name === indexName) ?? null,
+      activeRuns: secondaryIndex.secondaryIndexRuns.filter((run) => run.index_name === indexName),
+      retiredRuns: secondaryIndex.retiredSecondaryIndexRuns.filter((run) => run.index_name === indexName),
+    }));
+    return {
+      segmentCount,
+      uploadedSegmentCount,
+      manifest,
+      schemaRow,
+      uploadedSegmentBytes: await sumBigInt(this.pool, `SELECT COALESCE(SUM(size_bytes), 0) AS total FROM segments WHERE stream = $1 AND uploaded_at_ms IS NOT NULL;`, [stream]),
+      pendingSealedSegmentBytes: await sumBigInt(this.pool, `SELECT COALESCE(SUM(size_bytes), 0) AS total FROM segments WHERE stream = $1 AND uploaded_at_ms IS NULL;`, [stream]),
+      routingIndexStorage: await storageSummary(this.pool, "index_runs", stream),
+      secondaryIndexStorage: await secondaryStorageSummary(this.pool, stream),
+      lexiconIndexStorage: await lexiconStorageSummary(this.pool, stream),
+      bundledCompanionStorage: await storageSummary(this.pool, "search_segment_companions", stream),
+      routingState: routingIndex.indexState,
+      routingRuns: routingIndex.indexRuns,
+      retiredRoutingRuns: routingIndex.retiredRuns,
+      exactIndexes,
+      routingLexiconState:
+        lexiconIndex.lexiconIndexStates.find((state) => state.source_kind === "routing_key" && state.source_name === "") ?? null,
+      routingLexiconRuns: lexiconIndex.lexiconIndexRuns.filter((run) => run.source_kind === "routing_key" && run.source_name === ""),
+      retiredRoutingLexiconRuns: lexiconIndex.retiredLexiconIndexRuns.filter(
+        (run) => run.source_kind === "routing_key" && run.source_name === ""
+      ),
+      companionPlan,
+      companionRows,
+    };
+  }
+
+  async getFullModeLagSnapshot(request: FullModeLagSnapshotRequest): Promise<Map<number, bigint>> {
+    const meta = await getSegmentMeta(this.pool, request.stream);
+    const out = new Map<number, bigint>();
+    if (!meta) return out;
+    const indexes = Array.from(new Set(request.segmentIndexes.filter((index) => Number.isInteger(index) && index >= 0))).sort((a, b) => a - b);
+    for (const index of indexes) {
+      if (index >= meta.segmentCount) continue;
+      const offset = index * 8;
+      if (offset + 8 > meta.segmentLastTs.byteLength) continue;
+      out.set(index, readU64LE(meta.segmentLastTs, offset) / 1_000_000n);
+    }
+    return out;
+  }
+}
+
+async function countSegments(executor: PgExecutor, stream: string): Promise<number> {
+  const res = await executor.query<{ count: string }>(`SELECT COUNT(*) AS count FROM segments WHERE stream = $1;`, [stream]);
+  return Number(res.rows[0]?.count ?? 0);
+}
+
+async function countUploadedSegments(executor: PgExecutor, stream: string): Promise<number> {
+  const res = await executor.query<{ max_idx: number | string | null }>(
+    `SELECT MAX(segment_index) AS max_idx FROM segments WHERE stream = $1 AND r2_etag IS NOT NULL;`,
+    [stream]
+  );
+  const maxIdx = res.rows[0]?.max_idx == null ? -1 : Number(res.rows[0]!.max_idx);
+  return maxIdx >= 0 ? maxIdx + 1 : 0;
+}
+
+async function getManifestRow(executor: PgExecutor, stream: string): Promise<ManifestRow> {
+  const res = await executor.query(
+    `SELECT stream, generation, uploaded_generation, last_uploaded_at_ms, last_uploaded_etag, last_uploaded_size_bytes
+     FROM manifests WHERE stream = $1 LIMIT 1;`,
+    [stream]
+  );
+  if (!res.rows[0]) {
+    return { stream, generation: 0, uploaded_generation: 0, last_uploaded_at_ms: null, last_uploaded_etag: null, last_uploaded_size_bytes: null };
+  }
+  const row = res.rows[0];
+  return {
+    stream: String(row.stream),
+    generation: Number(row.generation),
+    uploaded_generation: Number(row.uploaded_generation),
+    last_uploaded_at_ms: row.last_uploaded_at_ms == null ? null : toBigInt(row.last_uploaded_at_ms),
+    last_uploaded_etag: row.last_uploaded_etag == null ? null : String(row.last_uploaded_etag),
+    last_uploaded_size_bytes: row.last_uploaded_size_bytes == null ? null : toBigInt(row.last_uploaded_size_bytes),
+  };
+}
+
+async function getSchemaRegistry(executor: PgExecutor, stream: string): Promise<SchemaRegistryRow | null> {
+  const res = await executor.query(`SELECT stream, schema_json, updated_at_ms, uploaded_size_bytes FROM schemas WHERE stream = $1 LIMIT 1;`, [stream]);
+  const row = res.rows[0];
+  if (!row) return null;
+  return {
+    stream: String(row.stream),
+    registry_json: String(row.schema_json),
+    updated_at_ms: toBigInt(row.updated_at_ms),
+    uploaded_size_bytes: toBigInt(row.uploaded_size_bytes ?? 0),
+  };
+}
+
+async function sumBigInt(executor: PgExecutor, sql: string, params: unknown[]): Promise<bigint> {
+  const res = await executor.query<{ total: string | number | bigint | null }>(sql, params);
+  return toBigInt(res.rows[0]?.total ?? 0);
+}
+
+async function storageSummary(executor: PgExecutor, tableName: "index_runs" | "search_segment_companions", stream: string): Promise<IndexStorageSummary> {
+  const res = await executor.query<{ cnt: string; total: string | number | bigint | null }>(
+    `SELECT COUNT(*) AS cnt, COALESCE(SUM(size_bytes), 0) AS total FROM ${tableName} WHERE stream = $1;`,
+    [stream]
+  );
+  return { object_count: Number(res.rows[0]?.cnt ?? 0), bytes: toBigInt(res.rows[0]?.total ?? 0) };
+}
+
+async function secondaryStorageSummary(executor: PgExecutor, stream: string): Promise<SecondaryIndexStorageSummary[]> {
+  const res = await executor.query(
+    `SELECT index_name, COUNT(*) AS cnt, COALESCE(SUM(size_bytes), 0) AS total
+     FROM secondary_index_runs
+     WHERE stream = $1
+     GROUP BY index_name
+     ORDER BY index_name ASC;`,
+    [stream]
+  );
+  return res.rows.map((row) => ({
+    index_name: String(row.index_name),
+    object_count: Number(row.cnt ?? 0),
+    bytes: toBigInt(row.total ?? 0),
+  }));
+}
+
+async function lexiconStorageSummary(executor: PgExecutor, stream: string): Promise<LexiconIndexStorageSummary[]> {
+  const res = await executor.query(
+    `SELECT source_kind, source_name, COUNT(*) AS cnt, COALESCE(SUM(size_bytes), 0) AS total
+     FROM lexicon_index_runs
+     WHERE stream = $1
+     GROUP BY source_kind, source_name
+     ORDER BY source_kind ASC, source_name ASC;`,
+    [stream]
+  );
+  return res.rows.map((row) => ({
+    source_kind: String(row.source_kind),
+    source_name: String(row.source_name),
+    object_count: Number(row.cnt ?? 0),
+    bytes: toBigInt(row.total ?? 0),
+  }));
+}
+
+async function getSegmentMeta(executor: PgExecutor, stream: string): Promise<{ segmentCount: number; segmentLastTs: Uint8Array } | null> {
+  const res = await executor.query(
+    `SELECT segment_count, segment_last_ts
+     FROM stream_segment_meta
+     WHERE stream = $1
+     LIMIT 1;`,
+    [stream]
+  );
+  const row = res.rows[0];
+  if (!row) return null;
+  return { segmentCount: Number(row.segment_count), segmentLastTs: toBytes(row.segment_last_ts) };
+}

package/src/postgres/lexicon_index.ts

@@ -0,0 +1,260 @@
+import type { LexiconIndexStore } from "../store/index_store";
+import type { LexiconIndexRunRow, LexiconIndexStateRow } from "../store/rows";
+import type { WalReadRow } from "../store/wal_store";
+import type { PgExecutor } from "./types";
+import { pgInt, toBigInt, PostgresIndexSharedStore } from "./rows";
+
+export class PostgresLexiconIndexStore extends PostgresIndexSharedStore implements LexiconIndexStore {
+  constructor(
+    pool: ConstructorParameters<typeof PostgresIndexSharedStore>[0],
+    nowMs: () => bigint,
+    private readonly readWal: (stream: string, startOffset: bigint, endOffset: bigint, routingKey?: Uint8Array) => AsyncIterable<WalReadRow>
+  ) {
+    super(pool, nowMs);
+  }
+
+  readWalRange(stream: string, startOffset: bigint, endOffset: bigint, routingKey?: Uint8Array): AsyncIterable<WalReadRow> {
+    return this.readWal(stream, startOffset, endOffset, routingKey);
+  }
+
+  async getLexiconIndexState(stream: string, sourceKind: string, sourceName: string): Promise<LexiconIndexStateRow | null> {
+    return getPostgresLexiconIndexState(this.pool, stream, sourceKind, sourceName);
+  }
+
+  async upsertLexiconIndexState(stream: string, sourceKind: string, sourceName: string, indexedThrough: number): Promise<void> {
+    await upsertPostgresLexiconIndexState(this.pool, this.nowMs(), stream, sourceKind, sourceName, indexedThrough);
+  }
+
+  async updateLexiconIndexedThrough(stream: string, sourceKind: string, sourceName: string, indexedThrough: number): Promise<void> {
+    await this.pool.query(
+      `UPDATE lexicon_index_state
+       SET indexed_through = $1, updated_at_ms = $2
+       WHERE stream = $3 AND source_kind = $4 AND source_name = $5;`,
+      [indexedThrough, pgInt(this.nowMs()), stream, sourceKind, sourceName]
+    );
+  }
+
+  async listLexiconIndexRuns(stream: string, sourceKind: string, sourceName: string): Promise<LexiconIndexRunRow[]> {
+    return listPostgresLexiconIndexRuns(this.pool, stream, sourceKind, sourceName, false);
+  }
+
+  async listLexiconIndexRunsAll(stream: string, sourceKind: string, sourceName: string): Promise<LexiconIndexRunRow[]> {
+    return listPostgresLexiconIndexRuns(this.pool, stream, sourceKind, sourceName, true);
+  }
+
+  async listRetiredLexiconIndexRuns(stream: string, sourceKind: string, sourceName: string): Promise<LexiconIndexRunRow[]> {
+    return listPostgresRetiredLexiconIndexRuns(this.pool, stream, sourceKind, sourceName);
+  }
+
+  async insertLexiconIndexRun(row: Omit<LexiconIndexRunRow, "retired_gen" | "retired_at_ms">): Promise<void> {
+    await insertPostgresLexiconIndexRun(this.pool, row);
+  }
+
+  async retireLexiconIndexRuns(runIds: string[], retiredGen: number, retiredAtMs: bigint): Promise<void> {
+    await retirePostgresLexiconIndexRuns(this.pool, runIds, retiredGen, retiredAtMs);
+  }
+
+  async deleteLexiconIndexRuns(runIds: string[]): Promise<void> {
+    if (runIds.length === 0) return;
+    await this.pool.query(`DELETE FROM lexicon_index_runs WHERE run_id = ANY($1::text[]);`, [runIds]);
+  }
+
+  async deleteLexiconIndexSource(stream: string, sourceKind: string, sourceName: string): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      await client.query(`DELETE FROM lexicon_index_runs WHERE stream = $1 AND source_kind = $2 AND source_name = $3;`, [
+        stream,
+        sourceKind,
+        sourceName,
+      ]);
+      await client.query(`DELETE FROM lexicon_index_state WHERE stream = $1 AND source_kind = $2 AND source_name = $3;`, [
+        stream,
+        sourceKind,
+        sourceName,
+      ]);
+      await client.query("COMMIT");
+    } catch (error) {
+      await client.query("ROLLBACK").catch(() => {});
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+}
+export async function insertPostgresLexiconIndexRun(
+  executor: PgExecutor,
+  row: Omit<LexiconIndexRunRow, "retired_gen" | "retired_at_ms">,
+  opts: { idempotent?: boolean } = {}
+): Promise<void> {
+  const conflictSql = opts.idempotent
+    ? `ON CONFLICT(run_id) DO UPDATE SET
+         stream = excluded.stream,
+         source_kind = excluded.source_kind,
+         source_name = excluded.source_name,
+         level = excluded.level,
+         start_segment = excluded.start_segment,
+         end_segment = excluded.end_segment,
+         object_key = excluded.object_key,
+         size_bytes = excluded.size_bytes,
+         record_count = excluded.record_count,
+         retired_gen = NULL,
+         retired_at_ms = NULL`
+    : "";
+  await executor.query(
+    `INSERT INTO lexicon_index_runs(
+       run_id, stream, source_kind, source_name, level, start_segment, end_segment, object_key, size_bytes, record_count
+     )
+     VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+     ${conflictSql};`,
+    [
+      row.run_id,
+      row.stream,
+      row.source_kind,
+      row.source_name,
+      row.level,
+      row.start_segment,
+      row.end_segment,
+      row.object_key,
+      row.size_bytes,
+      row.record_count,
+    ]
+  );
+}
+export async function upsertPostgresLexiconIndexState(
+  executor: PgExecutor,
+  nowMs: bigint,
+  stream: string,
+  sourceKind: string,
+  sourceName: string,
+  indexedThrough: number
+): Promise<void> {
+  await executor.query(
+    `INSERT INTO lexicon_index_state(stream, source_kind, source_name, indexed_through, updated_at_ms)
+     VALUES($1, $2, $3, $4, $5)
+     ON CONFLICT(stream, source_kind, source_name) DO UPDATE SET
+       indexed_through = excluded.indexed_through,
+       updated_at_ms = excluded.updated_at_ms;`,
+    [stream, sourceKind, sourceName, indexedThrough, pgInt(nowMs)]
+  );
+}
+
+export async function retirePostgresLexiconIndexRuns(
+  executor: PgExecutor,
+  runIds: string[],
+  retiredGen: number,
+  retiredAtMs: bigint
+): Promise<void> {
+  if (runIds.length === 0) return;
+  await executor.query(`UPDATE lexicon_index_runs SET retired_gen = $1, retired_at_ms = $2 WHERE run_id = ANY($3::text[]);`, [
+    retiredGen,
+    pgInt(retiredAtMs),
+    runIds,
+  ]);
+}
+
+export async function loadPostgresLexiconIndexManifest(
+  executor: PgExecutor,
+  stream: string
+): Promise<{
+  lexiconIndexStates: LexiconIndexStateRow[];
+  lexiconIndexRuns: LexiconIndexRunRow[];
+  retiredLexiconIndexRuns: LexiconIndexRunRow[];
+}> {
+  const lexiconIndexStates = await listPostgresLexiconIndexStates(executor, stream);
+  const lexiconIndexRuns = (
+    await Promise.all(
+      lexiconIndexStates.map((state) => listPostgresLexiconIndexRuns(executor, stream, state.source_kind, state.source_name, false))
+    )
+  ).flat();
+  const retiredLexiconIndexRuns = (
+    await Promise.all(
+      lexiconIndexStates.map((state) => listPostgresRetiredLexiconIndexRuns(executor, stream, state.source_kind, state.source_name))
+    )
+  ).flat();
+  return { lexiconIndexStates, lexiconIndexRuns, retiredLexiconIndexRuns };
+}
+
+async function getPostgresLexiconIndexState(
+  executor: PgExecutor,
+  stream: string,
+  sourceKind: string,
+  sourceName: string
+): Promise<LexiconIndexStateRow | null> {
+  const res = await executor.query(
+    `SELECT * FROM lexicon_index_state
+     WHERE stream = $1 AND source_kind = $2 AND source_name = $3
+     LIMIT 1;`,
+    [stream, sourceKind, sourceName]
+  );
+  return res.rows[0] ? coerceLexiconIndexStateRow(res.rows[0]) : null;
+}
+
+async function listPostgresLexiconIndexStates(executor: PgExecutor, stream: string): Promise<LexiconIndexStateRow[]> {
+  const res = await executor.query(
+    `SELECT * FROM lexicon_index_state
+     WHERE stream = $1
+     ORDER BY source_kind ASC, source_name ASC;`,
+    [stream]
+  );
+  return res.rows.map(coerceLexiconIndexStateRow);
+}
+
+async function listPostgresLexiconIndexRuns(
+  executor: PgExecutor,
+  stream: string,
+  sourceKind: string,
+  sourceName: string,
+  includeRetired: boolean
+): Promise<LexiconIndexRunRow[]> {
+  const retiredClause = includeRetired ? "" : " AND retired_gen IS NULL";
+  const res = await executor.query(
+    `SELECT * FROM lexicon_index_runs
+     WHERE stream = $1 AND source_kind = $2 AND source_name = $3${retiredClause}
+     ORDER BY level ASC, start_segment ASC, end_segment ASC;`,
+    [stream, sourceKind, sourceName]
+  );
+  return res.rows.map(coerceLexiconIndexRunRow);
+}
+
+async function listPostgresRetiredLexiconIndexRuns(
+  executor: PgExecutor,
+  stream: string,
+  sourceKind: string,
+  sourceName: string
+): Promise<LexiconIndexRunRow[]> {
+  const res = await executor.query(
+    `SELECT * FROM lexicon_index_runs
+     WHERE stream = $1 AND source_kind = $2 AND source_name = $3 AND retired_gen IS NOT NULL
+     ORDER BY retired_gen ASC, retired_at_ms ASC, level ASC, start_segment ASC;`,
+    [stream, sourceKind, sourceName]
+  );
+  return res.rows.map(coerceLexiconIndexRunRow);
+}
+
+function coerceLexiconIndexStateRow(row: any): LexiconIndexStateRow {
+  return {
+    stream: String(row.stream),
+    source_kind: String(row.source_kind),
+    source_name: String(row.source_name),
+    indexed_through: Number(row.indexed_through),
+    updated_at_ms: toBigInt(row.updated_at_ms),
+  };
+}
+
+function coerceLexiconIndexRunRow(row: any): LexiconIndexRunRow {
+  return {
+    run_id: String(row.run_id),
+    stream: String(row.stream),
+    source_kind: String(row.source_kind),
+    source_name: String(row.source_name),
+    level: Number(row.level),
+    start_segment: Number(row.start_segment),
+    end_segment: Number(row.end_segment),
+    object_key: String(row.object_key),
+    size_bytes: Number(row.size_bytes),
+    record_count: Number(row.record_count),
+    retired_gen: row.retired_gen == null ? null : Number(row.retired_gen),
+    retired_at_ms: row.retired_at_ms == null ? null : toBigInt(row.retired_at_ms),
+  };
+}

package/src/postgres/routing_index.ts

@@ -0,0 +1,189 @@
+import type { Pool } from "pg";
+import type { RoutingIndexStore } from "../store/index_store";
+import type { IndexRunRow, IndexStateRow } from "../store/rows";
+import type { PgExecutor } from "./types";
+import { pgInt, PostgresIndexSharedStore, toBigInt, toBytes } from "./rows";
+
+export class PostgresRoutingIndexStore extends PostgresIndexSharedStore implements RoutingIndexStore {
+  async getIndexState(stream: string): Promise<IndexStateRow | null> {
+    return getPostgresIndexState(this.pool, stream);
+  }
+
+  async upsertIndexState(stream: string, indexSecret: Uint8Array, indexedThrough: number): Promise<void> {
+    await upsertPostgresIndexState(this.pool, this.nowMs(), stream, indexSecret, indexedThrough);
+  }
+
+  async updateIndexedThrough(stream: string, indexedThrough: number): Promise<void> {
+    await this.pool.query(`UPDATE index_state SET indexed_through = $1, updated_at_ms = $2 WHERE stream = $3;`, [
+      indexedThrough,
+      pgInt(this.nowMs()),
+      stream,
+    ]);
+  }
+
+  async listIndexRuns(stream: string): Promise<IndexRunRow[]> {
+    return listPostgresIndexRuns(this.pool, stream, false);
+  }
+
+  async listIndexRunsAll(stream: string): Promise<IndexRunRow[]> {
+    return listPostgresIndexRuns(this.pool, stream, true);
+  }
+
+  async listRetiredIndexRuns(stream: string): Promise<IndexRunRow[]> {
+    const res = await this.pool.query(
+      `SELECT * FROM index_runs
+       WHERE stream = $1 AND retired_gen IS NOT NULL
+       ORDER BY retired_gen ASC, retired_at_ms ASC, level ASC, start_segment ASC;`,
+      [stream]
+    );
+    return res.rows.map(coerceIndexRunRow);
+  }
+
+  async insertIndexRun(row: Omit<IndexRunRow, "retired_gen" | "retired_at_ms">): Promise<void> {
+    await insertPostgresIndexRun(this.pool, row);
+  }
+
+  async retireIndexRuns(runIds: string[], retiredGen: number, retiredAtMs: bigint): Promise<void> {
+    await retirePostgresIndexRuns(this.pool, runIds, retiredGen, retiredAtMs);
+  }
+
+  async deleteIndexRuns(runIds: string[]): Promise<void> {
+    if (runIds.length === 0) return;
+    await this.pool.query(`DELETE FROM index_runs WHERE run_id = ANY($1::text[]);`, [runIds]);
+  }
+
+  async deleteIndex(stream: string): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      await client.query(`DELETE FROM index_runs WHERE stream = $1;`, [stream]);
+      await client.query(`DELETE FROM index_state WHERE stream = $1;`, [stream]);
+      await client.query("COMMIT");
+    } catch (error) {
+      await client.query("ROLLBACK").catch(() => {});
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+}
+
+export async function insertPostgresIndexRun(
+  executor: PgExecutor,
+  row: Omit<IndexRunRow, "retired_gen" | "retired_at_ms">,
+  opts: { idempotent?: boolean } = {}
+): Promise<void> {
+  const conflictSql = opts.idempotent
+    ? `ON CONFLICT(run_id) DO UPDATE SET
+         stream = excluded.stream,
+         level = excluded.level,
+         start_segment = excluded.start_segment,
+         end_segment = excluded.end_segment,
+         object_key = excluded.object_key,
+         size_bytes = excluded.size_bytes,
+         filter_len = excluded.filter_len,
+         record_count = excluded.record_count,
+         retired_gen = NULL,
+         retired_at_ms = NULL`
+    : "";
+  await executor.query(
+    `INSERT INTO index_runs(run_id, stream, level, start_segment, end_segment, object_key, size_bytes, filter_len, record_count)
+     VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)
+     ${conflictSql};`,
+    [row.run_id, row.stream, row.level, row.start_segment, row.end_segment, row.object_key, row.size_bytes, row.filter_len, row.record_count]
+  );
+}
+
+export async function upsertPostgresIndexState(
+  executor: PgExecutor,
+  nowMs: bigint,
+  stream: string,
+  indexSecret: Uint8Array,
+  indexedThrough: number
+): Promise<void> {
+  await executor.query(
+    `INSERT INTO index_state(stream, index_secret, indexed_through, updated_at_ms)
+     VALUES($1, $2, $3, $4)
+     ON CONFLICT(stream) DO UPDATE SET
+       index_secret = excluded.index_secret,
+       indexed_through = excluded.indexed_through,
+       updated_at_ms = excluded.updated_at_ms;`,
+    [stream, Buffer.from(indexSecret), indexedThrough, pgInt(nowMs)]
+  );
+}
+
+export async function retirePostgresIndexRuns(
+  executor: PgExecutor,
+  runIds: string[],
+  retiredGen: number,
+  retiredAtMs: bigint
+): Promise<void> {
+  if (runIds.length === 0) return;
+  await executor.query(`UPDATE index_runs SET retired_gen = $1, retired_at_ms = $2 WHERE run_id = ANY($3::text[]);`, [
+    retiredGen,
+    pgInt(retiredAtMs),
+    runIds,
+  ]);
+}
+
+export async function loadPostgresRoutingIndexManifest(
+  executor: PgExecutor,
+  stream: string
+): Promise<{ indexState: IndexStateRow | null; indexRuns: IndexRunRow[]; retiredRuns: IndexRunRow[] }> {
+  return {
+    indexState: await getPostgresIndexState(executor, stream),
+    indexRuns: await listPostgresIndexRuns(executor, stream, false),
+    retiredRuns: await listPostgresRetiredIndexRuns(executor, stream),
+  };
+}
+
+async function getPostgresIndexState(executor: PgExecutor, stream: string): Promise<IndexStateRow | null> {
+  const res = await executor.query(`SELECT * FROM index_state WHERE stream = $1 LIMIT 1;`, [stream]);
+  return res.rows[0] ? coerceIndexStateRow(res.rows[0]) : null;
+}
+
+async function listPostgresIndexRuns(executor: PgExecutor, stream: string, includeRetired: boolean): Promise<IndexRunRow[]> {
+  const retiredClause = includeRetired ? "" : " AND retired_gen IS NULL";
+  const res = await executor.query(
+    `SELECT * FROM index_runs
+     WHERE stream = $1${retiredClause}
+     ORDER BY level ASC, start_segment ASC, end_segment ASC;`,
+    [stream]
+  );
+  return res.rows.map(coerceIndexRunRow);
+}
+
+async function listPostgresRetiredIndexRuns(executor: PgExecutor, stream: string): Promise<IndexRunRow[]> {
+  const res = await executor.query(
+    `SELECT * FROM index_runs
+     WHERE stream = $1 AND retired_gen IS NOT NULL
+     ORDER BY retired_gen ASC, retired_at_ms ASC, level ASC, start_segment ASC;`,
+    [stream]
+  );
+  return res.rows.map(coerceIndexRunRow);
+}
+
+function coerceIndexStateRow(row: any): IndexStateRow {
+  return {
+    stream: String(row.stream),
+    index_secret: toBytes(row.index_secret),
+    indexed_through: Number(row.indexed_through),
+    updated_at_ms: toBigInt(row.updated_at_ms),
+  };
+}
+
+function coerceIndexRunRow(row: any): IndexRunRow {
+  return {
+    run_id: String(row.run_id),
+    stream: String(row.stream),
+    level: Number(row.level),
+    start_segment: Number(row.start_segment),
+    end_segment: Number(row.end_segment),
+    object_key: String(row.object_key),
+    size_bytes: Number(row.size_bytes),
+    filter_len: Number(row.filter_len),
+    record_count: Number(row.record_count),
+    retired_gen: row.retired_gen == null ? null : Number(row.retired_gen),
+    retired_at_ms: row.retired_at_ms == null ? null : toBigInt(row.retired_at_ms),
+  };
+}