@tungthedev/streams-server 0.2.0

package/src/offset.ts

@@ -0,0 +1,70 @@
+import { decodeCrockfordBase32Fixed26Result, encodeCrockfordBase32Fixed26Result } from "./util/base32_crockford";
+import { readU32BE, writeU32BE } from "./util/endian";
+import { Result } from "better-result";
+import { dsError } from "./util/ds_error.ts";
+
+export type ParsedOffset =
+  | { kind: "start" }
+  | { kind: "seq"; epoch: number; seq: bigint; inBlock: number };
+
+const DEFAULT_EPOCH = 0;
+
+export function parseOffsetResult(input: string | null | undefined): Result<ParsedOffset, { kind: "invalid_offset"; message: string }> {
+  if (input == null || input === "") {
+    return Result.err({ kind: "invalid_offset", message: "missing offset" });
+  }
+  if (input === "-1") return Result.ok({ kind: "start" });
+
+  if (input.length !== 26) {
+    return Result.err({ kind: "invalid_offset", message: `invalid offset length: ${input.length}` });
+  }
+
+  const bytesRes = decodeCrockfordBase32Fixed26Result(input);
+  if (Result.isError(bytesRes)) return Result.err({ kind: "invalid_offset", message: bytesRes.error.message });
+  const bytes = bytesRes.value;
+  const epoch = readU32BE(bytes, 0);
+  const hi = readU32BE(bytes, 4);
+  const lo = readU32BE(bytes, 8);
+  const inBlock = readU32BE(bytes, 12);
+  // Protocol offsets are shifted by +1 so we never emit reserved "-1".
+  const rawSeq = (BigInt(hi) << 32n) | BigInt(lo);
+  const seq = rawSeq - 1n;
+  return Result.ok({ kind: "seq", epoch, seq, inBlock });
+}
+
+export function parseOffset(input: string | null | undefined): ParsedOffset {
+  const res = parseOffsetResult(input);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+export function encodeOffsetResult(epoch: number, seq: bigint, inBlock = 0): Result<string, { kind: "invalid_offset"; message: string }> {
+  if (seq < -1n) return Result.err({ kind: "invalid_offset", message: "invalid offset" });
+  const bytes = new Uint8Array(16);
+  writeU32BE(bytes, 0, epoch >>> 0);
+  const rawSeq = seq + 1n;
+  const hi = Number((rawSeq >> 32n) & 0xffffffffn);
+  const lo = Number(rawSeq & 0xffffffffn);
+  writeU32BE(bytes, 4, hi);
+  writeU32BE(bytes, 8, lo);
+  writeU32BE(bytes, 12, inBlock >>> 0);
+  const encodedRes = encodeCrockfordBase32Fixed26Result(bytes);
+  if (Result.isError(encodedRes)) return Result.err({ kind: "invalid_offset", message: encodedRes.error.message });
+  return Result.ok(encodedRes.value);
+}
+
+export function encodeOffset(epoch: number, seq: bigint, inBlock = 0): string {
+  const res = encodeOffsetResult(epoch, seq, inBlock);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+export function canonicalizeOffset(input: string): string {
+  const p = parseOffset(input);
+  if (p.kind === "start") return encodeOffset(DEFAULT_EPOCH, -1n, 0);
+  return encodeOffset(p.epoch, p.seq, p.inBlock);
+}
+
+export function offsetToSeqOrNeg1(p: ParsedOffset): bigint {
+  return p.kind === "start" ? -1n : p.seq;
+}

package/src/postgres/bootstrap.ts

@@ -0,0 +1,269 @@
+import { mkdirSync, rmSync } from "node:fs";
+import { Pool, type PoolClient } from "pg";
+import type { Config } from "../config";
+import { bootstrapObjectStoreIntoRestoreStore } from "../bootstrap";
+import type { ObjectStore } from "../objectstore/interface";
+import type { BootstrapRestoreStore } from "../store/bootstrap_restore_store";
+import type { ProfileTouchControlStore } from "../store/touch_store";
+import type { IndexRunRow, LexiconIndexRunRow, SecondaryIndexRunRow } from "../store/rows";
+import type { StreamReadRow } from "../store/segment_read_store";
+import { migratePostgresStore } from "./schema";
+import type { PgExecutor } from "./types";
+import { dsError } from "../util/ds_error";
+import {
+  deletePostgresStreamProfile,
+  restorePostgresStreamRow,
+  upsertPostgresSchemaRegistry,
+  upsertPostgresStreamProfile,
+} from "./control_restore";
+import {
+  markPostgresSegmentUploaded,
+  restorePostgresManifestRow,
+  restorePostgresSegmentMeta,
+  restorePostgresSegmentRow,
+  setPostgresSchemaUploadedSizeBytes,
+} from "./segments";
+import { insertPostgresIndexRun, retirePostgresIndexRuns, upsertPostgresIndexState } from "./routing_index";
+import {
+  insertPostgresSecondaryIndexRun,
+  retirePostgresSecondaryIndexRuns,
+  upsertPostgresSecondaryIndexState,
+} from "./secondary_index";
+import {
+  insertPostgresLexiconIndexRun,
+  retirePostgresLexiconIndexRuns,
+  upsertPostgresLexiconIndexState,
+} from "./lexicon_index";
+import { upsertPostgresSearchCompanionPlan, upsertPostgresSearchSegmentCompanion } from "./companions";
+import { deletePostgresStreamTouchState, ensurePostgresStreamTouchStateFromStream } from "./touch";
+
+export async function bootstrapPostgresFromR2(
+  cfg: Config,
+  objectStore: ObjectStore,
+  connectionString: string,
+  opts: { clearLocal?: boolean } = {}
+): Promise<void> {
+  if (opts.clearLocal !== false) {
+    rmSync(`${cfg.rootDir}/local`, { recursive: true, force: true });
+    rmSync(`${cfg.rootDir}/cache`, { recursive: true, force: true });
+  }
+  mkdirSync(cfg.rootDir, { recursive: true });
+
+  const pool = new Pool({ connectionString });
+  try {
+    await migratePostgresStore(pool, { fullMode: true });
+    if (opts.clearLocal !== false) await clearPostgresRestoreTarget(pool);
+    else await assertPostgresRestoreTargetEmpty(pool);
+    await bootstrapObjectStoreIntoRestoreStore(cfg, objectStore, new PostgresBootstrapRestoreStore(pool));
+  } finally {
+    await pool.end();
+  }
+}
+
+async function clearPostgresRestoreTarget(executor: PgExecutor): Promise<void> {
+  await executor.query(`TRUNCATE TABLE objectstore_request_counts, streams RESTART IDENTITY CASCADE;`);
+}
+
+async function assertPostgresRestoreTargetEmpty(executor: PgExecutor): Promise<void> {
+  const res = await executor.query<{ count: string | number | bigint }>(
+    `SELECT
+       (SELECT COUNT(*) FROM streams) +
+       (SELECT COUNT(*) FROM objectstore_request_counts) AS count;`
+  );
+  if (BigInt(res.rows[0]?.count ?? 0) !== 0n) {
+    throw dsError("postgres bootstrap target is not empty; rerun with clearLocal enabled");
+  }
+}
+
+class PostgresBootstrapRestoreStore implements BootstrapRestoreStore {
+  private tx: PoolClient | null = null;
+
+  readonly touch: ProfileTouchControlStore = {
+    ensureStreamTouchState: (stream) => this.ensureStreamTouchState(stream),
+    deleteStreamTouchState: (stream) => this.deleteStreamTouchState(stream),
+  };
+
+  constructor(private readonly pool: Pool) {}
+
+  nowMs(): bigint {
+    return BigInt(Date.now());
+  }
+
+  async close(): Promise<void> {
+    await this.rollbackActiveTransaction();
+  }
+
+  async beginRestoreStream(_stream: string): Promise<void> {
+    if (this.tx) throw dsError("postgres bootstrap restore transaction is already active");
+    this.tx = await this.pool.connect();
+    await this.tx.query("BEGIN");
+  }
+
+  async commitRestoreStream(_stream: string): Promise<void> {
+    if (!this.tx) return;
+    const client = this.tx;
+    try {
+      await client.query("COMMIT");
+      this.tx = null;
+    } catch (error) {
+      await client.query("ROLLBACK").catch(() => {});
+      this.tx = null;
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  async rollbackRestoreStream(_stream: string): Promise<void> {
+    await this.rollbackActiveTransaction();
+  }
+
+  async restoreStreamRow(row: StreamReadRow): Promise<void> {
+    await restorePostgresStreamRow(this.executor, row);
+  }
+
+  async upsertStreamProfile(stream: string, profileJson: string): Promise<void> {
+    await upsertPostgresStreamProfile(this.executor, this.nowMs(), stream, profileJson);
+  }
+
+  async deleteStreamProfile(stream: string): Promise<void> {
+    await deletePostgresStreamProfile(this.executor, stream);
+  }
+
+  async upsertSegmentMeta(stream: string, count: number, offsets: Uint8Array, blocks: Uint8Array, lastTs: Uint8Array): Promise<void> {
+    await restorePostgresSegmentMeta(this.executor, stream, count, offsets, blocks, lastTs);
+  }
+
+  async upsertManifestRow(
+    stream: string,
+    generation: number,
+    uploadedGeneration: number,
+    uploadedAtMs: bigint | null,
+    etag: string | null,
+    sizeBytes: number | null
+  ): Promise<void> {
+    await restorePostgresManifestRow(this.executor, stream, generation, uploadedGeneration, uploadedAtMs, etag, sizeBytes);
+  }
+
+  async createSegmentRow(row: {
+    segmentId: string;
+    stream: string;
+    segmentIndex: number;
+    startOffset: bigint;
+    endOffset: bigint;
+    blockCount: number;
+    lastAppendMs: bigint;
+    payloadBytes: bigint;
+    sizeBytes: number;
+    localPath: string;
+  }): Promise<void> {
+    await restorePostgresSegmentRow(this.executor, this.nowMs(), row);
+  }
+
+  async markSegmentUploaded(segmentId: string, etag: string, uploadedAtMs: bigint): Promise<void> {
+    await markPostgresSegmentUploaded(this.executor, segmentId, etag, uploadedAtMs);
+  }
+
+  async upsertIndexState(stream: string, indexSecret: Uint8Array, indexedThrough: number): Promise<void> {
+    await upsertPostgresIndexState(this.executor, this.nowMs(), stream, indexSecret, indexedThrough);
+  }
+
+  async insertIndexRun(row: Omit<IndexRunRow, "retired_gen" | "retired_at_ms">): Promise<void> {
+    await insertPostgresIndexRun(this.executor, row, { idempotent: true });
+  }
+
+  async retireIndexRuns(runIds: string[], retiredGen: number, retiredAtMs: bigint): Promise<void> {
+    await retirePostgresIndexRuns(this.executor, runIds, retiredGen, retiredAtMs);
+  }
+
+  async upsertSecondaryIndexState(
+    stream: string,
+    indexName: string,
+    indexSecret: Uint8Array,
+    configHash: string,
+    indexedThrough: number
+  ): Promise<void> {
+    await upsertPostgresSecondaryIndexState(this.executor, this.nowMs(), stream, indexName, indexSecret, configHash, indexedThrough);
+  }
+
+  async insertSecondaryIndexRun(row: Omit<SecondaryIndexRunRow, "retired_gen" | "retired_at_ms">): Promise<void> {
+    await insertPostgresSecondaryIndexRun(this.executor, row, { idempotent: true });
+  }
+
+  async retireSecondaryIndexRuns(runIds: string[], retiredGen: number, retiredAtMs: bigint): Promise<void> {
+    await retirePostgresSecondaryIndexRuns(this.executor, runIds, retiredGen, retiredAtMs);
+  }
+
+  async upsertLexiconIndexState(stream: string, sourceKind: string, sourceName: string, indexedThrough: number): Promise<void> {
+    await upsertPostgresLexiconIndexState(this.executor, this.nowMs(), stream, sourceKind, sourceName, indexedThrough);
+  }
+
+  async insertLexiconIndexRun(row: Omit<LexiconIndexRunRow, "retired_gen" | "retired_at_ms">): Promise<void> {
+    await insertPostgresLexiconIndexRun(this.executor, row, { idempotent: true });
+  }
+
+  async retireLexiconIndexRuns(runIds: string[], retiredGen: number, retiredAtMs: bigint): Promise<void> {
+    await retirePostgresLexiconIndexRuns(this.executor, runIds, retiredGen, retiredAtMs);
+  }
+
+  async upsertSearchCompanionPlan(stream: string, generation: number, planHash: string, planJson: string): Promise<void> {
+    await upsertPostgresSearchCompanionPlan(this.executor, this.nowMs(), stream, generation, planHash, planJson);
+  }
+
+  async upsertSearchSegmentCompanion(
+    stream: string,
+    segmentIndex: number,
+    objectKey: string,
+    planGeneration: number,
+    sectionsJson: string,
+    sectionSizesJson: string,
+    sizeBytes: number,
+    primaryTimestampMinMs: bigint | null,
+    primaryTimestampMaxMs: bigint | null
+  ): Promise<void> {
+    await upsertPostgresSearchSegmentCompanion(
+      this.executor,
+      this.nowMs(),
+      stream,
+      segmentIndex,
+      objectKey,
+      planGeneration,
+      sectionsJson,
+      sectionSizesJson,
+      sizeBytes,
+      primaryTimestampMinMs,
+      primaryTimestampMaxMs
+    );
+  }
+
+  async upsertSchemaRegistry(stream: string, registryJson: string): Promise<void> {
+    await upsertPostgresSchemaRegistry(this.executor, this.nowMs(), stream, registryJson);
+  }
+
+  async setSchemaUploadedSizeBytes(stream: string, sizeBytes: number): Promise<void> {
+    await setPostgresSchemaUploadedSizeBytes(this.executor, this.nowMs(), stream, sizeBytes);
+  }
+
+  private async ensureStreamTouchState(stream: string): Promise<void> {
+    await ensurePostgresStreamTouchStateFromStream(this.executor, this.nowMs(), stream);
+  }
+
+  private async deleteStreamTouchState(stream: string): Promise<void> {
+    await deletePostgresStreamTouchState(this.executor, stream);
+  }
+
+  private get executor(): PgExecutor {
+    return this.tx ?? this.pool;
+  }
+
+  private async rollbackActiveTransaction(): Promise<void> {
+    if (!this.tx) return;
+    const client = this.tx;
+    this.tx = null;
+    try {
+      await client.query("ROLLBACK");
+    } finally {
+      client.release();
+    }
+  }
+}

package/src/postgres/companions.ts

@@ -0,0 +1,197 @@
+import type { Pool } from "pg";
+import type { CompanionProgressStore, SearchCompanionIndexStore } from "../store/index_store";
+import type { SearchCompanionPlanRow, SearchSegmentCompanionRow } from "../store/rows";
+import type { PgExecutor } from "./types";
+import { getSegmentByIndexWithExecutor, pgInt, toBigInt } from "./rows";
+
+export class PostgresCompanionIndexStore implements SearchCompanionIndexStore, CompanionProgressStore {
+  constructor(private readonly pool: Pool, private readonly currentTimeMs: () => bigint) {}
+
+  async getSegmentByIndex(stream: string, segmentIndex: number) {
+    return getSegmentByIndexWithExecutor(this.pool, stream, segmentIndex);
+  }
+
+  async countUploadedSegments(stream: string): Promise<number> {
+    const res = await this.pool.query<{ max_idx: number | string | null }>(
+      `SELECT MAX(segment_index) AS max_idx FROM segments WHERE stream = $1 AND r2_etag IS NOT NULL;`,
+      [stream]
+    );
+    const maxIdx = res.rows[0]?.max_idx == null ? -1 : Number(res.rows[0]!.max_idx);
+    return maxIdx >= 0 ? maxIdx + 1 : 0;
+  }
+
+  async getSearchCompanionPlan(stream: string): Promise<SearchCompanionPlanRow | null> {
+    return getPostgresSearchCompanionPlan(this.pool, stream);
+  }
+
+  async listSearchCompanionPlanStreams(): Promise<string[]> {
+    const res = await this.pool.query<{ stream: string }>(`SELECT stream FROM search_companion_plans ORDER BY stream ASC;`);
+    return res.rows.map((row) => String(row.stream));
+  }
+
+  async upsertSearchCompanionPlan(stream: string, generation: number, planHash: string, planJson: string): Promise<void> {
+    await upsertPostgresSearchCompanionPlan(this.pool, this.currentTimeMs(), stream, generation, planHash, planJson);
+  }
+
+  async deleteSearchCompanionPlan(stream: string): Promise<void> {
+    await this.pool.query(`DELETE FROM search_companion_plans WHERE stream = $1;`, [stream]);
+  }
+
+  async getSearchSegmentCompanion(stream: string, segmentIndex: number): Promise<SearchSegmentCompanionRow | null> {
+    return getPostgresSearchSegmentCompanion(this.pool, stream, segmentIndex);
+  }
+
+  async listSearchSegmentCompanions(stream: string): Promise<SearchSegmentCompanionRow[]> {
+    return listPostgresSearchSegmentCompanions(this.pool, stream);
+  }
+
+  async upsertSearchSegmentCompanion(
+    stream: string,
+    segmentIndex: number,
+    objectKey: string,
+    planGeneration: number,
+    sectionsJson: string,
+    sectionSizesJson: string,
+    sizeBytes: number,
+    primaryTimestampMinMs: bigint | null,
+    primaryTimestampMaxMs: bigint | null
+  ): Promise<void> {
+    await upsertPostgresSearchSegmentCompanion(
+      this.pool,
+      this.currentTimeMs(),
+      stream,
+      segmentIndex,
+      objectKey,
+      planGeneration,
+      sectionsJson,
+      sectionSizesJson,
+      sizeBytes,
+      primaryTimestampMinMs,
+      primaryTimestampMaxMs
+    );
+  }
+
+  async deleteSearchSegmentCompanions(stream: string): Promise<void> {
+    await this.pool.query(`DELETE FROM search_segment_companions WHERE stream = $1;`, [stream]);
+  }
+}
+
+export async function upsertPostgresSearchCompanionPlan(
+  executor: PgExecutor,
+  nowMs: bigint,
+  stream: string,
+  generation: number,
+  planHash: string,
+  planJson: string
+): Promise<void> {
+  await executor.query(
+    `INSERT INTO search_companion_plans(stream, generation, plan_hash, plan_json, updated_at_ms)
+     VALUES($1, $2, $3, $4, $5)
+     ON CONFLICT(stream) DO UPDATE SET
+       generation = excluded.generation,
+       plan_hash = excluded.plan_hash,
+       plan_json = excluded.plan_json,
+       updated_at_ms = excluded.updated_at_ms;`,
+    [stream, generation, planHash, planJson, pgInt(nowMs)]
+  );
+}
+
+export async function upsertPostgresSearchSegmentCompanion(
+  executor: PgExecutor,
+  nowMs: bigint,
+  stream: string,
+  segmentIndex: number,
+  objectKey: string,
+  planGeneration: number,
+  sectionsJson: string,
+  sectionSizesJson: string,
+  sizeBytes: number,
+  primaryTimestampMinMs: bigint | null,
+  primaryTimestampMaxMs: bigint | null
+): Promise<void> {
+  await executor.query(
+    `INSERT INTO search_segment_companions(
+       stream, segment_index, object_key, plan_generation, sections_json, section_sizes_json,
+       size_bytes, primary_timestamp_min_ms, primary_timestamp_max_ms, updated_at_ms
+     )
+     VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+     ON CONFLICT(stream, segment_index) DO UPDATE SET
+       object_key = excluded.object_key,
+       plan_generation = excluded.plan_generation,
+       sections_json = excluded.sections_json,
+       section_sizes_json = excluded.section_sizes_json,
+       size_bytes = excluded.size_bytes,
+       primary_timestamp_min_ms = excluded.primary_timestamp_min_ms,
+       primary_timestamp_max_ms = excluded.primary_timestamp_max_ms,
+       updated_at_ms = excluded.updated_at_ms;`,
+    [
+      stream,
+      segmentIndex,
+      objectKey,
+      planGeneration,
+      sectionsJson,
+      sectionSizesJson,
+      sizeBytes,
+      primaryTimestampMinMs == null ? null : pgInt(primaryTimestampMinMs),
+      primaryTimestampMaxMs == null ? null : pgInt(primaryTimestampMaxMs),
+      pgInt(nowMs),
+    ]
+  );
+}
+
+export async function loadPostgresSearchCompanionManifest(
+  executor: PgExecutor,
+  stream: string
+): Promise<{ searchCompanionPlan: SearchCompanionPlanRow | null; searchSegmentCompanions: SearchSegmentCompanionRow[] }> {
+  return {
+    searchCompanionPlan: await getPostgresSearchCompanionPlan(executor, stream),
+    searchSegmentCompanions: await listPostgresSearchSegmentCompanions(executor, stream),
+  };
+}
+
+export async function getPostgresSearchCompanionPlan(executor: PgExecutor, stream: string): Promise<SearchCompanionPlanRow | null> {
+  const res = await executor.query(`SELECT * FROM search_companion_plans WHERE stream = $1 LIMIT 1;`, [stream]);
+  return res.rows[0] ? coerceSearchCompanionPlanRow(res.rows[0]) : null;
+}
+
+export async function getPostgresSearchSegmentCompanion(
+  executor: PgExecutor,
+  stream: string,
+  segmentIndex: number
+): Promise<SearchSegmentCompanionRow | null> {
+  const res = await executor.query(`SELECT * FROM search_segment_companions WHERE stream = $1 AND segment_index = $2 LIMIT 1;`, [
+    stream,
+    segmentIndex,
+  ]);
+  return res.rows[0] ? coerceSearchSegmentCompanionRow(res.rows[0]) : null;
+}
+
+export async function listPostgresSearchSegmentCompanions(executor: PgExecutor, stream: string): Promise<SearchSegmentCompanionRow[]> {
+  const res = await executor.query(`SELECT * FROM search_segment_companions WHERE stream = $1 ORDER BY segment_index ASC;`, [stream]);
+  return res.rows.map(coerceSearchSegmentCompanionRow);
+}
+
+function coerceSearchCompanionPlanRow(row: any): SearchCompanionPlanRow {
+  return {
+    stream: String(row.stream),
+    generation: Number(row.generation),
+    plan_hash: String(row.plan_hash),
+    plan_json: String(row.plan_json),
+    updated_at_ms: toBigInt(row.updated_at_ms),
+  };
+}
+
+function coerceSearchSegmentCompanionRow(row: any): SearchSegmentCompanionRow {
+  return {
+    stream: String(row.stream),
+    segment_index: Number(row.segment_index),
+    object_key: String(row.object_key),
+    plan_generation: Number(row.plan_generation),
+    sections_json: String(row.sections_json),
+    section_sizes_json: String(row.section_sizes_json),
+    size_bytes: Number(row.size_bytes),
+    primary_timestamp_min_ms: row.primary_timestamp_min_ms == null ? null : toBigInt(row.primary_timestamp_min_ms),
+    primary_timestamp_max_ms: row.primary_timestamp_max_ms == null ? null : toBigInt(row.primary_timestamp_max_ms),
+    updated_at_ms: toBigInt(row.updated_at_ms),
+  };
+}

package/src/postgres/control_restore.ts

@@ -0,0 +1,109 @@
+import type { StreamReadRow } from "../store/segment_read_store";
+import type { PgExecutor } from "./types";
+import { pgInt } from "./rows";
+
+export async function restorePostgresStreamRow(executor: PgExecutor, row: StreamReadRow): Promise<void> {
+  await executor.query(
+    `INSERT INTO streams(
+       stream, created_at_ms, updated_at_ms, content_type, profile, stream_seq, closed,
+       closed_producer_id, closed_producer_epoch, closed_producer_seq, ttl_seconds,
+       epoch, next_offset, sealed_through, uploaded_through, uploaded_segment_count,
+       pending_rows, pending_bytes, logical_size_bytes, wal_rows, wal_bytes,
+       last_append_ms, last_segment_cut_ms, segment_in_progress, expires_at_ms, stream_flags
+     )
+     VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16,
+            $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)
+     ON CONFLICT(stream) DO UPDATE SET
+       created_at_ms = excluded.created_at_ms,
+       updated_at_ms = excluded.updated_at_ms,
+       content_type = excluded.content_type,
+       profile = excluded.profile,
+       stream_seq = excluded.stream_seq,
+       closed = excluded.closed,
+       closed_producer_id = excluded.closed_producer_id,
+       closed_producer_epoch = excluded.closed_producer_epoch,
+       closed_producer_seq = excluded.closed_producer_seq,
+       ttl_seconds = excluded.ttl_seconds,
+       epoch = excluded.epoch,
+       next_offset = excluded.next_offset,
+       sealed_through = excluded.sealed_through,
+       uploaded_through = excluded.uploaded_through,
+       uploaded_segment_count = excluded.uploaded_segment_count,
+       pending_rows = excluded.pending_rows,
+       pending_bytes = excluded.pending_bytes,
+       logical_size_bytes = excluded.logical_size_bytes,
+       wal_rows = excluded.wal_rows,
+       wal_bytes = excluded.wal_bytes,
+       last_append_ms = excluded.last_append_ms,
+       last_segment_cut_ms = excluded.last_segment_cut_ms,
+       segment_in_progress = excluded.segment_in_progress,
+       segment_claim_token = NULL,
+       segment_claimed_at_ms = NULL,
+       expires_at_ms = excluded.expires_at_ms,
+       stream_flags = excluded.stream_flags;`,
+    [
+      row.stream,
+      pgInt(row.created_at_ms),
+      pgInt(row.updated_at_ms),
+      row.content_type,
+      row.profile,
+      row.stream_seq,
+      row.closed,
+      row.closed_producer_id,
+      row.closed_producer_epoch,
+      row.closed_producer_seq,
+      row.ttl_seconds,
+      row.epoch,
+      pgInt(row.next_offset),
+      pgInt(row.sealed_through),
+      pgInt(row.uploaded_through),
+      row.uploaded_segment_count ?? 0,
+      pgInt(row.pending_rows),
+      pgInt(row.pending_bytes),
+      pgInt(row.logical_size_bytes),
+      pgInt(row.wal_rows),
+      pgInt(row.wal_bytes),
+      pgInt(row.last_append_ms),
+      pgInt(row.last_segment_cut_ms),
+      row.segment_in_progress ?? 0,
+      row.expires_at_ms == null ? null : pgInt(row.expires_at_ms),
+      row.stream_flags,
+    ]
+  );
+}
+
+export async function upsertPostgresStreamProfile(
+  executor: PgExecutor,
+  nowMs: bigint,
+  stream: string,
+  profileJson: string
+): Promise<void> {
+  await executor.query(
+    `INSERT INTO stream_profiles(stream, profile_json, updated_at_ms)
+     VALUES($1, $2, $3)
+     ON CONFLICT(stream) DO UPDATE SET
+       profile_json = excluded.profile_json,
+       updated_at_ms = excluded.updated_at_ms;`,
+    [stream, profileJson, pgInt(nowMs)]
+  );
+}
+
+export async function deletePostgresStreamProfile(executor: PgExecutor, stream: string): Promise<void> {
+  await executor.query(`DELETE FROM stream_profiles WHERE stream = $1;`, [stream]);
+}
+
+export async function upsertPostgresSchemaRegistry(
+  executor: PgExecutor,
+  nowMs: bigint,
+  stream: string,
+  registryJson: string
+): Promise<void> {
+  await executor.query(
+    `INSERT INTO schemas(stream, schema_json, updated_at_ms)
+     VALUES($1, $2, $3)
+     ON CONFLICT(stream) DO UPDATE SET
+       schema_json = excluded.schema_json,
+       updated_at_ms = excluded.updated_at_ms;`,
+    [stream, registryJson, pgInt(nowMs)]
+  );
+}