@sixb/pg 0.0.1

package/src/migrations.ts

@@ -0,0 +1,280 @@
+import { createHash } from "node:crypto"
+import type { StorageMigrator } from "@sixb/core"
+import type {
+  MigrationHistoryStore,
+  MigrationRecord,
+  MigrationSet,
+  MigrationStep,
+  MigrationStepOptions,
+} from "@sixb/core/storage"
+import {
+  defineMigrations,
+  describeMigrationHistory,
+  runMigrationSet,
+  step,
+} from "@sixb/core/storage"
+import initialSchemaSql from "./migrations/001-initial-schema.sql" with { type: "text" }
+import type { SQL, SQLClient } from "./pg-client"
+
+export interface PostgresMigrationContext {
+  exec(sqlText: string): Promise<void>
+}
+
+export const POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage"
+
+export function pgSql(id: string, sqlText: string): MigrationStep<PostgresMigrationContext> {
+  return pgStep(id, (context) => context.exec(sqlText), { checksum: checksum(sqlText) })
+}
+
+export function pgStep(
+  id: string,
+  up: (context: PostgresMigrationContext) => void | Promise<void>,
+  options: MigrationStepOptions = {}
+): MigrationStep<PostgresMigrationContext> {
+  return step<PostgresMigrationContext>(id, up, options)
+}
+
+export function createPostgresStorageMigrators(
+  sql: SQL,
+  schemaName: string
+): readonly StorageMigrator[] {
+  return [
+    createPostgresMigrator({
+      sql,
+      schemaName,
+      migrations: postgresStorageMigrations,
+    }),
+  ]
+}
+
+export function createPostgresMigrator(params: {
+  readonly sql: SQL
+  readonly schemaName: string
+  readonly migrations: MigrationSet<PostgresMigrationContext>
+}): StorageMigrator {
+  return {
+    adapterId: params.migrations.adapterId,
+    latestVersion: params.migrations.latestVersion,
+    async status() {
+      return describeMigrationHistory({
+        migrations: params.migrations,
+        rows: await readPostgresHistory(params.sql, params.schemaName, params.migrations.adapterId),
+      })
+    },
+    migrate() {
+      return withPostgresMigrationLock(params, async (sql) => {
+        const session = postgresMigrationSession(sql, params.schemaName)
+        return runMigrationSet({
+          context: session.context,
+          migrations: params.migrations,
+          state: session.state,
+        })
+      })
+    },
+  }
+}
+
+export async function migratePostgresStorage(sql: SQL, schemaName: string): Promise<void> {
+  await createPostgresMigrator({
+    sql,
+    schemaName,
+    migrations: postgresStorageMigrations,
+  }).migrate()
+}
+
+export async function dropSchema(sql: SQL, schemaName: string): Promise<void> {
+  await sql.unsafe(`DROP SCHEMA IF EXISTS ${quoteIdent(schemaName)} CASCADE`)
+}
+
+export function quoteIdent(identifier: string): string {
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(identifier)) {
+    throw new Error(`[SixbPg] Invalid identifier: ${identifier}`)
+  }
+  return `"${identifier}"`
+}
+
+function postgresMigrationSession(
+  sql: SQL,
+  schemaName: string
+): { context: PostgresMigrationContext; state: MigrationHistoryStore } {
+  const schema = quoteIdent(schemaName)
+  const migrationsTable = `${schema}.sixb_migrations`
+  // Migration steps receive a stable context; route exec() through the active transaction.
+  let active: SQLClient | null = null
+  const connection = (): SQLClient => active ?? sql
+
+  return {
+    context: {
+      async exec(sqlText) {
+        await connection().unsafe(sqlText)
+      },
+    },
+    state: {
+      async ensure() {
+        await sql.unsafe(`
+          CREATE SCHEMA IF NOT EXISTS ${schema};
+
+          CREATE TABLE IF NOT EXISTS ${migrationsTable} (
+            adapter_id TEXT NOT NULL,
+            version INTEGER NOT NULL,
+            id TEXT NOT NULL,
+            checksum TEXT,
+            status TEXT NOT NULL CHECK (status IN ('started', 'applied')),
+            started_at TIMESTAMPTZ NOT NULL,
+            finished_at TIMESTAMPTZ,
+            PRIMARY KEY (adapter_id, version)
+          );
+        `)
+      },
+      async readHistory(adapterId) {
+        const rows = await sql.unsafe<PostgresMigrationRow[]>(
+          `SELECT * FROM ${migrationsTable} WHERE adapter_id = $1 ORDER BY version`,
+          [adapterId]
+        )
+        return rows.map(rowToMigrationRecord)
+      },
+      async markStarted(adapterId, migration, at) {
+        await connection().unsafe(
+          `
+            INSERT INTO ${migrationsTable} (
+              adapter_id, version, id, checksum, status, started_at, finished_at
+            ) VALUES ($1, $2, $3, $4, 'started', $5, NULL)
+            ON CONFLICT(adapter_id, version) DO UPDATE SET
+              id = excluded.id,
+              checksum = excluded.checksum,
+              status = excluded.status,
+              started_at = excluded.started_at,
+              finished_at = NULL
+          `,
+          [adapterId, migration.version, migration.id, migration.checksum ?? null, at]
+        )
+      },
+      async markApplied(adapterId, migration, at) {
+        await connection().unsafe(
+          `
+            UPDATE ${migrationsTable}
+            SET status = 'applied', finished_at = $1
+            WHERE adapter_id = $2 AND version = $3
+          `,
+          [at, adapterId, migration.version]
+        )
+      },
+      async transaction(run) {
+        // `sql` is the reserved connection holding the migration advisory lock. porsager's
+        // reserved connection has no `.begin`, so drive the transaction manually on it and
+        // route exec()/markStarted()/markApplied() through the same connection via `active`.
+        const previous = active
+        active = sql
+        await sql.unsafe("BEGIN")
+        try {
+          await sql.unsafe(`SET LOCAL search_path TO ${schema}`)
+          const result = await run()
+          await sql.unsafe("COMMIT")
+          return result
+        } catch (error) {
+          await sql.unsafe("ROLLBACK")
+          throw error
+        } finally {
+          active = previous
+        }
+      },
+    },
+  }
+}
+
+/**
+ * Reads migration history without DDL and without the advisory lock. `null` means the
+ * history table does not exist, which is a state and not a failure.
+ *
+ * `migrate()` cannot be used for this: it calls `ensure()` first, so it runs
+ * `CREATE SCHEMA`/`CREATE TABLE` and reserves a connection to hold
+ * `pg_advisory_lock`. A probe must need no DDL grant — `/ready` is public and
+ * unauthenticated — and must not serialize N replicas behind one lock.
+ *
+ * `to_regclass` returns NULL for a missing relation (including a missing schema)
+ * instead of raising, and needs no catalog privileges.
+ */
+async function readPostgresHistory(
+  sql: SQL,
+  schemaName: string,
+  adapterId: string
+): Promise<readonly MigrationRecord[] | null> {
+  const schema = quoteIdent(schemaName)
+  const probe = await sql.unsafe<{ oid: string | null }[]>(`SELECT to_regclass($1) AS oid`, [
+    `${schema}.sixb_migrations`,
+  ])
+  if (!probe[0]?.oid) {
+    return null
+  }
+
+  const rows = await sql.unsafe<PostgresMigrationRow[]>(
+    `SELECT * FROM ${schema}.sixb_migrations WHERE adapter_id = $1 ORDER BY version`,
+    [adapterId]
+  )
+  return rows.map(rowToMigrationRecord)
+}
+
+async function withPostgresMigrationLock<T>(
+  params: {
+    readonly sql: SQL
+    readonly schemaName: string
+    readonly migrations: MigrationSet<PostgresMigrationContext>
+  },
+  run: (sql: SQL) => Promise<T>
+): Promise<T> {
+  const sql = await params.sql.reserve()
+  const [first, second] = advisoryLockParts(
+    `storage:migration:${params.schemaName}:${params.migrations.adapterId}`
+  )
+
+  try {
+    await sql`SELECT pg_advisory_lock(${first}, ${second})`
+    try {
+      return await run(sql)
+    } finally {
+      await sql`SELECT pg_advisory_unlock(${first}, ${second})`
+    }
+  } finally {
+    sql.release()
+  }
+}
+
+function advisoryLockParts(key: string): readonly [number, number] {
+  const hash = createHash("sha256").update(key).digest()
+  return [hash.readInt32BE(0), hash.readInt32BE(4)]
+}
+
+function rowToMigrationRecord(row: PostgresMigrationRow): MigrationRecord {
+  return {
+    adapterId: row.adapter_id,
+    version: row.version,
+    id: row.id,
+    checksum: row.checksum ?? undefined,
+    status: row.status,
+    startedAt: toIsoString(row.started_at),
+    finishedAt: row.finished_at ? toIsoString(row.finished_at) : undefined,
+  }
+}
+
+function toIsoString(value: Date | string): string {
+  return value instanceof Date ? value.toISOString() : new Date(value).toISOString()
+}
+
+function checksum(value: string): string {
+  return createHash("sha256").update(value).digest("hex")
+}
+
+export const postgresStorageMigrations = defineMigrations<PostgresMigrationContext>({
+  adapterId: POSTGRES_STORAGE_ADAPTER_ID,
+  steps: [pgSql("001-initial-schema", initialSchemaSql)],
+})
+
+interface PostgresMigrationRow {
+  readonly adapter_id: string
+  readonly version: number
+  readonly id: string
+  readonly checksum: string | null
+  readonly status: MigrationRecord["status"]
+  readonly started_at: Date | string
+  readonly finished_at: Date | string | null
+}

package/src/ontology-storage/commits.ts

@@ -0,0 +1,115 @@
+import type {
+  GetOntologyCommitByIdempotencyKeyInput,
+  GetOntologyCommitByIdInput,
+  GetOntologyCommitByOriginInput,
+  ListOntologyCommitsInput,
+  ListOntologyCommitsResult,
+  OntologyCommitRecord,
+  OntologyCommitStorage,
+} from "@sixb/core/storage"
+import {
+  assertNonblank,
+  assertNonnegativeInteger,
+  commitRecord,
+  originWhere,
+  type PgOntologyCommitRow,
+  type PgRootOperation,
+} from "./shared"
+
+export class PgOntologyCommitStorage implements OntologyCommitStorage {
+  constructor(private readonly runRootOperation: PgRootOperation) {}
+
+  async getByIdempotencyKey(
+    input: GetOntologyCommitByIdempotencyKeyInput
+  ): Promise<OntologyCommitRecord | null> {
+    return this.runRootOperation(async (sql) => {
+      assertNonblank(input.projectId, "Ontology commit project id")
+      assertNonblank(input.idempotencyKey, "Ontology commit idempotency key")
+      const [row] = await sql<PgOntologyCommitRow[]>`
+        SELECT * FROM ontology_commits
+        WHERE project_id = ${input.projectId} AND idempotency_key = ${input.idempotencyKey}
+      `
+      return row ? commitRecord(row) : null
+    })
+  }
+
+  async getById(input: GetOntologyCommitByIdInput): Promise<OntologyCommitRecord | null> {
+    return this.runRootOperation(async (sql) => {
+      assertNonblank(input.projectId, "Ontology commit project id")
+      assertNonblank(input.id, "Ontology commit id")
+      const [row] = await sql<PgOntologyCommitRow[]>`
+        SELECT * FROM ontology_commits
+        WHERE project_id = ${input.projectId} AND id = ${input.id}
+      `
+      return row ? commitRecord(row) : null
+    })
+  }
+
+  async getByOrigin(input: GetOntologyCommitByOriginInput): Promise<OntologyCommitRecord | null> {
+    return this.runRootOperation(async (sql) => {
+      assertNonblank(input.projectId, "Ontology commit project id")
+      const origin = originWhere(input.origin)
+      assertNonblank(origin.runId, "Ontology commit origin run id")
+      if (origin.batchOrdinal !== null) {
+        assertNonnegativeInteger(origin.batchOrdinal, "Ontology telemetry commit batch ordinal")
+      }
+      const [row] =
+        origin.batchOrdinal === null
+          ? await sql<PgOntologyCommitRow[]>`
+              SELECT * FROM ontology_commits
+              WHERE project_id = ${input.projectId}
+                AND origin_kind = ${origin.kind}
+                AND origin_run_id = ${origin.runId}
+                AND origin_batch_ordinal IS NULL
+            `
+          : await sql<PgOntologyCommitRow[]>`
+              SELECT * FROM ontology_commits
+              WHERE project_id = ${input.projectId}
+                AND origin_kind = ${origin.kind}
+                AND origin_run_id = ${origin.runId}
+                AND origin_batch_ordinal = ${origin.batchOrdinal}
+            `
+      return row ? commitRecord(row) : null
+    })
+  }
+
+  async list(input: ListOntologyCommitsInput): Promise<ListOntologyCommitsResult> {
+    return this.runRootOperation(async (sql) => {
+      assertNonblank(input.projectId, "Ontology commit project id")
+      if (input.run) assertNonblank(input.run.id, "Ontology commit run id")
+      const offset = input.offset ?? 0
+      const limit = input.limit
+      assertNonnegativeInteger(offset, "Ontology commit list offset")
+      if (limit !== undefined) assertNonnegativeInteger(limit, "Ontology commit list limit")
+
+      const runFilter =
+        input.run?.kind === "action"
+          ? sql`AND origin_kind = 'action' AND origin_run_id = ${input.run.id}`
+          : input.run?.kind === "projection"
+            ? sql`AND origin_run_id = ${input.run.id}
+                  AND origin_kind IN ('projection', 'telemetry')`
+            : sql``
+      const [{ total }] = await sql<{ readonly total: number | string }[]>`
+        SELECT COUNT(*) AS total
+        FROM ontology_commits
+        WHERE project_id = ${input.projectId} ${runFilter}
+      `
+      const direction = input.order === "desc" ? sql`DESC` : sql`ASC`
+      const order =
+        input.run?.kind === "projection"
+          ? sql`CASE WHEN origin_kind = 'telemetry' THEN origin_batch_ordinal ELSE -1 END
+                ${direction}, committed_at ${direction}, id ${direction}`
+          : sql`committed_at ${direction}, id ${direction}`
+      const limitFragment = limit === undefined ? sql`` : sql`LIMIT ${limit}`
+      const rows = await sql<PgOntologyCommitRow[]>`
+        SELECT * FROM ontology_commits
+        WHERE project_id = ${input.projectId} ${runFilter}
+        ORDER BY ${order}
+        ${limitFragment} OFFSET ${offset}
+      `
+      const count = Number(total)
+      const commits = rows.map(commitRecord)
+      return { commits, total: count, hasMore: offset + commits.length < count }
+    })
+  }
+}

package/src/ontology-storage/index.ts

@@ -0,0 +1,35 @@
+import type { OntologyStorage } from "@sixb/core/storage"
+import type { SQLClient } from "../pg-client"
+import { PgOntologyCommitStorage } from "./commits"
+import type { PgOntologyTransactionContext } from "./materialization-session"
+import { PgOntologyMaterializationStorage } from "./materializations"
+import { PgOntologyOutboxStorage } from "./outbox"
+import type { PgRootOperation } from "./shared"
+import { PgOntologySourceStorage } from "./sources"
+
+export class PgOntologyStorage implements OntologyStorage {
+  readonly commits: PgOntologyCommitStorage
+  readonly sources: PgOntologySourceStorage
+  readonly materializations: PgOntologyMaterializationStorage
+  readonly outbox: PgOntologyOutboxStorage
+
+  constructor(input: {
+    readonly sql: SQLClient
+    readonly runRootOperation: PgRootOperation
+    readonly transactionContext: PgOntologyTransactionContext | null
+  }) {
+    this.commits = new PgOntologyCommitStorage(input.runRootOperation)
+    this.sources = new PgOntologySourceStorage(input.runRootOperation)
+    this.materializations = new PgOntologyMaterializationStorage(
+      input.sql,
+      input.transactionContext
+    )
+    this.outbox = new PgOntologyOutboxStorage(input.runRootOperation)
+  }
+
+  deactivateSessions(): void {
+    this.materializations.deactivateSessions()
+  }
+}
+
+export type { PgOntologyTransactionContext } from "./materialization-session"