@sixb/pg 0.0.1

package/src/ontology-storage/outbox.ts

@@ -0,0 +1,216 @@
+import {
+  MaterializationConflictError,
+  MaterializationValidationError,
+} from "@sixb/core/internal/materialization"
+import type {
+  ClaimedOntologyOutboxRow,
+  ClaimOntologyOutboxInput,
+  CompleteOntologyOutboxLeaseInput,
+  OntologyOutboxStorage,
+  OntologyOutboxSummary,
+  PurgePublishedOntologyOutboxInput,
+  RescheduleOntologyOutboxLeaseInput,
+  SummarizeOntologyOutboxInput,
+} from "@sixb/core/storage"
+import {
+  assertNonblank,
+  assertPositiveInteger,
+  assertTimestamp,
+  outboxRecord,
+  type PgOntologyOutboxRow,
+  type PgRootOperation,
+} from "./shared"
+
+export class PgOntologyOutboxStorage implements OntologyOutboxStorage {
+  constructor(private readonly runRootOperation: PgRootOperation) {}
+
+  async claim(input: ClaimOntologyOutboxInput): Promise<readonly ClaimedOntologyOutboxRow[]> {
+    return this.runRootOperation(async (sql) => {
+      assertNonblank(input.projectId, "Ontology outbox project id")
+      assertPositiveInteger(input.limit, "Ontology outbox limit")
+      const now = assertTimestamp(input.now, "Ontology outbox claim time")
+      const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry")
+      assertNonblank(input.leaseId, "Ontology outbox lease id")
+      if (leaseExpiry <= now) {
+        throw new MaterializationValidationError(
+          "Ontology outbox lease expiry must be later than the claim time."
+        )
+      }
+
+      const rows = await sql<PgOntologyOutboxRow[]>`
+        WITH candidates AS (
+          SELECT project_id, id
+          FROM ontology_outbox
+          WHERE project_id = ${input.projectId}
+            AND published_at IS NULL
+            AND available_at <= ${input.now}
+            AND (lease_expires_at IS NULL OR lease_expires_at <= ${input.now})
+          ORDER BY created_at, commit_id, commit_ordinal
+          LIMIT ${input.limit}
+          FOR UPDATE SKIP LOCKED
+        ), claimed AS (
+          UPDATE ontology_outbox AS outbox
+          SET attempts = outbox.attempts + 1,
+            lease_id = ${input.leaseId},
+            lease_expires_at = ${input.leaseExpiresAt}
+          FROM candidates
+          WHERE outbox.project_id = candidates.project_id AND outbox.id = candidates.id
+          RETURNING outbox.id AS row_id, outbox.envelope, outbox.available_at,
+            outbox.attempts, outbox.lease_id, outbox.lease_expires_at,
+            outbox.published_at, outbox.last_error, outbox.created_at,
+            outbox.commit_id, outbox.commit_ordinal
+        )
+        SELECT * FROM claimed ORDER BY created_at, commit_id, commit_ordinal
+      `
+      return rows.map((row) => {
+        const record = outboxRecord(row)
+        if (record.leaseId === null || record.leaseExpiresAt === null) {
+          throw new MaterializationConflictError(
+            "outbox-lease",
+            "Ontology outbox claim returned an unpaired lease."
+          )
+        }
+        return { ...record, leaseId: record.leaseId, leaseExpiresAt: record.leaseExpiresAt }
+      })
+    })
+  }
+
+  async markPublished(input: CompleteOntologyOutboxLeaseInput): Promise<void> {
+    await this.runRootOperation(async (sql) => {
+      assertLeaseInput(input)
+      assertTimestamp(input.publishedAt, "Ontology outbox publishedAt")
+      const ids = validateLeaseIds(input.ids)
+      if (ids.length === 0) return
+      const rows = await sql<{ readonly id: string }[]>`
+        WITH leased AS MATERIALIZED (
+          SELECT id FROM ontology_outbox
+          WHERE project_id = ${input.projectId}
+            AND id = ANY(${sql.array(ids)}::text[])
+            AND lease_id = ${input.leaseId}
+            AND lease_expires_at IS NOT NULL
+          FOR UPDATE
+        ), eligible AS (
+          SELECT id FROM leased
+          WHERE (SELECT COUNT(*) FROM leased) = ${ids.length}
+        )
+        UPDATE ontology_outbox AS outbox
+        SET published_at = ${input.publishedAt}, lease_id = NULL, lease_expires_at = NULL
+        FROM eligible
+        WHERE outbox.project_id = ${input.projectId} AND outbox.id = eligible.id
+        RETURNING outbox.id
+      `
+      if (rows.length !== ids.length) throw leaseConflict()
+    })
+  }
+
+  async reschedule(input: RescheduleOntologyOutboxLeaseInput): Promise<void> {
+    await this.runRootOperation(async (sql) => {
+      assertLeaseInput(input)
+      assertTimestamp(input.availableAt, "Ontology outbox availableAt")
+      const ids = validateLeaseIds(input.ids)
+      if (ids.length === 0) return
+      const rows = await sql<{ readonly id: string }[]>`
+        WITH leased AS MATERIALIZED (
+          SELECT id FROM ontology_outbox
+          WHERE project_id = ${input.projectId}
+            AND id = ANY(${sql.array(ids)}::text[])
+            AND lease_id = ${input.leaseId}
+            AND lease_expires_at IS NOT NULL
+          FOR UPDATE
+        ), eligible AS (
+          SELECT id FROM leased
+          WHERE (SELECT COUNT(*) FROM leased) = ${ids.length}
+        )
+        UPDATE ontology_outbox AS outbox
+        SET available_at = ${input.availableAt}, last_error = ${input.error},
+          lease_id = NULL, lease_expires_at = NULL
+        FROM eligible
+        WHERE outbox.project_id = ${input.projectId} AND outbox.id = eligible.id
+        RETURNING outbox.id
+      `
+      if (rows.length !== ids.length) throw leaseConflict()
+    })
+  }
+
+  async purgePublished(input: PurgePublishedOntologyOutboxInput): Promise<number> {
+    return this.runRootOperation(async (sql) => {
+      assertNonblank(input.projectId, "Ontology outbox project id")
+      assertPositiveInteger(input.limit, "Ontology outbox limit")
+      assertTimestamp(input.publishedBefore, "Ontology outbox purge cutoff")
+      const rows = await sql<{ readonly id: string }[]>`
+        WITH selected AS (
+          SELECT project_id, id
+          FROM ontology_outbox
+          WHERE project_id = ${input.projectId}
+            AND published_at IS NOT NULL
+            AND published_at < ${input.publishedBefore}
+          ORDER BY published_at, id
+          LIMIT ${input.limit}
+          FOR UPDATE SKIP LOCKED
+        )
+        DELETE FROM ontology_outbox AS outbox
+        USING selected
+        WHERE outbox.project_id = selected.project_id AND outbox.id = selected.id
+        RETURNING outbox.id
+      `
+      return rows.length
+    })
+  }
+
+  async summarize(input: SummarizeOntologyOutboxInput): Promise<OntologyOutboxSummary> {
+    return this.runRootOperation(async (sql) => {
+      assertNonblank(input.projectId, "Ontology outbox project id")
+      const [summary] = await sql<
+        {
+          readonly pending_count: number
+          readonly oldest_pending_at: Date | string | null
+          readonly retrying_count: number
+          readonly max_attempts: number
+        }[]
+      >`
+        SELECT COUNT(*)::int AS pending_count,
+          MIN(created_at) AS oldest_pending_at,
+          COUNT(*) FILTER (WHERE attempts > 0)::int AS retrying_count,
+          COALESCE(MAX(attempts), 0)::int AS max_attempts
+        FROM ontology_outbox
+        WHERE project_id = ${input.projectId} AND published_at IS NULL
+      `
+      return {
+        pendingCount: summary?.pending_count ?? 0,
+        oldestPendingAt: toOptionalIsoString(summary?.oldest_pending_at),
+        retryingCount: summary?.retrying_count ?? 0,
+        maxAttempts: summary?.max_attempts ?? 0,
+      }
+    })
+  }
+}
+
+function toOptionalIsoString(value: Date | string | null | undefined): string | null {
+  if (value === null || value === undefined) return null
+  return value instanceof Date ? value.toISOString() : new Date(value).toISOString()
+}
+
+function validateLeaseIds(ids: readonly string[]): string[] {
+  const seen = new Set<string>()
+  for (const id of ids) {
+    assertNonblank(id, "Ontology outbox event id")
+    if (seen.has(id)) {
+      throw new MaterializationValidationError(`Ontology outbox lease batch repeats event '${id}'.`)
+    }
+    seen.add(id)
+  }
+  return [...seen]
+}
+
+function assertLeaseInput(input: {
+  readonly projectId: string
+  readonly ids: readonly string[]
+  readonly leaseId: string
+}): void {
+  assertNonblank(input.projectId, "Ontology outbox project id")
+  assertNonblank(input.leaseId, "Ontology outbox lease id")
+}
+
+function leaseConflict(): MaterializationConflictError {
+  return new MaterializationConflictError("outbox-lease", "Ontology outbox lease does not match.")
+}

package/src/ontology-storage/shared.ts

@@ -0,0 +1,293 @@
+import type { ProjectionEntityRef } from "@sixb/core/internal/materialization"
+import { MaterializationConflictError } from "@sixb/core/internal/materialization"
+import type {
+  AssertSourceMaterializationExecutionInput,
+  OntologyCommitRecord,
+  OntologyCommitWrite,
+  OntologyOutboxRecord,
+  OntologySourceRecord,
+  StageSourceAssertion,
+} from "@sixb/core/storage"
+import type { SQLClient } from "../pg-client"
+
+export {
+  assertNonblank,
+  assertNonnegativeInteger,
+  assertPositiveInteger,
+  assertTimestamp,
+  canonicalJson,
+  linkRefFromColumns,
+  objectRefFromColumns,
+  originColumns,
+  originWhere,
+  sourceEntityKey,
+} from "@sixb/core/internal/ontology-storage-provider"
+
+export type PgRootOperation = <T>(run: (sql: SQLClient) => Promise<T>) => Promise<T>
+
+type PgJsonValue = Parameters<SQLClient["json"]>[0]
+
+export interface PgOntologyCommitRow {
+  readonly project_id: string
+  readonly id: string
+  readonly idempotency_key: string
+  readonly request_hash: string
+  readonly origin_kind: string
+  readonly origin_run_id: string | null
+  readonly origin_batch_ordinal: number | string | null
+  readonly origin: unknown
+  readonly actor: unknown | null
+  readonly ontology_revision: string
+  readonly projection_revision: string | null
+  readonly ownership_hash: string | null
+  readonly intent: unknown
+  readonly result: unknown
+  readonly committed_at: Date | string
+}
+
+export interface PgOntologySourceRow {
+  readonly project_id: string
+  readonly source_id: string
+  readonly materialization_id: string
+  readonly projection_run_id: string
+  readonly projection_kind: "object" | "link"
+  readonly protocol: "replacement"
+  readonly status: OntologySourceRecord["status"]
+  readonly execution_token: string | null
+  readonly dataset_id: string
+  readonly dataset_version_id: string
+  readonly dataset_version_created_at: Date | string
+  readonly projection_revision: string
+  readonly ownership_hash: string
+  readonly ontology_revision: string
+  readonly root_count: number | string | null
+  readonly assertion_count: number | string | null
+  readonly created_at: Date | string
+  readonly ready_at: Date | string | null
+  readonly activated_at: Date | string | null
+  readonly terminal_at: Date | string | null
+  readonly last_commit_id: string | null
+  readonly updated_at: Date | string
+}
+
+export interface PgOntologySourceAssertionRow {
+  readonly project_id: string
+  readonly source_id: string
+  readonly materialization_id: string
+  readonly entity_kind: "object" | "link"
+  readonly entity_key: unknown
+  readonly entity_sort_key: string
+  readonly root_kind: "object" | "link"
+  readonly root_key: unknown
+  readonly root_sort_key: string
+  readonly staging_ordinal: number | string
+  readonly root: unknown
+  readonly assertion: unknown
+}
+
+export interface PgOntologyOverrideRow {
+  readonly entity_kind: "object" | "link"
+  readonly value: unknown
+  readonly last_commit_id: string
+  readonly updated_at: Date | string
+}
+
+export interface PgOntologyOutboxRow {
+  readonly envelope: unknown
+  readonly available_at: Date | string
+  readonly attempts: number | string
+  readonly lease_id: string | null
+  readonly lease_expires_at: Date | string | null
+  readonly published_at: Date | string | null
+  readonly last_error: string | null
+  readonly created_at: Date | string
+}
+
+export function jsonParameter(sql: SQLClient, value: unknown): ReturnType<SQLClient["json"]> {
+  return sql.json(value as PgJsonValue)
+}
+
+export function jsonKeyParameter(
+  sql: SQLClient,
+  canonicalKey: string
+): ReturnType<SQLClient["json"]> {
+  return jsonParameter(sql, JSON.parse(canonicalKey))
+}
+
+export function toIsoString(value: Date | string): string {
+  return value instanceof Date ? value.toISOString() : new Date(value).toISOString()
+}
+
+export function optionalIsoString(value: Date | string | null): string | null {
+  return value === null ? null : toIsoString(value)
+}
+
+export function databaseSafeInteger(value: number | string, label: string): number {
+  const result = Number(value)
+  if (!Number.isSafeInteger(result) || result < 0) {
+    throw new MaterializationConflictError(
+      "effective-state",
+      `${label} is outside the nonnegative safe-integer range.`
+    )
+  }
+  return result
+}
+
+export function sourceRecord(row: PgOntologySourceRow): OntologySourceRecord {
+  return {
+    projectId: row.project_id,
+    source: { projectionId: row.source_id },
+    materializationId: row.materialization_id,
+    projectionRunId: row.projection_run_id,
+    projectionKind: row.projection_kind,
+    protocol: row.protocol,
+    status: row.status,
+    executionToken: row.execution_token,
+    datasetVersion: {
+      datasetId: row.dataset_id,
+      versionId: row.dataset_version_id,
+      createdAt: toIsoString(row.dataset_version_created_at),
+    },
+    projectionRevision: row.projection_revision,
+    ownershipHash: row.ownership_hash,
+    ontologyRevision: row.ontology_revision,
+    rootCount:
+      row.root_count === null ? null : databaseSafeInteger(row.root_count, "Source root count"),
+    assertionCount:
+      row.assertion_count === null
+        ? null
+        : databaseSafeInteger(row.assertion_count, "Source assertion count"),
+    createdAt: toIsoString(row.created_at),
+    readyAt: optionalIsoString(row.ready_at),
+    activatedAt: optionalIsoString(row.activated_at),
+    terminalAt: optionalIsoString(row.terminal_at),
+    lastCommitId: row.last_commit_id,
+    updatedAt: toIsoString(row.updated_at),
+  }
+}
+
+export function sourceAssertion(row: PgOntologySourceAssertionRow): StageSourceAssertion {
+  return {
+    root: structuredClone(row.root) as ProjectionEntityRef,
+    assertion: structuredClone(row.assertion) as StageSourceAssertion["assertion"],
+    stagingOrdinal: databaseSafeInteger(row.staging_ordinal, "Source staging ordinal"),
+  }
+}
+
+export function commitRecord(row: PgOntologyCommitRow): OntologyCommitRecord {
+  const base = {
+    projectId: row.project_id,
+    id: row.id,
+    idempotencyKey: row.idempotency_key,
+    requestHash: row.request_hash,
+    origin: structuredClone(row.origin) as OntologyCommitWrite["origin"],
+    ...(row.actor === null
+      ? {}
+      : {
+          actor: structuredClone(row.actor) as NonNullable<OntologyCommitWrite["actor"]>,
+        }),
+    ontologyRevision: row.ontology_revision,
+    ...(row.projection_revision === null ? {} : { projectionRevision: row.projection_revision }),
+    ...(row.ownership_hash === null ? {} : { ownershipHash: row.ownership_hash }),
+    committedAt: toIsoString(row.committed_at),
+  }
+  const intent = structuredClone(row.intent) as OntologyCommitWrite["intent"]
+  const result = structuredClone(row.result) as OntologyCommitRecord["result"]
+  return { ...base, intent, result } as OntologyCommitRecord
+}
+
+export function outboxRecord(row: PgOntologyOutboxRow): OntologyOutboxRecord {
+  return {
+    envelope: structuredClone(row.envelope) as OntologyOutboxRecord["envelope"],
+    availableAt: toIsoString(row.available_at),
+    attempts: databaseSafeInteger(row.attempts, "Ontology outbox attempts"),
+    leaseId: row.lease_id,
+    leaseExpiresAt: optionalIsoString(row.lease_expires_at),
+    publishedAt: optionalIsoString(row.published_at),
+    lastError: row.last_error,
+    createdAt: toIsoString(row.created_at),
+  }
+}
+
+export async function assertProjectionExecution(
+  sql: SQLClient,
+  input: {
+    readonly projectId: string
+    readonly sourceId: string
+    readonly projectionRunId: string
+    readonly executionToken: string
+    readonly identity?: AssertSourceMaterializationExecutionInput["identity"]
+  }
+): Promise<void> {
+  const [run] = await sql<
+    {
+      readonly projection_id: string
+      readonly projection_kind: string
+      readonly status: string
+      readonly materialization_protocol: string | null
+      readonly execution_token: string | null
+      readonly dataset_id: string
+      readonly dataset_version_id: string
+      readonly dataset_version_created_at: Date | string | null
+      readonly ontology_revision: string | null
+      readonly projection_revision: string | null
+      readonly ownership_hash: string | null
+    }[]
+  >`
+    SELECT projection_id, projection_kind, status, materialization_protocol, execution_token,
+      dataset_id, dataset_version_id, dataset_version_created_at,
+      ontology_revision, projection_revision, ownership_hash
+    FROM projection_runs
+    WHERE project_id = ${input.projectId} AND id = ${input.projectionRunId}
+    FOR UPDATE
+  `
+  if (!run || run.status !== "running") {
+    throw new MaterializationConflictError(
+      "run-correlation",
+      `Projection run '${input.projectionRunId}' is missing or is not running.`
+    )
+  }
+  if (run.projection_id !== input.sourceId || run.materialization_protocol !== "replacement") {
+    throw new MaterializationConflictError(
+      "run-correlation",
+      `Projection run '${input.projectionRunId}' does not own replacement source '${input.sourceId}'.`
+    )
+  }
+  if (run.execution_token !== input.executionToken) {
+    throw new MaterializationConflictError(
+      "execution-lost",
+      `Projection run '${input.projectionRunId}' execution token is stale.`
+    )
+  }
+  const identity = input.identity
+  if (
+    identity &&
+    (run.projection_kind !== identity.projectionKind ||
+      run.materialization_protocol !== identity.protocol ||
+      run.dataset_id !== identity.datasetVersion.datasetId ||
+      run.dataset_version_id !== identity.datasetVersion.versionId ||
+      run.dataset_version_created_at === null ||
+      toIsoString(run.dataset_version_created_at) !== identity.datasetVersion.createdAt ||
+      run.ontology_revision !== identity.ontologyRevision ||
+      run.projection_revision !== identity.projectionRevision ||
+      run.ownership_hash !== identity.ownershipHash)
+  ) {
+    throw new MaterializationConflictError(
+      "run-correlation",
+      `Projection run '${input.projectionRunId}' immutable source identity does not match.`
+    )
+  }
+}
+
+export function requireRow<T>(
+  row: T | undefined,
+  kind: ConstructorParameters<typeof MaterializationConflictError>[0],
+  message: string
+): T {
+  if (!row) throw new MaterializationConflictError(kind, message)
+  return row
+}
+
+export function ontologyLockKey(kind: string, ...parts: readonly string[]): string {
+  return `ontology:${kind}:${JSON.stringify(parts)}`
+}