@sixb/sqlite 0.0.1

package/src/ontology-storage/outbox.ts

@@ -0,0 +1,237 @@
+import type { Database } from "bun:sqlite"
+import {
+  MaterializationConflictError,
+  MaterializationValidationError,
+} from "@sixb/core/internal/materialization"
+import type {
+  ClaimedOntologyOutboxRow,
+  ClaimOntologyOutboxInput,
+  CompleteOntologyOutboxLeaseInput,
+  OntologyOutboxStorage,
+  OntologyOutboxSummary,
+  PurgePublishedOntologyOutboxInput,
+  RescheduleOntologyOutboxLeaseInput,
+  SummarizeOntologyOutboxInput,
+} from "@sixb/core/storage"
+import {
+  assertNonblank,
+  assertPositiveInteger,
+  assertTimestamp,
+  outboxRecord,
+  type SqliteOntologyOutboxRow,
+  type SqliteRootOperation,
+} from "./shared"
+
+export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
+  constructor(
+    private readonly db: Database,
+    private readonly runRootOperation: SqliteRootOperation
+  ) {}
+
+  async claim(input: ClaimOntologyOutboxInput): Promise<readonly ClaimedOntologyOutboxRow[]> {
+    return this.runRootOperation(() => {
+      assertNonblank(input.projectId, "Ontology outbox project id")
+      assertPositiveInteger(input.limit, "Ontology outbox limit")
+      const now = assertTimestamp(input.now, "Ontology outbox claim time")
+      const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry")
+      assertNonblank(input.leaseId, "Ontology outbox lease id")
+      if (leaseExpiry <= now) {
+        throw new MaterializationValidationError(
+          "Ontology outbox lease expiry must be later than the claim time."
+        )
+      }
+
+      const rows = this.db
+        .query(
+          `
+            UPDATE ontology_outbox
+            SET attempts = attempts + 1, lease_id = ?, lease_expires_at = ?
+            WHERE rowid IN (
+              SELECT rowid
+              FROM ontology_outbox
+              WHERE project_id = ? AND published_at IS NULL AND available_at <= ?
+                AND (lease_expires_at IS NULL OR lease_expires_at <= ?)
+              ORDER BY created_at, commit_id, commit_ordinal
+              LIMIT ?
+            )
+            RETURNING envelope, available_at, attempts, lease_id, lease_expires_at,
+              published_at, last_error, created_at
+          `
+        )
+        .all(
+          input.leaseId,
+          input.leaseExpiresAt,
+          input.projectId,
+          input.now,
+          input.now,
+          input.limit
+        ) as SqliteOntologyOutboxRow[]
+      return rows
+        .map((row) => outboxRecord(row))
+        .sort(
+          (left, right) =>
+            left.createdAt.localeCompare(right.createdAt) ||
+            left.envelope.id.localeCompare(right.envelope.id)
+        )
+        .map((record) => {
+          if (record.leaseId === null || record.leaseExpiresAt === null) {
+            throw new MaterializationConflictError(
+              "outbox-lease",
+              "Ontology outbox claim returned an unpaired lease."
+            )
+          }
+          return { ...record, leaseId: record.leaseId, leaseExpiresAt: record.leaseExpiresAt }
+        })
+    })
+  }
+
+  async markPublished(input: CompleteOntologyOutboxLeaseInput): Promise<void> {
+    await this.runRootOperation(() => {
+      assertLeaseInput(input)
+      assertTimestamp(input.publishedAt, "Ontology outbox publishedAt")
+      const ids = validateLeaseIds(input.ids)
+      const changed = this.db
+        .query(
+          `
+            WITH requested(id) AS MATERIALIZED (
+              SELECT CAST(value AS TEXT) FROM json_each(?)
+            ), leased(id) AS MATERIALIZED (
+              SELECT outbox.id FROM ontology_outbox AS outbox
+              JOIN requested USING (id)
+              WHERE outbox.project_id = ? AND outbox.lease_id = ?
+                AND outbox.lease_expires_at IS NOT NULL
+            ), eligible(id) AS (
+              SELECT id FROM leased WHERE (SELECT COUNT(*) FROM leased) = ?
+            )
+            UPDATE ontology_outbox
+            SET published_at = ?, lease_id = NULL, lease_expires_at = NULL
+            WHERE project_id = ? AND id IN (SELECT id FROM eligible)
+              AND lease_id = ? AND lease_expires_at IS NOT NULL
+          `
+        )
+        .run(
+          JSON.stringify(ids),
+          input.projectId,
+          input.leaseId,
+          ids.length,
+          input.publishedAt,
+          input.projectId,
+          input.leaseId
+        ).changes
+      if (changed !== ids.length) throw leaseConflict()
+    })
+  }
+
+  async reschedule(input: RescheduleOntologyOutboxLeaseInput): Promise<void> {
+    await this.runRootOperation(() => {
+      assertLeaseInput(input)
+      assertTimestamp(input.availableAt, "Ontology outbox availableAt")
+      const ids = validateLeaseIds(input.ids)
+      const changed = this.db
+        .query(
+          `
+            WITH requested(id) AS MATERIALIZED (
+              SELECT CAST(value AS TEXT) FROM json_each(?)
+            ), leased(id) AS MATERIALIZED (
+              SELECT outbox.id FROM ontology_outbox AS outbox
+              JOIN requested USING (id)
+              WHERE outbox.project_id = ? AND outbox.lease_id = ?
+                AND outbox.lease_expires_at IS NOT NULL
+            ), eligible(id) AS (
+              SELECT id FROM leased WHERE (SELECT COUNT(*) FROM leased) = ?
+            )
+            UPDATE ontology_outbox
+            SET available_at = ?, last_error = ?, lease_id = NULL, lease_expires_at = NULL
+            WHERE project_id = ? AND id IN (SELECT id FROM eligible)
+              AND lease_id = ? AND lease_expires_at IS NOT NULL
+          `
+        )
+        .run(
+          JSON.stringify(ids),
+          input.projectId,
+          input.leaseId,
+          ids.length,
+          input.availableAt,
+          input.error,
+          input.projectId,
+          input.leaseId
+        ).changes
+      if (changed !== ids.length) throw leaseConflict()
+    })
+  }
+
+  async purgePublished(input: PurgePublishedOntologyOutboxInput): Promise<number> {
+    return this.runRootOperation(() => {
+      assertNonblank(input.projectId, "Ontology outbox project id")
+      assertPositiveInteger(input.limit, "Ontology outbox limit")
+      assertTimestamp(input.publishedBefore, "Ontology outbox purge cutoff")
+      return this.db
+        .query(
+          `
+            DELETE FROM ontology_outbox
+            WHERE rowid IN (
+              SELECT rowid FROM ontology_outbox
+              WHERE project_id = ? AND published_at IS NOT NULL AND published_at < ?
+              ORDER BY published_at, id
+              LIMIT ?
+            )
+          `
+        )
+        .run(input.projectId, input.publishedBefore, input.limit).changes
+    })
+  }
+
+  async summarize(input: SummarizeOntologyOutboxInput): Promise<OntologyOutboxSummary> {
+    return this.runRootOperation(() => {
+      assertNonblank(input.projectId, "Ontology outbox project id")
+      const summary = this.db
+        .query(
+          `
+            SELECT COUNT(*) AS pending_count,
+              MIN(created_at) AS oldest_pending_at,
+              SUM(CASE WHEN attempts > 0 THEN 1 ELSE 0 END) AS retrying_count,
+              COALESCE(MAX(attempts), 0) AS max_attempts
+            FROM ontology_outbox
+            WHERE project_id = ? AND published_at IS NULL
+          `
+        )
+        .get(input.projectId) as {
+        readonly pending_count: number
+        readonly oldest_pending_at: string | null
+        readonly retrying_count: number | null
+        readonly max_attempts: number
+      }
+      return {
+        pendingCount: summary.pending_count,
+        oldestPendingAt: summary.oldest_pending_at,
+        retryingCount: summary.retrying_count ?? 0,
+        maxAttempts: summary.max_attempts,
+      }
+    })
+  }
+}
+
+function validateLeaseIds(ids: readonly string[]): string[] {
+  const seen = new Set<string>()
+  for (const id of ids) {
+    assertNonblank(id, "Ontology outbox event id")
+    if (seen.has(id)) {
+      throw new MaterializationValidationError(`Ontology outbox lease batch repeats event '${id}'.`)
+    }
+    seen.add(id)
+  }
+  return [...seen]
+}
+
+function assertLeaseInput(input: {
+  readonly projectId: string
+  readonly ids: readonly string[]
+  readonly leaseId: string
+}): void {
+  assertNonblank(input.projectId, "Ontology outbox project id")
+  assertNonblank(input.leaseId, "Ontology outbox lease id")
+}
+
+function leaseConflict(): MaterializationConflictError {
+  return new MaterializationConflictError("outbox-lease", "Ontology outbox lease does not match.")
+}

package/src/ontology-storage/shared.ts

@@ -0,0 +1,269 @@
+import type { Database } from "bun:sqlite"
+import type { ProjectionEntityRef } from "@sixb/core/internal/materialization"
+import { MaterializationConflictError } from "@sixb/core/internal/materialization"
+import type {
+  AssertSourceMaterializationExecutionInput,
+  OntologyCommitRecord,
+  OntologyCommitWrite,
+  OntologyOutboxRecord,
+  OntologySourceRecord,
+  StageSourceAssertion,
+} from "@sixb/core/storage"
+
+export {
+  assertNonblank,
+  assertNonnegativeInteger,
+  assertPositiveInteger,
+  assertTimestamp,
+  canonicalJson,
+  linkRefFromColumns,
+  objectRefFromColumns,
+  originColumns,
+  originWhere,
+  sourceEntityKey,
+} from "@sixb/core/internal/ontology-storage-provider"
+
+export type SqliteRootOperation = <T>(run: () => Promise<T> | T) => Promise<T>
+
+export interface SqliteOntologyCommitRow {
+  readonly project_id: string
+  readonly id: string
+  readonly idempotency_key: string
+  readonly request_hash: string
+  readonly origin_kind: string
+  readonly origin_run_id: string | null
+  readonly origin_batch_ordinal: number | null
+  readonly origin: string
+  readonly actor: string | null
+  readonly ontology_revision: string
+  readonly projection_revision: string | null
+  readonly ownership_hash: string | null
+  readonly intent: string
+  readonly result: string
+  readonly committed_at: string
+}
+
+export interface SqliteOntologySourceRow {
+  readonly project_id: string
+  readonly source_id: string
+  readonly materialization_id: string
+  readonly projection_run_id: string
+  readonly projection_kind: "object" | "link"
+  readonly protocol: "replacement"
+  readonly status: OntologySourceRecord["status"]
+  readonly execution_token: string | null
+  readonly dataset_id: string
+  readonly dataset_version_id: string
+  readonly dataset_version_created_at: string
+  readonly projection_revision: string
+  readonly ownership_hash: string
+  readonly ontology_revision: string
+  readonly root_count: number | null
+  readonly assertion_count: number | null
+  readonly created_at: string
+  readonly ready_at: string | null
+  readonly activated_at: string | null
+  readonly terminal_at: string | null
+  readonly last_commit_id: string | null
+  readonly updated_at: string
+}
+
+export interface SqliteOntologySourceAssertionRow {
+  readonly project_id: string
+  readonly source_id: string
+  readonly materialization_id: string
+  readonly entity_kind: "object" | "link"
+  readonly entity_key: string
+  readonly entity_sort_key: string
+  readonly root_kind: "object" | "link"
+  readonly root_key: string
+  readonly root_sort_key: string
+  readonly staging_ordinal: number
+  readonly root: string
+  readonly assertion: string
+}
+
+export interface SqliteOntologyOverrideRow {
+  readonly entity_kind: "object" | "link"
+  readonly value: string
+  readonly last_commit_id: string
+  readonly updated_at: string
+}
+
+export interface SqliteOntologyOutboxRow {
+  readonly envelope: string
+  readonly available_at: string
+  readonly attempts: number
+  readonly lease_id: string | null
+  readonly lease_expires_at: string | null
+  readonly published_at: string | null
+  readonly last_error: string | null
+  readonly created_at: string
+}
+
+export function parseJson<T>(value: string): T {
+  return JSON.parse(value) as T
+}
+
+export function sourceRecord(row: SqliteOntologySourceRow): OntologySourceRecord {
+  return {
+    projectId: row.project_id,
+    source: { projectionId: row.source_id },
+    materializationId: row.materialization_id,
+    projectionRunId: row.projection_run_id,
+    projectionKind: row.projection_kind,
+    protocol: row.protocol,
+    status: row.status,
+    executionToken: row.execution_token,
+    datasetVersion: {
+      datasetId: row.dataset_id,
+      versionId: row.dataset_version_id,
+      createdAt: row.dataset_version_created_at,
+    },
+    projectionRevision: row.projection_revision,
+    ownershipHash: row.ownership_hash,
+    ontologyRevision: row.ontology_revision,
+    rootCount: row.root_count,
+    assertionCount: row.assertion_count,
+    createdAt: row.created_at,
+    readyAt: row.ready_at,
+    activatedAt: row.activated_at,
+    terminalAt: row.terminal_at,
+    lastCommitId: row.last_commit_id,
+    updatedAt: row.updated_at,
+  }
+}
+
+export function sourceAssertion(row: SqliteOntologySourceAssertionRow): StageSourceAssertion {
+  return {
+    root: parseJson<ProjectionEntityRef>(row.root),
+    assertion: parseJson<StageSourceAssertion["assertion"]>(row.assertion),
+    stagingOrdinal: row.staging_ordinal,
+  }
+}
+
+export function commitRecord(row: SqliteOntologyCommitRow): OntologyCommitRecord {
+  const base = {
+    projectId: row.project_id,
+    id: row.id,
+    idempotencyKey: row.idempotency_key,
+    requestHash: row.request_hash,
+    origin: parseJson<OntologyCommitWrite["origin"]>(row.origin),
+    ...(row.actor === null
+      ? {}
+      : { actor: parseJson<NonNullable<OntologyCommitWrite["actor"]>>(row.actor) }),
+    ontologyRevision: row.ontology_revision,
+    ...(row.projection_revision === null ? {} : { projectionRevision: row.projection_revision }),
+    ...(row.ownership_hash === null ? {} : { ownershipHash: row.ownership_hash }),
+    committedAt: row.committed_at,
+  }
+  const intent = parseJson<OntologyCommitWrite["intent"]>(row.intent)
+  const result = parseJson<OntologyCommitRecord["result"]>(row.result)
+  return { ...base, intent, result } as OntologyCommitRecord
+}
+
+export function outboxRecord(row: SqliteOntologyOutboxRow): OntologyOutboxRecord {
+  return {
+    envelope: parseJson<OntologyOutboxRecord["envelope"]>(row.envelope),
+    availableAt: row.available_at,
+    attempts: row.attempts,
+    leaseId: row.lease_id,
+    leaseExpiresAt: row.lease_expires_at,
+    publishedAt: row.published_at,
+    lastError: row.last_error,
+    createdAt: row.created_at,
+  }
+}
+
+export function assertProjectionExecution(
+  db: Database,
+  input: {
+    readonly projectId: string
+    readonly sourceId: string
+    readonly projectionRunId: string
+    readonly executionToken: string
+    readonly identity?: AssertSourceMaterializationExecutionInput["identity"]
+  }
+): void {
+  const run = db
+    .query(
+      `
+        SELECT projection_id, projection_kind, status, materialization_protocol, execution_token,
+          dataset_id, dataset_version_id, dataset_version_created_at,
+          ontology_revision, projection_revision, ownership_hash
+        FROM projection_runs
+        WHERE project_id = ? AND id = ?
+      `
+    )
+    .get(input.projectId, input.projectionRunId) as {
+    projection_id: string
+    projection_kind: string
+    status: string
+    materialization_protocol: string | null
+    execution_token: string | null
+    dataset_id: string
+    dataset_version_id: string
+    dataset_version_created_at: string | null
+    ontology_revision: string | null
+    projection_revision: string | null
+    ownership_hash: string | null
+  } | null
+  if (!run || run.status !== "running") {
+    throw new MaterializationConflictError(
+      "run-correlation",
+      `Projection run '${input.projectionRunId}' is missing or is not running.`
+    )
+  }
+  if (run.projection_id !== input.sourceId || run.materialization_protocol !== "replacement") {
+    throw new MaterializationConflictError(
+      "run-correlation",
+      `Projection run '${input.projectionRunId}' does not own replacement source '${input.sourceId}'.`
+    )
+  }
+  if (run.execution_token !== input.executionToken) {
+    throw new MaterializationConflictError(
+      "execution-lost",
+      `Projection run '${input.projectionRunId}' execution token is stale.`
+    )
+  }
+  const identity = input.identity
+  if (
+    identity &&
+    (run.projection_kind !== identity.projectionKind ||
+      run.materialization_protocol !== identity.protocol ||
+      run.dataset_id !== identity.datasetVersion.datasetId ||
+      run.dataset_version_id !== identity.datasetVersion.versionId ||
+      run.dataset_version_created_at !== identity.datasetVersion.createdAt ||
+      run.ontology_revision !== identity.ontologyRevision ||
+      run.projection_revision !== identity.projectionRevision ||
+      run.ownership_hash !== identity.ownershipHash)
+  ) {
+    throw new MaterializationConflictError(
+      "run-correlation",
+      `Projection run '${input.projectionRunId}' immutable source identity does not match.`
+    )
+  }
+}
+
+export function requireChanges(
+  changes: number,
+  kind: ConstructorParameters<typeof MaterializationConflictError>[0],
+  message: string
+): void {
+  if (changes !== 1) throw new MaterializationConflictError(kind, message)
+}
+
+export function isSqliteConstraintError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false
+  const details = error as Error & { readonly code?: unknown; readonly errno?: unknown }
+  if (typeof details.code === "string") {
+    return [
+      "SQLITE_CONSTRAINT_CHECK",
+      "SQLITE_CONSTRAINT_FOREIGNKEY",
+      "SQLITE_CONSTRAINT_NOTNULL",
+      "SQLITE_CONSTRAINT_PRIMARYKEY",
+      "SQLITE_CONSTRAINT_UNIQUE",
+    ].includes(details.code)
+  }
+  return details.errno === 19
+}