@sixb/sqlite 0.0.1

package/src/ontology-storage/commits.ts

@@ -0,0 +1,127 @@
+import type { Database } from "bun:sqlite"
+import type {
+  GetOntologyCommitByIdempotencyKeyInput,
+  GetOntologyCommitByIdInput,
+  GetOntologyCommitByOriginInput,
+  ListOntologyCommitsInput,
+  ListOntologyCommitsResult,
+  OntologyCommitRecord,
+  OntologyCommitStorage,
+} from "@sixb/core/storage"
+import {
+  assertNonblank,
+  assertNonnegativeInteger,
+  commitRecord,
+  originWhere,
+  type SqliteOntologyCommitRow,
+  type SqliteRootOperation,
+} from "./shared"
+
+export class SqliteOntologyCommitStorage implements OntologyCommitStorage {
+  constructor(
+    private readonly db: Database,
+    private readonly runRootOperation: SqliteRootOperation
+  ) {}
+
+  async getByIdempotencyKey(
+    input: GetOntologyCommitByIdempotencyKeyInput
+  ): Promise<OntologyCommitRecord | null> {
+    return this.runRootOperation(() => {
+      assertNonblank(input.projectId, "Ontology commit project id")
+      assertNonblank(input.idempotencyKey, "Ontology commit idempotency key")
+      return this.readOne(
+        "project_id = ? AND idempotency_key = ?",
+        input.projectId,
+        input.idempotencyKey
+      )
+    })
+  }
+
+  async getById(input: GetOntologyCommitByIdInput): Promise<OntologyCommitRecord | null> {
+    return this.runRootOperation(() => {
+      assertNonblank(input.projectId, "Ontology commit project id")
+      assertNonblank(input.id, "Ontology commit id")
+      return this.readOne("project_id = ? AND id = ?", input.projectId, input.id)
+    })
+  }
+
+  async getByOrigin(input: GetOntologyCommitByOriginInput): Promise<OntologyCommitRecord | null> {
+    return this.runRootOperation(() => {
+      assertNonblank(input.projectId, "Ontology commit project id")
+      const origin = originWhere(input.origin)
+      assertNonblank(origin.runId, "Ontology commit origin run id")
+      if (origin.batchOrdinal !== null) {
+        assertNonnegativeInteger(origin.batchOrdinal, "Ontology telemetry commit batch ordinal")
+      }
+      return origin.batchOrdinal === null
+        ? this.readOne(
+            "project_id = ? AND origin_kind = ? AND origin_run_id = ? AND origin_batch_ordinal IS NULL",
+            input.projectId,
+            origin.kind,
+            origin.runId
+          )
+        : this.readOne(
+            "project_id = ? AND origin_kind = ? AND origin_run_id = ? AND origin_batch_ordinal = ?",
+            input.projectId,
+            origin.kind,
+            origin.runId,
+            origin.batchOrdinal
+          )
+    })
+  }
+
+  async list(input: ListOntologyCommitsInput): Promise<ListOntologyCommitsResult> {
+    return this.runRootOperation(() => {
+      assertNonblank(input.projectId, "Ontology commit project id")
+      if (input.run) assertNonblank(input.run.id, "Ontology commit run id")
+      const offset = input.offset ?? 0
+      const limit = input.limit ?? -1
+      assertNonnegativeInteger(offset, "Ontology commit list offset")
+      if (limit !== -1) assertNonnegativeInteger(limit, "Ontology commit list limit")
+      const filters = ["project_id = ?"]
+      const values: (string | number)[] = [input.projectId]
+      if (input.run?.kind === "action") {
+        filters.push("origin_kind = 'action' AND origin_run_id = ?")
+        values.push(input.run.id)
+      } else if (input.run?.kind === "projection") {
+        filters.push("origin_run_id = ? AND origin_kind IN ('projection', 'telemetry')")
+        values.push(input.run.id)
+      }
+      const where = filters.join(" AND ")
+      const totalRow = this.db
+        .query(`SELECT COUNT(*) AS total FROM ontology_commits WHERE ${where}`)
+        .get(...values) as { readonly total: number }
+      const direction = input.order === "desc" ? "DESC" : "ASC"
+      const order =
+        input.run?.kind === "projection"
+          ? `CASE WHEN origin_kind = 'telemetry' THEN origin_batch_ordinal ELSE -1 END ${direction}, committed_at ${direction}, id ${direction}`
+          : `committed_at ${direction}, id ${direction}`
+      const rows = this.db
+        .query(
+          `
+            SELECT * FROM ontology_commits
+            WHERE ${where}
+            ORDER BY ${order}
+            LIMIT ? OFFSET ?
+          `
+        )
+        .all(...values, limit, offset) as SqliteOntologyCommitRow[]
+      const commits = rows.map(commitRecord)
+      return {
+        commits,
+        total: totalRow.total,
+        hasMore: offset + commits.length < totalRow.total,
+      }
+    })
+  }
+
+  private readOne(
+    where: string,
+    ...values: readonly (string | number)[]
+  ): OntologyCommitRecord | null {
+    const row = this.db
+      .query(`SELECT * FROM ontology_commits WHERE ${where}`)
+      .get(...values) as SqliteOntologyCommitRow | null
+    return row ? commitRecord(row) : null
+  }
+}

package/src/ontology-storage/index.ts

@@ -0,0 +1,35 @@
+import type { Database } from "bun:sqlite"
+import type { OntologyStorage } from "@sixb/core/storage"
+import { SqliteOntologyCommitStorage } from "./commits"
+import type { SqliteOntologyTransactionContext } from "./materialization-session"
+import { SqliteOntologyMaterializationStorage } from "./materializations"
+import { SqliteOntologyOutboxStorage } from "./outbox"
+import type { SqliteRootOperation } from "./shared"
+import { SqliteOntologySourceStorage } from "./sources"
+
+export class SqliteOntologyStorage implements OntologyStorage {
+  readonly commits: SqliteOntologyCommitStorage
+  readonly sources: SqliteOntologySourceStorage
+  readonly materializations: SqliteOntologyMaterializationStorage
+  readonly outbox: SqliteOntologyOutboxStorage
+
+  constructor(input: {
+    readonly db: Database
+    readonly runRootOperation: SqliteRootOperation
+    readonly transactionContext: SqliteOntologyTransactionContext | null
+  }) {
+    this.commits = new SqliteOntologyCommitStorage(input.db, input.runRootOperation)
+    this.sources = new SqliteOntologySourceStorage(input.db, input.runRootOperation)
+    this.materializations = new SqliteOntologyMaterializationStorage(
+      input.db,
+      input.transactionContext
+    )
+    this.outbox = new SqliteOntologyOutboxStorage(input.db, input.runRootOperation)
+  }
+
+  deactivateSessions(): void {
+    this.materializations.deactivateSessions()
+  }
+}
+
+export type { SqliteOntologyTransactionContext } from "./materialization-session"

package/src/ontology-storage/materialization-session.ts

@@ -0,0 +1,393 @@
+import type { Database } from "bun:sqlite"
+import {
+  linkRefKey,
+  MaterializationConflictError,
+  objectRefKey,
+} from "@sixb/core/internal/materialization"
+import {
+  correlateMaterializationChunk,
+  duplicateMaterializationWork as duplicateWork,
+  invalidCorrelation,
+  materializationPlanItems,
+  ProviderMaterializationSessionState,
+  type ProviderMaterializationTransactionLifecycle,
+  prepareMaterializationWork,
+} from "@sixb/core/internal/ontology-storage-provider"
+import type {
+  MaterializationEventWorkRecord,
+  MaterializationPlanChunk,
+  MaterializationPlanHeader,
+  MaterializationPlanWorkItem,
+  MaterializationSession,
+  MaterializationWorkPage,
+  MaterializationWorkRecord,
+  StageMaterializationWorkInput,
+  StreamMaterializationWorkInput,
+} from "@sixb/core/storage"
+import {
+  type ReplacementIdentity,
+  SQLITE_MATERIALIZATION_WORK_TABLE,
+  SQLITE_REPLACEMENT_WORK_TABLE,
+} from "./materialization-state"
+import { canonicalJson, isSqliteConstraintError, parseJson } from "./shared"
+
+export interface SqliteOntologyTransactionContext {
+  readonly id: object
+  readonly materializations: ProviderMaterializationTransactionLifecycle
+  active: boolean
+}
+
+export class SqliteMaterializationSessionState extends ProviderMaterializationSessionState {}
+
+export class SqliteMaterializationSessions {
+  private readonly sessions = new WeakMap<object, SqliteMaterializationSessionState>()
+  private readonly live = new Set<SqliteMaterializationSessionState>()
+
+  constructor(
+    private readonly db: Database,
+    private readonly context: SqliteOntologyTransactionContext | null
+  ) {}
+
+  create(header: MaterializationPlanHeader): SqliteMaterializationSessionState {
+    if (!this.context?.active) {
+      throw new MaterializationConflictError(
+        "effective-state",
+        "Materialization sessions require an active storage transaction."
+      )
+    }
+    this.ensureTables()
+    const session = new SqliteMaterializationSessionState(structuredClone(header), this.context.id)
+    this.sessions.set(session.providerToken, session)
+    this.live.add(session)
+    this.context.materializations.register(session.providerToken)
+    return session
+  }
+
+  require(session: MaterializationSession): SqliteMaterializationSessionState {
+    const value = this.sessions.get(session.providerToken)
+    if (
+      !value ||
+      !value.active ||
+      !this.context?.active ||
+      value.transactionId !== this.context.id
+    ) {
+      throw new MaterializationConflictError(
+        "effective-state",
+        "Materialization session is inactive."
+      )
+    }
+    return value
+  }
+
+  release(session: SqliteMaterializationSessionState): void {
+    if (!session.active) return
+    this.deleteWork(session.id)
+    session.active = false
+    session.replacement = null
+    this.live.delete(session)
+    this.context?.materializations.complete(session.providerToken)
+  }
+
+  deactivateAll(): void {
+    for (const session of this.live) this.release(session)
+  }
+
+  stage(input: StageMaterializationWorkInput): void {
+    const session = this.require(input.session)
+    const records = prepareMaterializationWork(session, input)
+    const insert = this.db.query(
+      `
+        INSERT INTO ${SQLITE_MATERIALIZATION_WORK_TABLE} (
+          session_id, record_key, unique_key, kind, lane,
+          major_order, minor_order, sort_one, sort_two, payload
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, json(?))
+      `
+    )
+    try {
+      for (const { record, uniqueKey, columns } of records) {
+        insert.run(
+          session.id,
+          record.recordKey,
+          uniqueKey,
+          record.kind,
+          columns.lane,
+          columns.majorOrder,
+          columns.minorOrder,
+          columns.sortOne,
+          columns.sortTwo,
+          canonicalJson(record)
+        )
+      }
+    } catch (error) {
+      if (isSqliteConstraintError(error)) {
+        throw duplicateWork(input.records[0]?.recordKey ?? "unknown")
+      }
+      throw error
+    }
+  }
+
+  async *stream(input: StreamMaterializationWorkInput): AsyncIterable<MaterializationWorkPage> {
+    const session = this.require(input.session)
+    const stream = session.workStreams[input.order]
+    if (stream.started) {
+      throw new MaterializationConflictError(
+        "effective-state",
+        `Materialization ${input.order} work may only be streamed once per session.`
+      )
+    }
+    stream.started = true
+    session.workSealed = true
+    let offset = 0
+    while (true) {
+      this.require(input.session)
+      const rows = this.db
+        .query(
+          `
+            SELECT payload
+            FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
+            WHERE session_id = ? AND lane = ?
+            ORDER BY major_order, minor_order, sort_one, sort_two, record_key
+            LIMIT ? OFFSET ?
+          `
+        )
+        .all(session.id, input.order, input.pageRows, offset) as { readonly payload: string }[]
+      if (rows.length === 0) break
+      const records = rows.map((row) =>
+        parseJson<MaterializationWorkRecord>(row.payload)
+      ) as MaterializationWorkPage["records"]
+      offset += records.length
+      stream.emittedCount = offset
+      yield { records }
+    }
+    this.require(input.session)
+    stream.completed = true
+  }
+
+  readObjectExistence(
+    session: SqliteMaterializationSessionState,
+    refs: readonly { readonly objectTypeId: string; readonly primaryId: string }[]
+  ): readonly {
+    readonly ref: { readonly objectTypeId: string; readonly primaryId: string }
+    readonly exists: boolean
+  }[] {
+    const lookup = this.db.query(
+      `
+        SELECT payload FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
+        WHERE session_id = ? AND unique_key = ? AND kind = 'object-existence'
+      `
+    )
+    return refs.flatMap((ref) => {
+      const key = `object-existence:${JSON.stringify([ref.objectTypeId, ref.primaryId])}`
+      const row = lookup.get(session.id, key) as { readonly payload: string } | null
+      if (!row) return []
+      const record = parseJson<
+        Extract<MaterializationWorkRecord, { readonly kind: "object-existence" }>
+      >(row.payload)
+      return [{ ref: record.ref, exists: record.exists }]
+    })
+  }
+
+  assertChunkSequence(
+    session: SqliteMaterializationSessionState,
+    chunk: MaterializationPlanChunk
+  ): void {
+    const items = materializationPlanItems(chunk)
+    const progress = correlateMaterializationChunk(
+      session,
+      chunk,
+      items,
+      this.planItems(session, session.appliedPlanCount, items.length),
+      this.eventRecords(session, session.appliedOutboxCount, chunk.outbox.length)
+    )
+    session.appliedPlanCount = progress.appliedPlanCount
+    session.appliedOutboxCount = progress.appliedOutboxCount
+  }
+
+  recordReplacementIdentities(
+    session: SqliteMaterializationSessionState,
+    identities: readonly ReplacementIdentity[]
+  ): void {
+    const upsert = this.db.query(
+      `
+        INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
+          session_id, entity_kind, identity_key, diff_required
+        ) VALUES (?, ?, ?, ?)
+        ON CONFLICT(session_id, entity_kind, identity_key) DO UPDATE SET
+          diff_required = MAX(diff_required, excluded.diff_required)
+      `
+    )
+    for (const identity of identities) {
+      upsert.run(
+        session.id,
+        identity.kind,
+        identity.kind === "object" ? objectRefKey(identity.ref) : linkRefKey(identity.ref),
+        identity.diffRequired ? 1 : 0
+      )
+    }
+  }
+
+  count(session: SqliteMaterializationSessionState, kind: string): number {
+    const row = this.db
+      .query(
+        `SELECT COUNT(*) AS count FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
+         WHERE session_id = ? AND kind = ?`
+      )
+      .get(session.id, kind) as { readonly count: number }
+    return row.count
+  }
+
+  laneCounts(session: SqliteMaterializationSessionState): {
+    readonly apply: number
+    readonly cardinality: number
+    readonly event: number
+  } {
+    const rows = this.db
+      .query(
+        `SELECT lane, COUNT(*) AS count FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
+         WHERE session_id = ? AND lane <> 'none'
+         GROUP BY lane`
+      )
+      .all(session.id) as {
+      readonly lane: "apply" | "cardinality" | "event"
+      readonly count: number
+    }[]
+    const counts = { apply: 0, cardinality: 0, event: 0 }
+    for (const row of rows) counts[row.lane] = row.count
+    return counts
+  }
+
+  records(session: SqliteMaterializationSessionState, kind?: string): MaterializationWorkRecord[] {
+    const rows = this.db
+      .query(
+        `SELECT payload FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
+         WHERE session_id = ? ${kind ? "AND kind = ?" : ""}`
+      )
+      .all(...(kind ? [session.id, kind] : [session.id])) as { readonly payload: string }[]
+    return rows.map((row) => parseJson<MaterializationWorkRecord>(row.payload))
+  }
+
+  assertClassificationCoverage(session: SqliteMaterializationSessionState): void {
+    if (!session.replacement) return
+    const expectedObject = session.replacement.projectionKind === "object" ? 1 : 0
+    const mismatch = this.db
+      .query(
+        `
+          WITH expected AS (
+            SELECT entity_kind, identity_key
+            FROM ${SQLITE_REPLACEMENT_WORK_TABLE}
+            WHERE session_id = ? AND diff_required = 1
+              AND (entity_kind = 'link' OR ? = 1)
+          ), actual AS (
+            SELECT json_extract(payload, '$.entityKind') AS entity_kind,
+              json_extract(payload, '$.identityKey') AS identity_key
+            FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
+            WHERE session_id = ? AND kind = 'classification'
+              AND json_extract(payload, '$.entityKind') IN ('object', 'link')
+          ), differences AS (
+            SELECT * FROM expected EXCEPT SELECT * FROM actual
+            UNION ALL
+            SELECT * FROM actual EXCEPT SELECT * FROM expected
+          )
+          SELECT 1 FROM differences LIMIT 1
+        `
+      )
+      .get(session.id, expectedObject, session.id)
+    const pointClassifications = this.db
+      .query(
+        `
+          SELECT 1 FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
+          WHERE session_id = ? AND kind = 'classification'
+            AND json_extract(payload, '$.entityKind') = 'point'
+          LIMIT 1
+        `
+      )
+      .get(session.id)
+    if (mismatch || pointClassifications) {
+      invalidCorrelation(
+        "Projection replacement classification coverage does not match its streamed state."
+      )
+    }
+  }
+
+  private planItems(
+    session: SqliteMaterializationSessionState,
+    offset: number,
+    limit: number
+  ): MaterializationPlanWorkItem[] {
+    if (limit === 0) return []
+    const rows = this.db
+      .query(
+        `
+          SELECT payload FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
+          WHERE session_id = ? AND lane = 'apply'
+          ORDER BY major_order, minor_order, sort_one, sort_two, record_key
+          LIMIT ? OFFSET ?
+        `
+      )
+      .all(session.id, limit, offset) as { readonly payload: string }[]
+    return rows.map((row) => {
+      const record = parseJson<Extract<MaterializationWorkRecord, { readonly kind: "plan" }>>(
+        row.payload
+      )
+      return record.item
+    })
+  }
+
+  private eventRecords(
+    session: SqliteMaterializationSessionState,
+    offset: number,
+    limit: number
+  ): MaterializationEventWorkRecord[] {
+    if (limit === 0) return []
+    const rows = this.db
+      .query(
+        `
+          SELECT payload FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
+          WHERE session_id = ? AND lane = 'event'
+          ORDER BY major_order, minor_order, sort_one, sort_two, record_key
+          LIMIT ? OFFSET ?
+        `
+      )
+      .all(session.id, limit, offset) as { readonly payload: string }[]
+    return rows.map((row) => parseJson<MaterializationEventWorkRecord>(row.payload))
+  }
+
+  private ensureTables(): void {
+    this.db.run(`
+      CREATE TEMP TABLE IF NOT EXISTS ${SQLITE_MATERIALIZATION_WORK_TABLE} (
+        session_id TEXT NOT NULL,
+        record_key TEXT NOT NULL,
+        unique_key TEXT NOT NULL,
+        kind TEXT NOT NULL,
+        lane TEXT NOT NULL,
+        major_order INTEGER NOT NULL,
+        minor_order INTEGER NOT NULL,
+        sort_one TEXT NOT NULL,
+        sort_two TEXT NOT NULL,
+        payload TEXT NOT NULL CHECK (json_valid(payload)),
+        PRIMARY KEY (session_id, record_key),
+        UNIQUE (session_id, unique_key)
+      );
+      CREATE INDEX IF NOT EXISTS idx_ontology_materialization_work_lane
+        ON ${SQLITE_MATERIALIZATION_WORK_TABLE}(
+          session_id, lane, major_order, minor_order, sort_one, sort_two, record_key
+        );
+      CREATE TEMP TABLE IF NOT EXISTS ${SQLITE_REPLACEMENT_WORK_TABLE} (
+        session_id TEXT NOT NULL,
+        entity_kind TEXT NOT NULL,
+        identity_key TEXT NOT NULL,
+        diff_required INTEGER NOT NULL CHECK (diff_required IN (0, 1)),
+        PRIMARY KEY (session_id, entity_kind, identity_key)
+      );
+    `)
+  }
+
+  private deleteWork(sessionId: string): void {
+    this.db
+      .query(`DELETE FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} WHERE session_id = ?`)
+      .run(sessionId)
+    this.db
+      .query(`DELETE FROM ${SQLITE_REPLACEMENT_WORK_TABLE} WHERE session_id = ?`)
+      .run(sessionId)
+  }
+}