@sixb/sqlite 0.1.13 → 0.1.15

package/src/migrations/051-projection-run-supersession.sql

@@ -0,0 +1,126 @@
+-- Projection runs gain the terminal status `superseded`, and a running run may carry the failure
+-- of its last attempt in `error` while the queue retries it. SQLite cannot alter a CHECK
+-- constraint, so the table is rebuilt.
+CREATE TABLE projection_runs_v2 (
+  project_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  execution_id TEXT NOT NULL,
+  projection_id TEXT NOT NULL,
+  projection_kind TEXT NOT NULL CHECK (projection_kind IN ('object', 'link', 'telemetry')),
+  materialization_protocol TEXT NOT NULL CHECK (
+    materialization_protocol IN ('replacement', 'telemetry')
+  ),
+  dataset_id TEXT NOT NULL,
+  dataset_version_id TEXT NOT NULL,
+  dataset_version_created_at TEXT NOT NULL,
+  ontology_revision TEXT NOT NULL,
+  projection_revision TEXT NOT NULL,
+  ownership_hash TEXT NOT NULL,
+  object_type_id TEXT,
+  source_object_type_id TEXT,
+  target_object_type_id TEXT,
+  status TEXT NOT NULL CHECK (
+    status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled', 'superseded')
+  ),
+  queued_at TEXT NOT NULL,
+  started_at TEXT,
+  finished_at TEXT,
+  attempt INTEGER NOT NULL CHECK (attempt >= 0),
+  execution_token TEXT,
+  fixed_batch_size INTEGER CHECK (fixed_batch_size > 0),
+  next_batch_ordinal INTEGER CHECK (next_batch_ordinal >= 0),
+  next_row_offset INTEGER CHECK (next_row_offset >= 0),
+  input_exhausted INTEGER CHECK (input_exhausted IN (0, 1)),
+  missing_target_object_type_id TEXT,
+  missing_target_object_id TEXT,
+  missing_target_batch_ordinal INTEGER CHECK (
+    missing_target_batch_ordinal IS NULL OR missing_target_batch_ordinal >= 0
+  ),
+  missing_target_first_seen_at TEXT,
+  source_rows_read INTEGER NOT NULL DEFAULT 0 CHECK (source_rows_read >= 0),
+  source_rows_skipped INTEGER NOT NULL DEFAULT 0 CHECK (source_rows_skipped >= 0),
+  source_changes_read INTEGER CHECK (source_changes_read IS NULL OR source_changes_read >= 0),
+  error TEXT CHECK (error IS NULL OR json_valid(error)),
+  PRIMARY KEY (project_id, id),
+  UNIQUE (project_id, execution_id),
+  FOREIGN KEY (project_id, execution_id) REFERENCES executions (project_id, id) ON DELETE RESTRICT,
+  CHECK (source_rows_skipped <= source_rows_read),
+  CHECK ((status = 'running') = (execution_token IS NOT NULL)),
+  CHECK (
+    (status = 'queued' AND attempt = 0 AND started_at IS NULL AND finished_at IS NULL
+      AND error IS NULL)
+    OR (status = 'running' AND attempt >= 1 AND started_at IS NOT NULL AND finished_at IS NULL)
+    OR (
+      status IN ('succeeded', 'failed', 'cancelled', 'superseded')
+      AND finished_at IS NOT NULL
+      AND (
+        (json_extract(error, '$.code') = 'queue.enqueue_failed'
+          AND status = 'failed' AND attempt = 0 AND started_at IS NULL)
+        OR (COALESCE(json_extract(error, '$.code'), '') != 'queue.enqueue_failed'
+          AND attempt >= 1 AND started_at IS NOT NULL)
+      )
+    )
+  ),
+  CHECK (status != 'succeeded' OR error IS NULL),
+  -- A run is superseded instead of materialized, so it carries no failure, and only replacement
+  -- runs can be: telemetry appends readings rather than replace a source.
+  CHECK (status != 'superseded' OR (error IS NULL AND materialization_protocol = 'replacement')),
+  CHECK ((projection_kind = 'telemetry') = (materialization_protocol = 'telemetry')),
+  CHECK (
+    (projection_kind = 'link' AND object_type_id IS NULL
+      AND source_object_type_id IS NOT NULL AND target_object_type_id IS NOT NULL)
+    OR (projection_kind IN ('object', 'telemetry') AND object_type_id IS NOT NULL
+      AND source_object_type_id IS NULL AND target_object_type_id IS NULL)
+  ),
+  CHECK (
+    (materialization_protocol = 'replacement' AND fixed_batch_size IS NULL
+      AND next_batch_ordinal IS NULL AND next_row_offset IS NULL AND input_exhausted IS NULL)
+    OR (materialization_protocol = 'telemetry' AND fixed_batch_size IS NOT NULL
+      AND next_batch_ordinal IS NOT NULL AND next_row_offset IS NOT NULL
+      AND input_exhausted IS NOT NULL)
+  ),
+  CHECK (
+    (missing_target_object_type_id IS NULL AND missing_target_object_id IS NULL
+      AND missing_target_batch_ordinal IS NULL AND missing_target_first_seen_at IS NULL)
+    OR (projection_kind = 'telemetry' AND missing_target_object_type_id = object_type_id
+      AND missing_target_object_id IS NOT NULL
+      AND missing_target_batch_ordinal = next_batch_ordinal
+      AND missing_target_first_seen_at IS NOT NULL)
+  ),
+  CHECK (projection_kind != 'telemetry' OR status != 'succeeded' OR input_exhausted = 1)
+);
+
+INSERT INTO projection_runs_v2 (
+  project_id, id, execution_id, projection_id, projection_kind, materialization_protocol,
+  dataset_id, dataset_version_id, dataset_version_created_at, ontology_revision,
+  projection_revision, ownership_hash, object_type_id, source_object_type_id,
+  target_object_type_id, status, queued_at, started_at, finished_at, attempt, execution_token,
+  fixed_batch_size, next_batch_ordinal, next_row_offset, input_exhausted,
+  missing_target_object_type_id, missing_target_object_id, missing_target_batch_ordinal,
+  missing_target_first_seen_at, source_rows_read, source_rows_skipped, source_changes_read, error
+)
+SELECT
+  project_id, id, execution_id, projection_id, projection_kind, materialization_protocol,
+  dataset_id, dataset_version_id, dataset_version_created_at, ontology_revision,
+  projection_revision, ownership_hash, object_type_id, source_object_type_id,
+  target_object_type_id, status, queued_at, started_at, finished_at, attempt, execution_token,
+  fixed_batch_size, next_batch_ordinal, next_row_offset, input_exhausted,
+  missing_target_object_type_id, missing_target_object_id, missing_target_batch_ordinal,
+  missing_target_first_seen_at, source_rows_read, source_rows_skipped, source_changes_read, error
+FROM projection_runs;
+
+DROP TABLE projection_runs;
+ALTER TABLE projection_runs_v2 RENAME TO projection_runs;
+
+CREATE INDEX idx_projection_runs_project_started
+  ON projection_runs(project_id, COALESCE(started_at, queued_at) DESC);
+CREATE INDEX idx_projection_runs_project_projection_started
+  ON projection_runs(project_id, projection_id, COALESCE(started_at, queued_at) DESC);
+CREATE INDEX idx_projection_runs_project_dataset_started
+  ON projection_runs(project_id, dataset_id, COALESCE(started_at, queued_at) DESC);
+CREATE INDEX idx_projection_runs_project_version_started
+  ON projection_runs(project_id, dataset_version_id, COALESCE(started_at, queued_at) DESC);
+CREATE INDEX idx_projection_runs_project_status_started
+  ON projection_runs(project_id, status, COALESCE(started_at, queued_at) DESC);
+CREATE INDEX idx_projection_runs_project_object_type_started
+  ON projection_runs(project_id, object_type_id, COALESCE(started_at, queued_at) DESC);

package/src/migrations/052-compact-source-storage.sql

@@ -0,0 +1,181 @@
+-- Source versions get a compact surrogate key; roots and rows stop repeating the text triple.
+-- SQLite cannot add a key column to a table, so the three source tables are rebuilt. Index
+-- names are schema-wide: the old ones are dropped first to free them.
+DROP INDEX idx_ontology_sources_active;
+DROP INDEX idx_ontology_sources_run_candidate;
+DROP INDEX idx_ontology_sources_cleanup;
+DROP INDEX idx_ontology_sources_run;
+DROP INDEX idx_ontology_source_roots_active;
+DROP INDEX idx_ontology_source_roots_lookup;
+DROP INDEX idx_ontology_source_roots_cleanup;
+DROP INDEX idx_ontology_source_rows_root;
+DROP INDEX idx_ontology_source_rows_staging_ordinal;
+DROP INDEX idx_ontology_source_rows_entity_sort;
+DROP INDEX idx_ontology_source_rows_object;
+DROP INDEX idx_ontology_source_rows_link_source;
+DROP INDEX idx_ontology_source_rows_link_target;
+ALTER TABLE ontology_source_rows RENAME TO legacy_ontology_source_rows;
+ALTER TABLE ontology_source_roots RENAME TO legacy_ontology_source_roots;
+ALTER TABLE ontology_sources RENAME TO legacy_ontology_sources;
+
+CREATE TABLE ontology_sources (
+  version_id INTEGER PRIMARY KEY,
+  project_id TEXT NOT NULL,
+  source_id TEXT NOT NULL,
+  materialization_id TEXT NOT NULL,
+  projection_run_id TEXT NOT NULL,
+  projection_kind TEXT NOT NULL CHECK (projection_kind IN ('object', 'link')),
+  protocol TEXT NOT NULL CHECK (protocol = 'replacement'),
+  status TEXT NOT NULL CHECK (
+    status IN ('staging', 'ready', 'active', 'superseded', 'abandoned')
+  ),
+  execution_token TEXT,
+  dataset_id TEXT NOT NULL,
+  dataset_version_id TEXT NOT NULL,
+  dataset_version_created_at TEXT NOT NULL,
+  projection_revision TEXT NOT NULL,
+  ownership_hash TEXT NOT NULL,
+  ontology_revision TEXT NOT NULL,
+  root_count INTEGER CHECK (root_count IS NULL OR root_count >= 0),
+  assertion_count INTEGER CHECK (assertion_count IS NULL OR assertion_count >= 0),
+  created_at TEXT NOT NULL,
+  ready_at TEXT,
+  activated_at TEXT,
+  terminal_at TEXT,
+  last_commit_id TEXT,
+  updated_at TEXT NOT NULL,
+  base_materialization_id TEXT,
+  base_commit_id TEXT,
+  UNIQUE (project_id, source_id, materialization_id),
+  CHECK ((base_materialization_id IS NULL) = (base_commit_id IS NULL)),
+  CHECK ((root_count IS NULL) = (assertion_count IS NULL)),
+  CHECK ((root_count IS NULL) = (ready_at IS NULL)),
+  CHECK ((status IN ('staging', 'ready')) = (execution_token IS NOT NULL)),
+  CHECK (
+    (status = 'staging' AND ready_at IS NULL)
+    OR (status IN ('ready', 'active', 'superseded') AND ready_at IS NOT NULL)
+    OR status = 'abandoned'
+  ),
+  CHECK ((status IN ('active', 'superseded')) = (activated_at IS NOT NULL)),
+  CHECK ((status IN ('active', 'superseded')) = (last_commit_id IS NOT NULL)),
+  CHECK ((status IN ('superseded', 'abandoned')) = (terminal_at IS NOT NULL)),
+  CHECK (ready_at IS NULL OR created_at <= ready_at),
+  CHECK (activated_at IS NULL OR (ready_at IS NOT NULL AND ready_at <= activated_at)),
+  CHECK (terminal_at IS NULL OR created_at <= terminal_at),
+  CHECK (terminal_at IS NULL OR ready_at IS NULL OR ready_at <= terminal_at),
+  CHECK (terminal_at IS NULL OR activated_at IS NULL OR activated_at <= terminal_at),
+  CHECK (created_at <= updated_at)
+);
+INSERT INTO ontology_sources (
+  project_id, source_id, materialization_id, projection_run_id, projection_kind, protocol,
+  status, execution_token, dataset_id, dataset_version_id, dataset_version_created_at,
+  projection_revision, ownership_hash, ontology_revision, root_count, assertion_count,
+  created_at, ready_at, activated_at, terminal_at, last_commit_id, updated_at,
+  base_materialization_id, base_commit_id
+)
+SELECT project_id, source_id, materialization_id, projection_run_id, projection_kind, protocol,
+  status, execution_token, dataset_id, dataset_version_id, dataset_version_created_at,
+  projection_revision, ownership_hash, ontology_revision, root_count, assertion_count,
+  created_at, ready_at, activated_at, terminal_at, last_commit_id, updated_at,
+  base_materialization_id, base_commit_id
+FROM legacy_ontology_sources
+ORDER BY rowid;
+CREATE UNIQUE INDEX idx_ontology_sources_active
+  ON ontology_sources(project_id, source_id)
+  WHERE status = 'active';
+CREATE UNIQUE INDEX idx_ontology_sources_run_candidate
+  ON ontology_sources(project_id, projection_run_id)
+  WHERE status IN ('staging', 'ready');
+CREATE INDEX idx_ontology_sources_cleanup
+  ON ontology_sources(project_id, status, terminal_at, source_id, materialization_id);
+CREATE INDEX idx_ontology_sources_run
+  ON ontology_sources(project_id, projection_run_id);
+
+-- A root is live while its version is published (active or superseded), it is not retired and
+-- it is not a deletion. Activation therefore writes only the roots it replaces, never its own.
+-- `root_key` is the canonical key; SQLite's default BINARY collation orders it by its UTF-8 bytes.
+CREATE TABLE ontology_source_roots (
+  id INTEGER PRIMARY KEY,
+  version_id INTEGER NOT NULL REFERENCES ontology_sources (version_id) ON DELETE RESTRICT,
+  project_id TEXT NOT NULL,
+  root_key TEXT NOT NULL,
+  staging_ordinal INTEGER NOT NULL CHECK (staging_ordinal >= 0),
+  deleted INTEGER NOT NULL DEFAULT 0 CHECK (deleted IN (0, 1)),
+  retired_at TEXT,
+  CONSTRAINT ontology_source_roots_key UNIQUE (version_id, root_key),
+  CONSTRAINT ontology_source_roots_ordinal UNIQUE (version_id, staging_ordinal)
+);
+CREATE INDEX idx_ontology_source_roots_live
+  ON ontology_source_roots (project_id, root_key) WHERE retired_at IS NULL AND deleted = 0;
+CREATE INDEX idx_ontology_source_roots_retired
+  ON ontology_source_roots (project_id, retired_at) WHERE retired_at IS NOT NULL;
+
+-- A row's identity lives in its typed columns; `payload` is the assertion without kind and ref.
+CREATE TABLE ontology_source_rows (
+  root_id INTEGER NOT NULL REFERENCES ontology_source_roots (id) ON DELETE RESTRICT,
+  entity_kind TEXT NOT NULL CHECK (entity_kind IN ('object', 'link')),
+  object_type_id TEXT,
+  primary_id TEXT,
+  source_type_id TEXT,
+  source_primary_id TEXT,
+  link_id TEXT,
+  target_type_id TEXT,
+  target_primary_id TEXT,
+  payload TEXT CHECK (json_valid(payload)),
+  CHECK (
+    (
+      entity_kind = 'object'
+      AND object_type_id IS NOT NULL AND primary_id IS NOT NULL
+      AND source_type_id IS NULL AND source_primary_id IS NULL AND link_id IS NULL
+      AND target_type_id IS NULL AND target_primary_id IS NULL
+    )
+    OR
+    (
+      entity_kind = 'link'
+      AND object_type_id IS NULL AND primary_id IS NULL
+      AND source_type_id IS NOT NULL AND source_primary_id IS NOT NULL AND link_id IS NOT NULL
+      AND target_type_id IS NOT NULL AND target_primary_id IS NOT NULL
+    )
+  )
+);
+CREATE INDEX idx_ontology_source_rows_root ON ontology_source_rows (root_id);
+CREATE INDEX idx_ontology_source_rows_link_source
+  ON ontology_source_rows (source_type_id, source_primary_id, link_id)
+  WHERE entity_kind = 'link';
+CREATE INDEX idx_ontology_source_rows_link_target
+  ON ontology_source_rows (target_type_id, target_primary_id)
+  WHERE entity_kind = 'link';
+
+-- A published root that was not active is retired, so liveness stays exactly what it was.
+INSERT INTO ontology_source_roots (
+  version_id, project_id, root_key, staging_ordinal, deleted, retired_at
+)
+SELECT versions.version_id, roots.project_id, roots.root_key, roots.staging_ordinal,
+  roots.deleted,
+  CASE
+    WHEN roots.active = 1 THEN NULL
+    WHEN roots.retired_at IS NOT NULL THEN roots.retired_at
+    WHEN versions.status IN ('active', 'superseded') THEN versions.updated_at
+  END
+FROM legacy_ontology_source_roots AS roots
+JOIN ontology_sources AS versions USING (project_id, source_id, materialization_id)
+ORDER BY versions.version_id, roots.staging_ordinal;
+
+INSERT INTO ontology_source_rows (
+  root_id, entity_kind, object_type_id, primary_id,
+  source_type_id, source_primary_id, link_id, target_type_id, target_primary_id, payload
+)
+SELECT roots.id, rows.entity_kind, rows.object_type_id, rows.primary_id,
+  rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
+  rows.target_primary_id, NULLIF(json_remove(rows.assertion, '$.kind', '$.ref'), '{}')
+FROM legacy_ontology_source_rows AS rows
+JOIN ontology_sources AS versions USING (project_id, source_id, materialization_id)
+JOIN ontology_source_roots AS roots
+  ON roots.version_id = versions.version_id AND roots.root_key = rows.root_key
+ORDER BY roots.id;
+
+DROP TABLE legacy_ontology_source_rows;
+DROP TABLE legacy_ontology_source_roots;
+DROP TABLE legacy_ontology_sources;
+-- No ANALYZE: nothing refreshes SQLite statistics afterwards, and on a fresh install they would
+-- describe empty tables forever and steer the planner to partial indexes over primary keys.

package/src/migrations.ts

@@ -112,6 +112,15 @@ import ontologyCommitAttributionSql from "./migrations/047-ontology-commit-attri
   type: "text",
 }
 import nativeSessionsSql from "./migrations/049-native-sessions.sql" with { type: "text" }
+import directoryGroupMembershipsSql from "./migrations/050-directory-group-memberships.sql" with {
+  type: "text",
+}
+import projectionRunSupersessionSql from "./migrations/051-projection-run-supersession.sql" with {
+  type: "text",
+}
+import compactSourceStorageSql from "./migrations/052-compact-source-storage.sql" with {
+  type: "text",
+}
 
 const MIGRATIONS_TABLE_SQL = `
   CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -216,6 +225,27 @@ export const sqliteStorageMigrations = defineMigrations({
     sqliteSql("047-ontology-commit-attribution", ontologyCommitAttributionSql),
     sqliteSql("048-file-upload-sessions", fileUploadSessionsSql),
     sqliteSql("049-native-sessions", nativeSessionsSql),
+    sqliteSql("050-directory-group-memberships", directoryGroupMembershipsSql),
+    sqliteSql("051-projection-run-supersession", projectionRunSupersessionSql),
+    sqliteStep(
+      "052-compact-source-storage",
+      (db) => {
+        // SQLite has no procedural block to check the copy in SQL: count around it instead.
+        const before = sourceStorageCounts(db)
+        db.run(compactSourceStorageSql)
+        const after = sourceStorageCounts(db)
+        if (
+          after.versions !== before.versions ||
+          after.roots !== before.roots ||
+          after.rows !== before.rows
+        ) {
+          throw new Error(
+            "[SixbSqliteStorage] Source versions, roots or rows were lost while compacting source storage."
+          )
+        }
+      },
+      { checksum: checksum(compactSourceStorageSql) }
+    ),
   ],
 })
 
@@ -374,6 +404,18 @@ function rowToMigrationRecord(row: unknown): MigrationRecord {
   }
 }
 
+function sourceStorageCounts(db: Database): {
+  readonly versions: number
+  readonly roots: number
+  readonly rows: number
+} {
+  return db
+    .query(`SELECT (SELECT count(*) FROM ontology_sources) AS versions,
+      (SELECT count(*) FROM ontology_source_roots) AS roots,
+      (SELECT count(*) FROM ontology_source_rows) AS rows`)
+    .get() as { readonly versions: number; readonly roots: number; readonly rows: number }
+}
+
 function checksum(value: string): string {
   return createHash("sha256").update(value).digest("hex")
 }

package/src/ontology-storage/materialization-state.ts

@@ -43,8 +43,15 @@ import {
   parseJson,
   type SqliteOntologySourceAssertionRow,
   type SqliteStoredOverrideRow,
+  sourceAssertion,
 } from "./shared"
-import { activeSourceRows, replacementAssertionRows, replacementSourceRows } from "./source-roots"
+import {
+  activeSourceRows,
+  LIVE_SOURCE_ROWS_JOIN,
+  replacementAssertionRows,
+  replacementSourceRows,
+  SOURCE_ASSERTION_COLUMNS,
+} from "./source-roots"
 
 export const SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work"
 export const SQLITE_REPLACEMENT_WORK_TABLE = "ontology_replacement_work"
@@ -438,19 +445,13 @@ export class SqliteMaterializationStateReader {
              json_extract(value, '$.linkId') AS link_id
            FROM json_each(?)
          )
-         SELECT rows.* FROM requested
+         SELECT ${SOURCE_ASSERTION_COLUMNS} FROM requested
          CROSS JOIN ontology_source_rows AS rows
-           ON rows.project_id = ?
-          AND rows.entity_kind = 'link'
+           ON rows.entity_kind = 'link'
           AND rows.source_type_id = requested.source_type_id
           AND rows.source_primary_id = requested.source_primary_id
           AND rows.link_id = requested.link_id
-         JOIN ontology_source_roots AS sources
-           ON sources.project_id = rows.project_id
-          AND sources.source_id = rows.source_id
-          AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-         WHERE sources.active = 1`
+         ${LIVE_SOURCE_ROWS_JOIN}`
       )
       .all(requestedJson, this.projectId) as SqliteOntologySourceAssertionRow[]
     const slotOverrideRows = this.db
@@ -570,12 +571,8 @@ export class SqliteMaterializationStateReader {
               SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
                 rows.target_type_id, rows.target_primary_id
               FROM ontology_source_rows AS rows
-              JOIN ontology_source_roots AS sources
-                ON sources.project_id = rows.project_id
-               AND sources.source_id = rows.source_id
-               AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-              WHERE rows.project_id = ? AND rows.entity_kind = 'link' AND sources.active = 1
+              ${LIVE_SOURCE_ROWS_JOIN}
+              WHERE rows.entity_kind = 'link'
             ), selected AS (
               SELECT DISTINCT links.*,
                 ${linkSortExpression("links")} AS sort_key
@@ -752,7 +749,7 @@ export class SqliteMaterializationStateReader {
   }
 
   private prepareReplacementObjects(input: ReplacementIdentityInput): void {
-    const rows = replacementSourceRows({
+    const rows = replacementSourceRows(this.db, {
       projectId: this.projectId,
       sourceId: input.sourceId,
       materializationId: input.candidateMaterializationId,
@@ -765,7 +762,7 @@ export class SqliteMaterializationStateReader {
             session_id, entity_kind, identity_key, sort_key, diff_required
           )
           SELECT ?, 'object', json_array(object_type_id, primary_id),
-            MIN(entity_sort_key), 1
+            ${objectSortExpression()}, 1
           FROM (${rows.sql}) AS rows
           WHERE entity_kind = 'object'
           GROUP BY object_type_id, primary_id
@@ -775,7 +772,7 @@ export class SqliteMaterializationStateReader {
   }
 
   private prepareReplacementLinks(input: ReplacementIdentityInput): void {
-    const rows = replacementSourceRows({
+    const rows = replacementSourceRows(this.db, {
       projectId: this.projectId,
       sourceId: input.sourceId,
       materializationId: input.candidateMaterializationId,
@@ -834,31 +831,21 @@ export class SqliteMaterializationStateReader {
             UNION
             SELECT rows.source_type_id, rows.source_primary_id AS source_id,
               rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
-            FROM ontology_source_rows AS rows
-            JOIN ontology_source_roots AS sources
-              ON sources.project_id = rows.project_id
-             AND sources.source_id = rows.source_id
-             AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-            JOIN incident_objects
-              ON incident_objects.object_type_id = rows.source_type_id
-             AND incident_objects.primary_id = rows.source_primary_id
-            WHERE rows.project_id = ? AND rows.entity_kind = 'link'
-              AND sources.active = 1
+            FROM incident_objects
+            CROSS JOIN ontology_source_rows AS rows
+              ON rows.entity_kind = 'link'
+             AND rows.source_type_id = incident_objects.object_type_id
+             AND rows.source_primary_id = incident_objects.primary_id
+            ${LIVE_SOURCE_ROWS_JOIN}
             UNION
             SELECT rows.source_type_id, rows.source_primary_id AS source_id,
               rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
-            FROM ontology_source_rows AS rows
-            JOIN ontology_source_roots AS sources
-              ON sources.project_id = rows.project_id
-             AND sources.source_id = rows.source_id
-             AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-            JOIN incident_objects
-              ON incident_objects.object_type_id = rows.target_type_id
-             AND incident_objects.primary_id = rows.target_primary_id
-            WHERE rows.project_id = ? AND rows.entity_kind = 'link'
-              AND sources.active = 1
+            FROM incident_objects
+            CROSS JOIN ontology_source_rows AS rows
+              ON rows.entity_kind = 'link'
+             AND rows.target_type_id = incident_objects.object_type_id
+             AND rows.target_primary_id = incident_objects.primary_id
+            ${LIVE_SOURCE_ROWS_JOIN}
           ), diff_links AS (
             SELECT * FROM replacement_links
             UNION SELECT * FROM incident_links
@@ -1062,9 +1049,7 @@ function storedSource(row: SqliteOntologySourceAssertionRow): StoredSourceAssert
   return {
     source: { projectionId: row.source_id },
     materializationId: row.materialization_id,
-    root: parseJson<StoredSourceAssertion["root"]>(row.root),
-    assertion: parseJson<StoredSourceAssertion["assertion"]>(row.assertion),
-    stagingOrdinal: row.staging_ordinal,
+    ...sourceAssertion(row),
   } as StoredSourceAssertion
 }
 
@@ -1130,6 +1115,11 @@ function storedPoint(row: TelemetryRow): StoredTelemetryPoint {
   }
 }
 
+function objectSortExpression(alias?: string): string {
+  const prefix = alias ? `${alias}.` : ""
+  return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}primary_id) AS BLOB)))`
+}
+
 export function linkSortExpression(alias?: string): string {
   const prefix = alias ? `${alias}.` : ""
   return `LOWER(HEX(CAST(json_array(${prefix}source_type_id, ${prefix}source_id, ${prefix}link_id, ${prefix}target_type_id, ${prefix}target_id) AS BLOB)))`

package/src/ontology-storage/shared.ts

@@ -1,7 +1,10 @@
 import type { Database } from "bun:sqlite"
 import { parseSixbFailure } from "@sixb/core/internal/errors"
-import type { ProjectionEntityRef } from "@sixb/core/internal/materialization"
 import { MaterializationConflictError } from "@sixb/core/internal/materialization"
+import {
+  sourceAssertionFromColumns,
+  sourceEntityFromKey,
+} from "@sixb/core/internal/ontology-storage-provider"
 import type {
   AssertSourceMaterializationExecutionInput,
   OntologyCommitRecord,
@@ -48,6 +51,7 @@ export interface SqliteOntologyCommitRow {
 }
 
 export interface SqliteOntologySourceRow {
+  readonly version_id: number
   readonly project_id: string
   readonly source_id: string
   readonly materialization_id: string
@@ -74,19 +78,21 @@ export interface SqliteOntologySourceRow {
   readonly updated_at: string
 }
 
+/** An assertion with its root and version, as `SOURCE_ASSERTION_COLUMNS` selects it. */
 export interface SqliteOntologySourceAssertionRow {
-  readonly project_id: string
   readonly source_id: string
   readonly materialization_id: string
-  readonly entity_kind: "object" | "link"
-  readonly entity_key: string
-  readonly entity_sort_key: string
-  readonly root_kind: "object" | "link"
   readonly root_key: string
-  readonly root_sort_key: string
   readonly staging_ordinal: number
-  readonly root: string
-  readonly assertion: string
+  readonly entity_kind: "object" | "link"
+  readonly object_type_id: string | null
+  readonly primary_id: string | null
+  readonly source_type_id: string | null
+  readonly source_primary_id: string | null
+  readonly link_id: string | null
+  readonly target_type_id: string | null
+  readonly target_primary_id: string | null
+  readonly payload: string | null
 }
 
 export interface SqliteStoredOverrideRow {
@@ -149,8 +155,11 @@ export function sourceRecord(row: SqliteOntologySourceRow): OntologySourceRecord
 
 export function sourceAssertion(row: SqliteOntologySourceAssertionRow): StageSourceAssertion {
   return {
-    root: parseJson<ProjectionEntityRef>(row.root),
-    assertion: parseJson<StageSourceAssertion["assertion"]>(row.assertion),
+    root: sourceEntityFromKey(row.root_key),
+    assertion: sourceAssertionFromColumns(
+      row,
+      row.payload === null ? null : parseJson<unknown>(row.payload)
+    ),
     stagingOrdinal: row.staging_ordinal,
   }
 }