@sixb/sqlite 0.1.14 → 0.1.16

package/src/migrations/052-compact-source-storage.sql

@@ -0,0 +1,181 @@
+-- Source versions get a compact surrogate key; roots and rows stop repeating the text triple.
+-- SQLite cannot add a key column to a table, so the three source tables are rebuilt. Index
+-- names are schema-wide: the old ones are dropped first to free them.
+DROP INDEX idx_ontology_sources_active;
+DROP INDEX idx_ontology_sources_run_candidate;
+DROP INDEX idx_ontology_sources_cleanup;
+DROP INDEX idx_ontology_sources_run;
+DROP INDEX idx_ontology_source_roots_active;
+DROP INDEX idx_ontology_source_roots_lookup;
+DROP INDEX idx_ontology_source_roots_cleanup;
+DROP INDEX idx_ontology_source_rows_root;
+DROP INDEX idx_ontology_source_rows_staging_ordinal;
+DROP INDEX idx_ontology_source_rows_entity_sort;
+DROP INDEX idx_ontology_source_rows_object;
+DROP INDEX idx_ontology_source_rows_link_source;
+DROP INDEX idx_ontology_source_rows_link_target;
+ALTER TABLE ontology_source_rows RENAME TO legacy_ontology_source_rows;
+ALTER TABLE ontology_source_roots RENAME TO legacy_ontology_source_roots;
+ALTER TABLE ontology_sources RENAME TO legacy_ontology_sources;
+
+CREATE TABLE ontology_sources (
+  version_id INTEGER PRIMARY KEY,
+  project_id TEXT NOT NULL,
+  source_id TEXT NOT NULL,
+  materialization_id TEXT NOT NULL,
+  projection_run_id TEXT NOT NULL,
+  projection_kind TEXT NOT NULL CHECK (projection_kind IN ('object', 'link')),
+  protocol TEXT NOT NULL CHECK (protocol = 'replacement'),
+  status TEXT NOT NULL CHECK (
+    status IN ('staging', 'ready', 'active', 'superseded', 'abandoned')
+  ),
+  execution_token TEXT,
+  dataset_id TEXT NOT NULL,
+  dataset_version_id TEXT NOT NULL,
+  dataset_version_created_at TEXT NOT NULL,
+  projection_revision TEXT NOT NULL,
+  ownership_hash TEXT NOT NULL,
+  ontology_revision TEXT NOT NULL,
+  root_count INTEGER CHECK (root_count IS NULL OR root_count >= 0),
+  assertion_count INTEGER CHECK (assertion_count IS NULL OR assertion_count >= 0),
+  created_at TEXT NOT NULL,
+  ready_at TEXT,
+  activated_at TEXT,
+  terminal_at TEXT,
+  last_commit_id TEXT,
+  updated_at TEXT NOT NULL,
+  base_materialization_id TEXT,
+  base_commit_id TEXT,
+  UNIQUE (project_id, source_id, materialization_id),
+  CHECK ((base_materialization_id IS NULL) = (base_commit_id IS NULL)),
+  CHECK ((root_count IS NULL) = (assertion_count IS NULL)),
+  CHECK ((root_count IS NULL) = (ready_at IS NULL)),
+  CHECK ((status IN ('staging', 'ready')) = (execution_token IS NOT NULL)),
+  CHECK (
+    (status = 'staging' AND ready_at IS NULL)
+    OR (status IN ('ready', 'active', 'superseded') AND ready_at IS NOT NULL)
+    OR status = 'abandoned'
+  ),
+  CHECK ((status IN ('active', 'superseded')) = (activated_at IS NOT NULL)),
+  CHECK ((status IN ('active', 'superseded')) = (last_commit_id IS NOT NULL)),
+  CHECK ((status IN ('superseded', 'abandoned')) = (terminal_at IS NOT NULL)),
+  CHECK (ready_at IS NULL OR created_at <= ready_at),
+  CHECK (activated_at IS NULL OR (ready_at IS NOT NULL AND ready_at <= activated_at)),
+  CHECK (terminal_at IS NULL OR created_at <= terminal_at),
+  CHECK (terminal_at IS NULL OR ready_at IS NULL OR ready_at <= terminal_at),
+  CHECK (terminal_at IS NULL OR activated_at IS NULL OR activated_at <= terminal_at),
+  CHECK (created_at <= updated_at)
+);
+INSERT INTO ontology_sources (
+  project_id, source_id, materialization_id, projection_run_id, projection_kind, protocol,
+  status, execution_token, dataset_id, dataset_version_id, dataset_version_created_at,
+  projection_revision, ownership_hash, ontology_revision, root_count, assertion_count,
+  created_at, ready_at, activated_at, terminal_at, last_commit_id, updated_at,
+  base_materialization_id, base_commit_id
+)
+SELECT project_id, source_id, materialization_id, projection_run_id, projection_kind, protocol,
+  status, execution_token, dataset_id, dataset_version_id, dataset_version_created_at,
+  projection_revision, ownership_hash, ontology_revision, root_count, assertion_count,
+  created_at, ready_at, activated_at, terminal_at, last_commit_id, updated_at,
+  base_materialization_id, base_commit_id
+FROM legacy_ontology_sources
+ORDER BY rowid;
+CREATE UNIQUE INDEX idx_ontology_sources_active
+  ON ontology_sources(project_id, source_id)
+  WHERE status = 'active';
+CREATE UNIQUE INDEX idx_ontology_sources_run_candidate
+  ON ontology_sources(project_id, projection_run_id)
+  WHERE status IN ('staging', 'ready');
+CREATE INDEX idx_ontology_sources_cleanup
+  ON ontology_sources(project_id, status, terminal_at, source_id, materialization_id);
+CREATE INDEX idx_ontology_sources_run
+  ON ontology_sources(project_id, projection_run_id);
+
+-- A root is live while its version is published (active or superseded), it is not retired and
+-- it is not a deletion. Activation therefore writes only the roots it replaces, never its own.
+-- `root_key` is the canonical key; SQLite's default BINARY collation orders it by its UTF-8 bytes.
+CREATE TABLE ontology_source_roots (
+  id INTEGER PRIMARY KEY,
+  version_id INTEGER NOT NULL REFERENCES ontology_sources (version_id) ON DELETE RESTRICT,
+  project_id TEXT NOT NULL,
+  root_key TEXT NOT NULL,
+  staging_ordinal INTEGER NOT NULL CHECK (staging_ordinal >= 0),
+  deleted INTEGER NOT NULL DEFAULT 0 CHECK (deleted IN (0, 1)),
+  retired_at TEXT,
+  CONSTRAINT ontology_source_roots_key UNIQUE (version_id, root_key),
+  CONSTRAINT ontology_source_roots_ordinal UNIQUE (version_id, staging_ordinal)
+);
+CREATE INDEX idx_ontology_source_roots_live
+  ON ontology_source_roots (project_id, root_key) WHERE retired_at IS NULL AND deleted = 0;
+CREATE INDEX idx_ontology_source_roots_retired
+  ON ontology_source_roots (project_id, retired_at) WHERE retired_at IS NOT NULL;
+
+-- A row's identity lives in its typed columns; `payload` is the assertion without kind and ref.
+CREATE TABLE ontology_source_rows (
+  root_id INTEGER NOT NULL REFERENCES ontology_source_roots (id) ON DELETE RESTRICT,
+  entity_kind TEXT NOT NULL CHECK (entity_kind IN ('object', 'link')),
+  object_type_id TEXT,
+  primary_id TEXT,
+  source_type_id TEXT,
+  source_primary_id TEXT,
+  link_id TEXT,
+  target_type_id TEXT,
+  target_primary_id TEXT,
+  payload TEXT CHECK (json_valid(payload)),
+  CHECK (
+    (
+      entity_kind = 'object'
+      AND object_type_id IS NOT NULL AND primary_id IS NOT NULL
+      AND source_type_id IS NULL AND source_primary_id IS NULL AND link_id IS NULL
+      AND target_type_id IS NULL AND target_primary_id IS NULL
+    )
+    OR
+    (
+      entity_kind = 'link'
+      AND object_type_id IS NULL AND primary_id IS NULL
+      AND source_type_id IS NOT NULL AND source_primary_id IS NOT NULL AND link_id IS NOT NULL
+      AND target_type_id IS NOT NULL AND target_primary_id IS NOT NULL
+    )
+  )
+);
+CREATE INDEX idx_ontology_source_rows_root ON ontology_source_rows (root_id);
+CREATE INDEX idx_ontology_source_rows_link_source
+  ON ontology_source_rows (source_type_id, source_primary_id, link_id)
+  WHERE entity_kind = 'link';
+CREATE INDEX idx_ontology_source_rows_link_target
+  ON ontology_source_rows (target_type_id, target_primary_id)
+  WHERE entity_kind = 'link';
+
+-- A published root that was not active is retired, so liveness stays exactly what it was.
+INSERT INTO ontology_source_roots (
+  version_id, project_id, root_key, staging_ordinal, deleted, retired_at
+)
+SELECT versions.version_id, roots.project_id, roots.root_key, roots.staging_ordinal,
+  roots.deleted,
+  CASE
+    WHEN roots.active = 1 THEN NULL
+    WHEN roots.retired_at IS NOT NULL THEN roots.retired_at
+    WHEN versions.status IN ('active', 'superseded') THEN versions.updated_at
+  END
+FROM legacy_ontology_source_roots AS roots
+JOIN ontology_sources AS versions USING (project_id, source_id, materialization_id)
+ORDER BY versions.version_id, roots.staging_ordinal;
+
+INSERT INTO ontology_source_rows (
+  root_id, entity_kind, object_type_id, primary_id,
+  source_type_id, source_primary_id, link_id, target_type_id, target_primary_id, payload
+)
+SELECT roots.id, rows.entity_kind, rows.object_type_id, rows.primary_id,
+  rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
+  rows.target_primary_id, NULLIF(json_remove(rows.assertion, '$.kind', '$.ref'), '{}')
+FROM legacy_ontology_source_rows AS rows
+JOIN ontology_sources AS versions USING (project_id, source_id, materialization_id)
+JOIN ontology_source_roots AS roots
+  ON roots.version_id = versions.version_id AND roots.root_key = rows.root_key
+ORDER BY roots.id;
+
+DROP TABLE legacy_ontology_source_rows;
+DROP TABLE legacy_ontology_source_roots;
+DROP TABLE legacy_ontology_sources;
+-- No ANALYZE: nothing refreshes SQLite statistics afterwards, and on a fresh install they would
+-- describe empty tables forever and steer the planner to partial indexes over primary keys.

package/src/migrations.ts

@@ -115,6 +115,12 @@ import nativeSessionsSql from "./migrations/049-native-sessions.sql" with { type
 import directoryGroupMembershipsSql from "./migrations/050-directory-group-memberships.sql" with {
   type: "text",
 }
+import projectionRunSupersessionSql from "./migrations/051-projection-run-supersession.sql" with {
+  type: "text",
+}
+import compactSourceStorageSql from "./migrations/052-compact-source-storage.sql" with {
+  type: "text",
+}
 
 const MIGRATIONS_TABLE_SQL = `
   CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -220,6 +226,26 @@ export const sqliteStorageMigrations = defineMigrations({
     sqliteSql("048-file-upload-sessions", fileUploadSessionsSql),
     sqliteSql("049-native-sessions", nativeSessionsSql),
     sqliteSql("050-directory-group-memberships", directoryGroupMembershipsSql),
+    sqliteSql("051-projection-run-supersession", projectionRunSupersessionSql),
+    sqliteStep(
+      "052-compact-source-storage",
+      (db) => {
+        // SQLite has no procedural block to check the copy in SQL: count around it instead.
+        const before = sourceStorageCounts(db)
+        db.run(compactSourceStorageSql)
+        const after = sourceStorageCounts(db)
+        if (
+          after.versions !== before.versions ||
+          after.roots !== before.roots ||
+          after.rows !== before.rows
+        ) {
+          throw new Error(
+            "[SixbSqliteStorage] Source versions, roots or rows were lost while compacting source storage."
+          )
+        }
+      },
+      { checksum: checksum(compactSourceStorageSql) }
+    ),
   ],
 })
 
@@ -378,6 +404,18 @@ function rowToMigrationRecord(row: unknown): MigrationRecord {
   }
 }
 
+function sourceStorageCounts(db: Database): {
+  readonly versions: number
+  readonly roots: number
+  readonly rows: number
+} {
+  return db
+    .query(`SELECT (SELECT count(*) FROM ontology_sources) AS versions,
+      (SELECT count(*) FROM ontology_source_roots) AS roots,
+      (SELECT count(*) FROM ontology_source_rows) AS rows`)
+    .get() as { readonly versions: number; readonly roots: number; readonly rows: number }
+}
+
 function checksum(value: string): string {
   return createHash("sha256").update(value).digest("hex")
 }

package/src/objects/README.md

@@ -0,0 +1,19 @@
+# Object reads
+
+`read-scope.ts` derives visible objects, links and property grants from the selected roots and
+paths. The selected reader holds its traversal-budget probe and terminal statements on one WAL
+snapshot. SQL must filter by those relations before ranking or limiting results.
+
+SQLite flattens CTEs. Expose identity columns from the grant/visible relation and put authorized
+parents before indexed lookups with `CROSS JOIN`; exposing stored identities can cause a full
+grant scan per traversed edge. The query-plan regression test checks those lookups.
+
+Vector `list()` materializes at most the candidate budget plus one identity/vector pair per
+eligible object. Its count controls distance evaluation and travels with the ranked rows, so
+admission, total and ranking share one scan. Returned objects are hydrated after top-k; candidate
+filters and traversals may still evaluate masked properties while deciding eligibility.
+Overflow fails before exposing rows; it never silently ranks a truncated candidate set.
+
+An outer zero limit can suppress the count row; that case uses the separate bounded probe.
+Count, existence and facet terminals also retain that probe. Totals count the top-k set before
+the outer limit, and all candidates must grant every profile source property.

package/src/objects/query-compiler.ts

@@ -24,7 +24,14 @@ export interface SqliteObjectQueryPageRow {
 }
 
 export interface CompiledObjectQuery {
-  vectorProbe?: { sql: string; args: SqliteValue[]; limit: number }
+  vectorProbe?: {
+    sql: string
+    args: SqliteValue[]
+    limit: number
+    /** Maximum returned rows and total before the outermost limit, respectively. */
+    rowLimit: number
+    totalLimit: number
+  }
   sql: string
   args: SqliteValue[]
   totalSql: string
@@ -98,7 +105,7 @@ export function compileObjectQuery(
   if (hasVectorProfile(query) && !isVectorProfileQuery(query)) {
     throw new ObjectQueryExecutionError(
       "unsupported_vector_composition",
-      "Vector search supports one profile with filters before ranking and limit/project after."
+      "Vector search supports one profile with filters, traversals or sets before ranking and limit/project after."
     )
   }
   const source = options.source ?? DEFAULT_OBJECT_QUERY_SOURCE
@@ -231,9 +238,31 @@ function compileVector(
     { kind: "column", column: "_vector_score", direction: "desc" },
     ...identityOrderFields().map((field) => ({ ...field, binary: true })),
   ])
-  const sql = `SELECT * FROM (SELECT input.*, max(-1, min(1, 1 - vec_distance_cosine(vectors.embedding, ?))) AS _vector_score
-    ${candidates}) AS ranked ORDER BY ${order.sql} LIMIT ?`
-  const args = [encodeSqliteVector(values), ...candidateArgs, query.k]
+  const rankedOrder = compileOrder(order.fields, "ranked")
+  // Materialize only bounded, authorized candidates once. CASE prevents distance evaluation
+  // above the envelope; the reader rejects the count before exposing any rows or scores.
+  const sql = `SELECT stored.project_id, stored.object_type_id, stored.primary_id,
+    stored.properties, stored.properties AS _cursor_properties,
+    stored.created_at, stored.updated_at, stored.version, stored.last_commit_id,
+    ranked._vector_candidate_count, ranked._vector_score
+    FROM (
+    WITH _sixb_vector_candidates AS MATERIALIZED (
+      SELECT input.project_id, input.object_type_id, input.primary_id,
+        vectors.embedding AS _vector_embedding ${candidates} LIMIT ?
+    ), _sixb_vector_budget AS (
+      SELECT COUNT(*) AS _vector_candidate_count FROM _sixb_vector_candidates
+    )
+    SELECT input.project_id, input.object_type_id, input.primary_id, budget._vector_candidate_count,
+      CASE WHEN budget._vector_candidate_count <= ?
+        THEN max(-1, min(1, 1 - vec_distance_cosine(input._vector_embedding, ?)))
+      END AS _vector_score
+    FROM _sixb_vector_candidates AS input CROSS JOIN _sixb_vector_budget AS budget
+    ORDER BY ${order.sql} LIMIT ?
+  ) AS ranked CROSS JOIN ${ctx.source.objectsTable} AS stored
+    ON stored.project_id = ranked.project_id AND stored.object_type_id = ranked.object_type_id
+      AND stored.primary_id = ranked.primary_id
+  ORDER BY ${rankedOrder.sql}`
+  const args = [...candidateArgs, limit + 1, limit, encodeSqliteVector(values), query.k]
   return {
     sql,
     args,
@@ -245,6 +274,8 @@ function compileVector(
       sql: `SELECT COUNT(*) AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS candidates`,
       args: [...candidateArgs, limit + 1],
       limit,
+      rowLimit: query.k,
+      totalLimit: query.k,
     },
     hasMore: () => false,
     trimRows: identityRows,
@@ -451,7 +482,13 @@ function compileLimit(
       FROM (${input.sql}) AS input
     `,
     totalArgs: input.args,
-    vectorProbe: input.vectorProbe,
+    vectorProbe: input.vectorProbe
+      ? {
+          ...input.vectorProbe,
+          rowLimit: Math.min(input.vectorProbe.rowLimit, rowLimit),
+          totalLimit: input.vectorProbe.rowLimit,
+        }
+      : undefined,
     order: input.order,
     hasMore: (rowCount, total) =>
       total === undefined ? ctx.probeLimit && rowCount > limit : limit < total,
@@ -852,7 +889,7 @@ function compileProject(
         input.created_at,
         input.updated_at,
         input.version,
-        input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
+        input.last_commit_id${input.vectorProbe ? ", input._vector_score, input._vector_candidate_count" : ""}
       FROM (${input.sql}) AS input
       ORDER BY ${inputOrder.sql}
     `,

package/src/objects/read-scope.ts

@@ -22,6 +22,8 @@ export interface SqliteSelectedObjectReadSource extends SqliteObjectQuerySource
  * CROSS JOIN keeps authorized parents/grants before indexed edge/object lookups, even when
  * statistics would otherwise make SQLite rescan all grants for every stored object.
  * Property grants are grouped only after path-sensitive reachability has been resolved.
+ * Expose identities from grants/visible, so flattened joins can seek those relations by key
+ * before loading stored values. Exposing stored identities makes traversal rescan grants per edge.
  */
 export function compileSqliteSelectedObjectReadSource(
   projectId: string,
@@ -105,7 +107,7 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
       _sixb_scope_root_facts(project_id, root_id, node_id, object_type_id, primary_id) AS (
         SELECT anchor.project_id, root.root_id, root.node_id, root.object_type_id, root.primary_id
         FROM _sixb_scope_roots AS root
-        JOIN objects AS anchor
+        CROSS JOIN objects AS anchor
           ON anchor.project_id = ?
          AND anchor.object_type_id = root.object_type_id
          AND anchor.primary_id = root.primary_id
@@ -187,7 +189,7 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
         GROUP BY reachable.project_id, reachable.object_type_id, reachable.primary_id
       ),
       _sixb_scope_objects AS (
-        SELECT stored.project_id, stored.object_type_id, stored.primary_id,
+        SELECT grants.project_id, grants.object_type_id, grants.primary_id,
           COALESCE((
             SELECT json_group_object(property.key, ${sqliteJsonEachValue("property")})
             FROM json_each(stored.properties) AS property
@@ -277,12 +279,12 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
       ),
       _sixb_scope_links AS (
         SELECT
-          stored.project_id,
-          stored.source_type_id,
-          stored.source_id,
-          stored.link_id,
-          stored.target_type_id,
-          stored.target_id,
+          visible.project_id,
+          visible.source_type_id,
+          visible.source_id,
+          visible.link_id,
+          visible.target_type_id,
+          visible.target_id,
           CASE
             WHEN stored.properties IS NULL THEN NULL
             ELSE NULLIF(

package/src/objects/reader.ts

@@ -54,23 +54,23 @@ export class SqliteObjectReader {
     ensureSqliteVectorSearch(this.db)
     const probe = compiled.vectorProbe
     const row = this.db.query<{ total: number }, SqliteValue[]>(probe.sql).get(...probe.args)
-    if (Number(row?.total ?? 0) > probe.limit) {
-      throw new ObjectQueryExecutionError(
-        "vector_work_limit_exceeded",
-        `Exact vector search supports at most ${probe.limit} eligible vectors at this dimension. Narrow the query filters.`
-      )
-    }
+    assertVectorCandidateCount(row?.total, probe.limit)
   }
 
   queryObjects(params: QueryObjectsInput): QueryObjectsResult {
-    this.assertVectorSearch(params)
     const source = this.source
     const compiled = compileObjectQuery(params.projectId, params.query, {
       includeTotal: params.includeTotal,
       source,
     })
-    const total = params.includeTotal === false ? undefined : readTotal(this.db, compiled)
+    if (compiled.vectorProbe) ensureSqliteVectorSearch(this.db)
+    const ordinaryTotal =
+      params.includeTotal === false || compiled.vectorProbe
+        ? undefined
+        : readTotal(this.db, compiled)
     const rawRows = this.db.query(compiled.sql).all(...compiled.args) as ObjectQueryDatabaseRow[]
+    const vectorTotal = this.vectorResultTotal(compiled, rawRows)
+    const total = params.includeTotal === false ? undefined : (vectorTotal ?? ordinaryTotal)
     const rows = compiled.trimRows(rawRows) as readonly ObjectQueryDatabaseRow[]
     const hasMore =
       total === undefined && compiled.hasMoreProbe
@@ -87,6 +87,26 @@ export class SqliteObjectReader {
     }
   }
 
+  private vectorResultTotal(
+    compiled: CompiledObjectQuery,
+    rows: readonly ObjectQueryDatabaseRow[]
+  ): number | undefined {
+    const probe = compiled.vectorProbe
+    if (!probe) return undefined
+
+    // A zero outer limit can hide the count row. Keep admission and total semantics for that
+    // uncommon case, without rescanning candidates for ordinary vector queries.
+    let count = rows[0]?._vector_candidate_count
+    if (rows.length === 0) {
+      count =
+        probe.rowLimit === 0
+          ? this.db.query<{ total: number }, SqliteValue[]>(probe.sql).get(...probe.args)?.total
+          : 0
+    }
+    assertVectorCandidateCount(count, probe.limit)
+    return Math.min(count, probe.totalLimit)
+  }
+
   countObjects(params: CountObjectsInput): CountObjectsResult {
     this.assertVectorSearch(params)
     const source = this.source
@@ -519,6 +539,21 @@ function sqliteFacetValue(valueType: string | null, value: unknown): unknown {
   return value
 }
 
+function assertVectorCandidateCount(count: unknown, limit: number): asserts count is number {
+  if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0) {
+    throw new ObjectQueryExecutionError(
+      "invalid_vector_candidate_count",
+      "Storage did not return a valid vector candidate count."
+    )
+  }
+  if (count > limit) {
+    throw new ObjectQueryExecutionError(
+      "vector_work_limit_exceeded",
+      `Exact vector search supports at most ${limit} eligible vectors at this dimension. Narrow the query filters.`
+    )
+  }
+}
+
 function existsProbeQuery(query: ObjectQuery): ObjectQuery {
   return { kind: "limit", limit: 1, input: stripOuterRowShape(query) }
 }

package/src/objects/rows.ts

@@ -107,6 +107,7 @@ export interface DatabaseRow {
 }
 
 export interface ObjectQueryDatabaseRow extends DatabaseRow {
+  _vector_candidate_count?: number
   _vector_score?: number
   _cursor_properties?: string
   /** `json_object(linkId, value, ...)` serialized text from an `expand` pushdown; absent otherwise. */

package/src/ontology-storage/materialization-state.ts

@@ -43,8 +43,15 @@ import {
   parseJson,
   type SqliteOntologySourceAssertionRow,
   type SqliteStoredOverrideRow,
+  sourceAssertion,
 } from "./shared"
-import { activeSourceRows, replacementAssertionRows, replacementSourceRows } from "./source-roots"
+import {
+  activeSourceRows,
+  LIVE_SOURCE_ROWS_JOIN,
+  replacementAssertionRows,
+  replacementSourceRows,
+  SOURCE_ASSERTION_COLUMNS,
+} from "./source-roots"
 
 export const SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work"
 export const SQLITE_REPLACEMENT_WORK_TABLE = "ontology_replacement_work"
@@ -438,19 +445,13 @@ export class SqliteMaterializationStateReader {
              json_extract(value, '$.linkId') AS link_id
            FROM json_each(?)
          )
-         SELECT rows.* FROM requested
+         SELECT ${SOURCE_ASSERTION_COLUMNS} FROM requested
          CROSS JOIN ontology_source_rows AS rows
-           ON rows.project_id = ?
-          AND rows.entity_kind = 'link'
+           ON rows.entity_kind = 'link'
           AND rows.source_type_id = requested.source_type_id
           AND rows.source_primary_id = requested.source_primary_id
           AND rows.link_id = requested.link_id
-         JOIN ontology_source_roots AS sources
-           ON sources.project_id = rows.project_id
-          AND sources.source_id = rows.source_id
-          AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-         WHERE sources.active = 1`
+         ${LIVE_SOURCE_ROWS_JOIN}`
       )
       .all(requestedJson, this.projectId) as SqliteOntologySourceAssertionRow[]
     const slotOverrideRows = this.db
@@ -570,12 +571,8 @@ export class SqliteMaterializationStateReader {
               SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
                 rows.target_type_id, rows.target_primary_id
               FROM ontology_source_rows AS rows
-              JOIN ontology_source_roots AS sources
-                ON sources.project_id = rows.project_id
-               AND sources.source_id = rows.source_id
-               AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-              WHERE rows.project_id = ? AND rows.entity_kind = 'link' AND sources.active = 1
+              ${LIVE_SOURCE_ROWS_JOIN}
+              WHERE rows.entity_kind = 'link'
             ), selected AS (
               SELECT DISTINCT links.*,
                 ${linkSortExpression("links")} AS sort_key
@@ -752,7 +749,7 @@ export class SqliteMaterializationStateReader {
   }
 
   private prepareReplacementObjects(input: ReplacementIdentityInput): void {
-    const rows = replacementSourceRows({
+    const rows = replacementSourceRows(this.db, {
       projectId: this.projectId,
       sourceId: input.sourceId,
       materializationId: input.candidateMaterializationId,
@@ -765,7 +762,7 @@ export class SqliteMaterializationStateReader {
             session_id, entity_kind, identity_key, sort_key, diff_required
           )
           SELECT ?, 'object', json_array(object_type_id, primary_id),
-            MIN(entity_sort_key), 1
+            ${objectSortExpression()}, 1
           FROM (${rows.sql}) AS rows
           WHERE entity_kind = 'object'
           GROUP BY object_type_id, primary_id
@@ -775,7 +772,7 @@ export class SqliteMaterializationStateReader {
   }
 
   private prepareReplacementLinks(input: ReplacementIdentityInput): void {
-    const rows = replacementSourceRows({
+    const rows = replacementSourceRows(this.db, {
       projectId: this.projectId,
       sourceId: input.sourceId,
       materializationId: input.candidateMaterializationId,
@@ -834,31 +831,21 @@ export class SqliteMaterializationStateReader {
             UNION
             SELECT rows.source_type_id, rows.source_primary_id AS source_id,
               rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
-            FROM ontology_source_rows AS rows
-            JOIN ontology_source_roots AS sources
-              ON sources.project_id = rows.project_id
-             AND sources.source_id = rows.source_id
-             AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-            JOIN incident_objects
-              ON incident_objects.object_type_id = rows.source_type_id
-             AND incident_objects.primary_id = rows.source_primary_id
-            WHERE rows.project_id = ? AND rows.entity_kind = 'link'
-              AND sources.active = 1
+            FROM incident_objects
+            CROSS JOIN ontology_source_rows AS rows
+              ON rows.entity_kind = 'link'
+             AND rows.source_type_id = incident_objects.object_type_id
+             AND rows.source_primary_id = incident_objects.primary_id
+            ${LIVE_SOURCE_ROWS_JOIN}
             UNION
             SELECT rows.source_type_id, rows.source_primary_id AS source_id,
               rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
-            FROM ontology_source_rows AS rows
-            JOIN ontology_source_roots AS sources
-              ON sources.project_id = rows.project_id
-             AND sources.source_id = rows.source_id
-             AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-            JOIN incident_objects
-              ON incident_objects.object_type_id = rows.target_type_id
-             AND incident_objects.primary_id = rows.target_primary_id
-            WHERE rows.project_id = ? AND rows.entity_kind = 'link'
-              AND sources.active = 1
+            FROM incident_objects
+            CROSS JOIN ontology_source_rows AS rows
+              ON rows.entity_kind = 'link'
+             AND rows.target_type_id = incident_objects.object_type_id
+             AND rows.target_primary_id = incident_objects.primary_id
+            ${LIVE_SOURCE_ROWS_JOIN}
           ), diff_links AS (
             SELECT * FROM replacement_links
             UNION SELECT * FROM incident_links
@@ -1062,9 +1049,7 @@ function storedSource(row: SqliteOntologySourceAssertionRow): StoredSourceAssert
   return {
     source: { projectionId: row.source_id },
     materializationId: row.materialization_id,
-    root: parseJson<StoredSourceAssertion["root"]>(row.root),
-    assertion: parseJson<StoredSourceAssertion["assertion"]>(row.assertion),
-    stagingOrdinal: row.staging_ordinal,
+    ...sourceAssertion(row),
   } as StoredSourceAssertion
 }
 
@@ -1130,6 +1115,11 @@ function storedPoint(row: TelemetryRow): StoredTelemetryPoint {
   }
 }
 
+function objectSortExpression(alias?: string): string {
+  const prefix = alias ? `${alias}.` : ""
+  return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}primary_id) AS BLOB)))`
+}
+
 export function linkSortExpression(alias?: string): string {
   const prefix = alias ? `${alias}.` : ""
   return `LOWER(HEX(CAST(json_array(${prefix}source_type_id, ${prefix}source_id, ${prefix}link_id, ${prefix}target_type_id, ${prefix}target_id) AS BLOB)))`