@sixb/sqlite 0.1.14 → 0.1.16

package/src/ontology-storage/shared.ts

@@ -1,7 +1,10 @@
 import type { Database } from "bun:sqlite"
 import { parseSixbFailure } from "@sixb/core/internal/errors"
-import type { ProjectionEntityRef } from "@sixb/core/internal/materialization"
 import { MaterializationConflictError } from "@sixb/core/internal/materialization"
+import {
+  sourceAssertionFromColumns,
+  sourceEntityFromKey,
+} from "@sixb/core/internal/ontology-storage-provider"
 import type {
   AssertSourceMaterializationExecutionInput,
   OntologyCommitRecord,
@@ -48,6 +51,7 @@ export interface SqliteOntologyCommitRow {
 }
 
 export interface SqliteOntologySourceRow {
+  readonly version_id: number
   readonly project_id: string
   readonly source_id: string
   readonly materialization_id: string
@@ -74,19 +78,21 @@ export interface SqliteOntologySourceRow {
   readonly updated_at: string
 }
 
+/** An assertion with its root and version, as `SOURCE_ASSERTION_COLUMNS` selects it. */
 export interface SqliteOntologySourceAssertionRow {
-  readonly project_id: string
   readonly source_id: string
   readonly materialization_id: string
-  readonly entity_kind: "object" | "link"
-  readonly entity_key: string
-  readonly entity_sort_key: string
-  readonly root_kind: "object" | "link"
   readonly root_key: string
-  readonly root_sort_key: string
   readonly staging_ordinal: number
-  readonly root: string
-  readonly assertion: string
+  readonly entity_kind: "object" | "link"
+  readonly object_type_id: string | null
+  readonly primary_id: string | null
+  readonly source_type_id: string | null
+  readonly source_primary_id: string | null
+  readonly link_id: string | null
+  readonly target_type_id: string | null
+  readonly target_primary_id: string | null
+  readonly payload: string | null
 }
 
 export interface SqliteStoredOverrideRow {
@@ -149,8 +155,11 @@ export function sourceRecord(row: SqliteOntologySourceRow): OntologySourceRecord
 
 export function sourceAssertion(row: SqliteOntologySourceAssertionRow): StageSourceAssertion {
   return {
-    root: parseJson<ProjectionEntityRef>(row.root),
-    assertion: parseJson<StageSourceAssertion["assertion"]>(row.assertion),
+    root: sourceEntityFromKey(row.root_key),
+    assertion: sourceAssertionFromColumns(
+      row,
+      row.payload === null ? null : parseJson<unknown>(row.payload)
+    ),
     stagingOrdinal: row.staging_ordinal,
   }
 }

package/src/ontology-storage/source-cleanup.ts

@@ -2,38 +2,79 @@ import type { Database } from "bun:sqlite"
 import type {
   CleanupTerminalSourceMaterializationsInput,
   CleanupTerminalSourceMaterializationsResult,
+  PurgeAbandonedSourceMaterializationsInput,
 } from "@sixb/core/storage"
 
+/**
+ * Superseded versions can still hold live roots (a delta keeps unchanged roots where they were),
+ * so they are cleaned root by root once retired. An abandoned candidate was never published:
+ * everything it holds goes, one whole version at a time, each delete on its own version prefix.
+ */
+export function purgeAbandonedSourceVersions(
+  db: Database,
+  input: PurgeAbandonedSourceMaterializationsInput
+): CleanupTerminalSourceMaterializationsResult {
+  let remaining = input.limit
+  let rowsDeleted = 0
+  let materializationsDeleted = 0
+  const nextCandidate = db.query(`
+    SELECT version_id FROM ontology_sources
+    WHERE project_id = ? AND status = 'abandoned'
+    ORDER BY terminal_at, source_id, materialization_id LIMIT 1
+  `)
+  const deleteRows = db.query(`DELETE FROM ontology_source_rows WHERE rowid IN (
+    SELECT rows.rowid FROM ontology_source_roots AS roots
+    CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
+    WHERE roots.version_id = ? LIMIT ?)`)
+  const deleteRoots = db.query(`DELETE FROM ontology_source_roots WHERE id IN (
+    SELECT id FROM ontology_source_roots WHERE version_id = ? LIMIT ?)`)
+  const deleteManifest = db.query(
+    "DELETE FROM ontology_sources WHERE version_id = ? AND status = 'abandoned'"
+  )
+  while (remaining > 0) {
+    const candidate = nextCandidate.get(input.projectId) as { readonly version_id: number } | null
+    if (!candidate) break
+    const rows = deleteRows.run(candidate.version_id, remaining).changes
+    rowsDeleted += rows
+    remaining -= rows
+    if (remaining === 0) break
+    const roots = deleteRoots.run(candidate.version_id, remaining).changes
+    rowsDeleted += roots
+    remaining -= roots
+    if (remaining === 0) break
+    const manifest = deleteManifest.run(candidate.version_id).changes
+    if (manifest === 0) break
+    materializationsDeleted += manifest
+    remaining -= manifest
+  }
+  return { rowsDeleted, materializationsDeleted }
+}
+
 export function cleanupSourceVersions(
   db: Database,
   input: CleanupTerminalSourceMaterializationsInput
 ): CleanupTerminalSourceMaterializationsResult {
   let remaining = input.limit
+  const retired = `roots.project_id = ? AND roots.retired_at < ? AND versions.status = 'superseded'`
   let rowsDeleted = db
     .query(`DELETE FROM ontology_source_rows WHERE rowid IN (
     SELECT rows.rowid FROM ontology_source_roots AS roots
-    JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
-    CROSS JOIN ontology_source_rows AS rows
-      ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-      AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
-    WHERE roots.project_id = ? AND roots.active = 0 AND roots.retired_at < ?
-      AND sources.status IN ('superseded', 'abandoned')
-    ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
+    JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+    CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
+    WHERE ${retired}
+    ORDER BY roots.retired_at, roots.id
     LIMIT ?
   )`)
     .run(input.projectId, input.terminalBefore, remaining).changes
   remaining -= rowsDeleted
   if (remaining > 0) {
     const removed = db
-      .query(`DELETE FROM ontology_source_roots WHERE rowid IN (
-      SELECT roots.rowid FROM ontology_source_roots AS roots
-      JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
-      WHERE roots.project_id = ? AND roots.active = 0 AND roots.retired_at < ?
-        AND sources.status IN ('superseded', 'abandoned')
-        AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
-          WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-            AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
-      ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key LIMIT ?
+      .query(`DELETE FROM ontology_source_roots WHERE id IN (
+      SELECT roots.id FROM ontology_source_roots AS roots
+      JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+      WHERE ${retired}
+        AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id)
+      ORDER BY roots.retired_at, roots.id LIMIT ?
     )`)
       .run(input.projectId, input.terminalBefore, remaining).changes
     rowsDeleted += removed
@@ -43,15 +84,11 @@ export function cleanupSourceVersions(
     remaining === 0
       ? 0
       : db
-          .query(`DELETE FROM ontology_sources WHERE rowid IN (
-    SELECT sources.rowid FROM ontology_sources AS sources
-    WHERE project_id = ? AND status IN ('superseded', 'abandoned') AND terminal_at < ?
+          .query(`DELETE FROM ontology_sources WHERE version_id IN (
+    SELECT versions.version_id FROM ontology_sources AS versions
+    WHERE project_id = ? AND status = 'superseded' AND terminal_at < ?
       AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
-        WHERE roots.project_id = sources.project_id AND roots.source_id = sources.source_id
-          AND roots.materialization_id = sources.materialization_id)
-      AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
-        WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
-          AND rows.materialization_id = sources.materialization_id)
+        WHERE roots.version_id = versions.version_id)
     ORDER BY terminal_at, source_id, materialization_id LIMIT ?
   )`)
           .run(input.projectId, input.terminalBefore, remaining).changes

package/src/ontology-storage/source-roots.ts

@@ -5,40 +5,121 @@ import {
   MaterializationValidationError,
   projectionEntityKey,
 } from "@sixb/core/internal/materialization"
-import { sourceStageRoots, utf8SortKey } from "@sixb/core/internal/ontology-storage-provider"
+import {
+  sourceEntityColumns,
+  sourceStageRoots,
+} from "@sixb/core/internal/ontology-storage-provider"
 import type { SourceActivationWrite, StageSourceRowsInput } from "@sixb/core/storage"
 import type { SqliteOntologySourceAssertionRow } from "./shared"
 import { canonicalJson, type SqliteOntologySourceRow } from "./shared"
 
-export function replacementSourceRows(input: {
-  readonly projectId: string
-  readonly sourceId: string
-  readonly materializationId: string
-  readonly incremental: boolean
-}): { readonly sql: string; readonly values: readonly string[] } {
-  const roots = input.incremental
-    ? `ontology_source_roots AS changed
-    CROSS JOIN ontology_source_roots AS roots ON roots.project_id = changed.project_id
-      AND roots.source_id = changed.source_id AND roots.root_sort_key = changed.root_sort_key AND roots.active = 1`
-    : "ontology_source_roots AS roots"
-  const selected = input.incremental ? "changed" : "roots"
+/**
+ * Source storage in three levels: a version (`ontology_sources`, one per run), its roots (one per
+ * logical dataset row, keyed by the canonical entity key) and their rows (what each root asserts).
+ *
+ * A root is live while its version is published (active or superseded), it is not retired and it
+ * is not a deletion. Activation retires the live roots its candidate replaces and flips the
+ * version; the candidate's own roots are never rewritten. One live root per source and key holds
+ * because activation, under the source fence, retires every root the candidate replaces.
+ */
+const PUBLISHED = "('active', 'superseded')"
+
+/** The columns `sourceAssertion` reads, over `versions`, `roots` and `rows`. */
+export const SOURCE_ASSERTION_COLUMNS = `versions.source_id, versions.materialization_id,
+  roots.root_key, roots.staging_ordinal, rows.entity_kind, rows.object_type_id, rows.primary_id,
+  rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
+  rows.target_primary_id, rows.payload`
+
+/**
+ * Keeps `rows` to those of live roots, joined as `roots` and `versions`; binds the project id.
+ * CROSS JOIN keeps it a filter on the rows already selected: as a plain JOIN, SQLite may start
+ * from every live root of the project instead.
+ */
+export const LIVE_SOURCE_ROWS_JOIN = `
+  CROSS JOIN ontology_source_roots AS roots ON roots.id = rows.root_id
+    AND roots.project_id = ? AND roots.retired_at IS NULL AND roots.deleted = 0
+  CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+    AND versions.status IN ${PUBLISHED}`
+
+/** The typed identity of `rows` equals the one requested (`IS` matches NULLs). */
+const ENTITY_MATCH = `rows.entity_kind = requested.entity_kind
+  AND rows.object_type_id IS requested.object_type_id
+  AND rows.primary_id IS requested.primary_id
+  AND rows.source_type_id IS requested.source_type_id
+  AND rows.source_primary_id IS requested.source_primary_id
+  AND rows.link_id IS requested.link_id
+  AND rows.target_type_id IS requested.target_type_id
+  AND rows.target_primary_id IS requested.target_primary_id`
+
+const REQUESTED_ENTITY = `json_extract(value, '$.entityKind') AS entity_kind,
+  json_extract(value, '$.objectTypeId') AS object_type_id,
+  json_extract(value, '$.primaryId') AS primary_id,
+  json_extract(value, '$.sourceTypeId') AS source_type_id,
+  json_extract(value, '$.sourcePrimaryId') AS source_primary_id,
+  json_extract(value, '$.linkId') AS link_id,
+  json_extract(value, '$.targetTypeId') AS target_type_id,
+  json_extract(value, '$.targetPrimaryId') AS target_primary_id`
+
+/** An entity's own key, and for a link the key of the object root that may assert it. */
+function requestedEntity(ref: ProjectionEntityRef) {
+  const key = projectionEntityKey(ref)
+  return {
+    entityKind: ref.kind,
+    ...sourceEntityColumns(ref),
+    rootKeys:
+      ref.kind === "object"
+        ? [key]
+        : [key, projectionEntityKey({ kind: "object", ref: ref.ref.source })],
+  }
+}
+
+function sourceVersionId(
+  db: Database,
+  input: {
+    readonly projectId: string
+    readonly sourceId: string
+    readonly materializationId: string
+  }
+): number | null {
+  const version = db
+    .query(`SELECT version_id FROM ontology_sources
+      WHERE project_id = ? AND source_id = ? AND materialization_id = ?`)
+    .get(input.projectId, input.sourceId, input.materializationId) as {
+    readonly version_id: number
+  } | null
+  return version?.version_id ?? null
+}
+
+/** The candidate's rows, plus the rows of the live roots it may replace. */
+export function replacementSourceRows(
+  db: Database,
+  input: {
+    readonly projectId: string
+    readonly sourceId: string
+    readonly materializationId: string
+    readonly incremental: boolean
+  }
+): { readonly sql: string; readonly values: readonly (string | number | null)[] } {
+  const version = sourceVersionId(db, input)
+  const live = `live.retired_at IS NULL AND live.deleted = 0
+    AND versions.source_id = ? AND versions.status IN ${PUBLISHED}`
+  const replaced = input.incremental
+    ? `SELECT live.id FROM ontology_source_roots AS changed
+      CROSS JOIN ontology_source_roots AS live
+        ON live.project_id = ? AND live.root_key = changed.root_key
+      CROSS JOIN ontology_sources AS versions ON versions.version_id = live.version_id
+      WHERE changed.version_id = ? AND ${live}`
+    : `SELECT live.id FROM ontology_sources AS versions
+      CROSS JOIN ontology_source_roots AS live ON live.version_id = versions.version_id
+      WHERE versions.project_id = ? AND ${live}`
   return {
-    sql: `SELECT * FROM ontology_source_rows WHERE project_id = ? AND source_id = ? AND materialization_id = ?
+    sql: `SELECT rows.* FROM ontology_source_roots AS roots
+      CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
+      WHERE roots.version_id = ?
       UNION ALL
-      SELECT rows.* FROM ${roots}
-      CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id
-        AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
-        AND rows.root_sort_key = roots.root_sort_key
-      WHERE ${selected}.project_id = ? AND ${selected}.source_id = ?
-        AND ${input.incremental ? "changed.materialization_id = ?" : "roots.active = 1"}`,
-    values: [
-      input.projectId,
-      input.sourceId,
-      input.materializationId,
-      input.projectId,
-      input.sourceId,
-      ...(input.incremental ? [input.materializationId] : []),
-    ],
+      SELECT rows.* FROM (${replaced}) AS roots
+      CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id`,
+    values: [version, input.projectId, ...(input.incremental ? [version] : []), input.sourceId],
   }
 }
 
@@ -52,29 +133,23 @@ export function stageSourceRoots(
     throw new MaterializationValidationError("Root deletions require a source delta base.")
   }
   const existing = db.query(`SELECT staging_ordinal, deleted FROM ontology_source_roots
-    WHERE project_id = ? AND source_id = ? AND materialization_id = ? AND root_sort_key = ?`)
+    WHERE version_id = ? AND root_key = ?`)
   const insert = db.query(`INSERT INTO ontology_source_roots (
-    project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
-  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING RETURNING root_sort_key`)
+    version_id, project_id, root_key, staging_ordinal, deleted
+  ) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING RETURNING id`)
   for (const root of roots) {
     const inserted = insert.get(
+      manifest.version_id,
       input.projectId,
-      input.source.projectionId,
-      input.materializationId,
-      root.sortKey,
-      root.root.kind,
       root.rootKey,
-      canonicalJson(root.root),
       root.stagingOrdinal,
       Number(root.deleted)
     )
     if (inserted) continue
-    const previous = existing.get(
-      input.projectId,
-      input.source.projectionId,
-      input.materializationId,
-      root.sortKey
-    ) as { staging_ordinal: number; deleted: number } | null
+    const previous = existing.get(manifest.version_id, root.rootKey) as {
+      staging_ordinal: number
+      deleted: number
+    } | null
     if (
       !previous ||
       previous.staging_ordinal !== root.stagingOrdinal ||
@@ -87,6 +162,7 @@ export function stageSourceRoots(
   }
 }
 
+/** Retires what the candidate replaces; its own roots go live when its version turns active. */
 export function activateSourceRoots(
   db: Database,
   projectId: string,
@@ -102,76 +178,66 @@ export function activateSourceRoots(
       "projection-fence",
       "Source delta base changed before activation."
     )
-
   if (candidate.base_materialization_id !== null && candidate.root_count === 0) return
-  const scope = [projectId, activation.source.projectionId, activation.materializationId] as const
-  db.query(`UPDATE ontology_source_roots AS roots SET active = 0, retired_at = ?
-    WHERE project_id = ? AND source_id = ? AND active = 1
+  const delta = candidate.base_materialization_id !== null
+  db.query(`UPDATE ontology_source_roots SET retired_at = ?
+    WHERE retired_at IS NULL AND deleted = 0
+      AND version_id IN (SELECT version_id FROM ontology_sources
+        WHERE project_id = ? AND source_id = ? AND status IN ${PUBLISHED})
       ${
-        candidate.base_materialization_id === null
-          ? ""
-          : `AND root_sort_key IN (
-        SELECT root_sort_key FROM ontology_source_roots
-        WHERE project_id = ? AND source_id = ? AND materialization_id = ?)`
+        delta
+          ? `AND project_id = ? AND root_key IN (
+        SELECT root_key FROM ontology_source_roots WHERE version_id = ?)`
+          : ""
       }`).run(
     activation.updatedAt,
     projectId,
     activation.source.projectionId,
-    ...(candidate.base_materialization_id === null ? [] : [...scope])
-  )
-  db.query(`UPDATE ontology_source_roots SET active = CASE WHEN deleted = 1 THEN 0 ELSE 1 END,
-      retired_at = CASE WHEN deleted = 1 THEN ? ELSE NULL END
-    WHERE project_id = ? AND source_id = ? AND materialization_id = ?`).run(
-    activation.updatedAt,
-    ...scope
+    ...(delta ? [projectId, candidate.version_id] : [])
   )
+  // Deletions are never live; retiring them lets cleanup remove them with their version.
+  db.query(`UPDATE ontology_source_roots SET retired_at = ?
+    WHERE version_id = ? AND deleted = 1`).run(activation.updatedAt, candidate.version_id)
 }
 
 export function assertSourceRootCoverage(db: Database, manifest: SqliteOntologySourceRow): void {
   const invalid = db
     .query(`SELECT 1 FROM ontology_source_roots AS roots
-    WHERE project_id = ? AND source_id = ? AND materialization_id = ?
+    WHERE roots.version_id = ?
       AND roots.deleted = EXISTS (
-        SELECT 1 FROM ontology_source_rows AS rows
-        WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-          AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
+        SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id
       ) LIMIT 1`)
-    .get(manifest.project_id, manifest.source_id, manifest.materialization_id)
+    .get(manifest.version_id)
   if (invalid)
     throw new MaterializationValidationError(
       "Source roots must have complete assertions or an explicit deletion, never both."
     )
 }
 
+/** The live assertions of `refs`, whichever source asserts them. */
 export function activeSourceRows(
   db: Database,
   projectId: string,
   refs: readonly ProjectionEntityRef[]
 ): SqliteOntologySourceAssertionRow[] {
-  const keys = refs.flatMap((entity) => {
-    const entitySortKey = utf8SortKey(projectionEntityKey(entity))
-    const roots =
-      entity.kind === "object"
-        ? [entitySortKey]
-        : [
-            entitySortKey,
-            utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source })),
-          ]
-    return roots.map((rootSortKey) => ({ rootSortKey, entitySortKey }))
+  const requested = refs.flatMap((ref) => {
+    const { rootKeys, ...entity } = requestedEntity(ref)
+    return rootKeys.map((rootKey) => ({ ...entity, rootKey }))
   })
   return db
     .query(`WITH requested AS (
-    SELECT DISTINCT json_extract(value, '$.rootSortKey') AS root_sort_key,
-      json_extract(value, '$.entitySortKey') AS entity_sort_key FROM json_each(?)
-  ) SELECT rows.* FROM requested
+    SELECT DISTINCT json_extract(value, '$.rootKey') AS root_key, ${REQUESTED_ENTITY}
+    FROM json_each(?)
+  ) SELECT ${SOURCE_ASSERTION_COLUMNS} FROM requested
     CROSS JOIN ontology_source_roots AS roots ON roots.project_id = ?
-      AND roots.active = 1 AND roots.root_sort_key = requested.root_sort_key
-    CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-      AND rows.materialization_id = roots.materialization_id AND rows.entity_sort_key = requested.entity_sort_key
-      AND rows.root_sort_key = roots.root_sort_key`)
-    .all(canonicalJson(keys), projectId) as SqliteOntologySourceAssertionRow[]
+      AND roots.root_key = requested.root_key AND roots.retired_at IS NULL AND roots.deleted = 0
+    CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+      AND versions.status IN ${PUBLISHED}
+    CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id AND ${ENTITY_MATCH}`)
+    .all(canonicalJson(requested), projectId) as SqliteOntologySourceAssertionRow[]
 }
 
+/** The candidate's assertions of `refs`, and optionally the live ones they would replace. */
 export function replacementAssertionRows(
   db: Database,
   input: {
@@ -183,48 +249,38 @@ export function replacementAssertionRows(
     refs: readonly ProjectionEntityRef[]
   }
 ): SqliteOntologySourceAssertionRow[] {
-  const keys = input.refs.map((ref) => ({
-    entityKind: ref.kind,
-    entityKey: projectionEntityKey(ref),
-    rootKeys:
-      ref.kind === "object"
-        ? [utf8SortKey(projectionEntityKey(ref))]
-        : [
-            utf8SortKey(projectionEntityKey(ref)),
-            utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source })),
-          ],
-  }))
+  const version = sourceVersionId(db, input)
   const previous = !input.includePrevious
     ? ""
     : `UNION ALL
-    SELECT rows.* FROM requested_entities AS requested
-    CROSS JOIN ontology_source_roots AS roots ON roots.project_id = ? AND roots.source_id = ?
-      AND roots.active = 1 AND roots.root_sort_key IN (SELECT value FROM json_each(requested.root_keys))
-    CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-      AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
-      AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
+    SELECT ${SOURCE_ASSERTION_COLUMNS} FROM requested_entities AS requested
+    CROSS JOIN ontology_source_roots AS roots ON roots.project_id = ?
+      AND roots.root_key IN (SELECT value FROM json_each(requested.root_keys))
+      AND roots.retired_at IS NULL AND roots.deleted = 0
+    CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+      AND versions.source_id = ? AND versions.status IN ${PUBLISHED}
+    CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id AND ${ENTITY_MATCH}
     ${
       !input.incremental
         ? ""
         : `WHERE EXISTS (SELECT 1 FROM ontology_source_roots AS changed
-      WHERE changed.project_id = roots.project_id AND changed.source_id = roots.source_id
-        AND changed.materialization_id = ? AND changed.root_sort_key = roots.root_sort_key)`
+      WHERE changed.version_id = ? AND changed.root_key = roots.root_key)`
     }`
   return db
     .query(`WITH requested_entities AS (
-    SELECT DISTINCT json_extract(value, '$.entityKind') AS entity_kind,
-      json_extract(value, '$.entityKey') AS entity_key, json_extract(value, '$.rootKeys') AS root_keys FROM json_each(?)
-  ) SELECT rows.* FROM requested_entities AS requested
-    CROSS JOIN ontology_source_rows AS rows ON rows.project_id = ? AND rows.source_id = ?
-      AND rows.materialization_id = ? AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
+    SELECT DISTINCT ${REQUESTED_ENTITY}, json_extract(value, '$.rootKeys') AS root_keys
+    FROM json_each(?)
+  ) SELECT ${SOURCE_ASSERTION_COLUMNS} FROM requested_entities AS requested
+    CROSS JOIN ontology_source_roots AS roots ON roots.version_id = ?
+      AND roots.root_key IN (SELECT value FROM json_each(requested.root_keys))
+    CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+    CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id AND ${ENTITY_MATCH}
     ${previous}`)
     .all(
-      canonicalJson(keys),
-      input.projectId,
-      input.sourceId,
-      input.materializationId,
+      canonicalJson(input.refs.map(requestedEntity)),
+      version,
       ...(input.includePrevious
-        ? [input.projectId, input.sourceId, ...(input.incremental ? [input.materializationId] : [])]
+        ? [input.projectId, input.sourceId, ...(input.incremental ? [version] : [])]
         : [])
     ) as SqliteOntologySourceAssertionRow[]
 }