@sixb/sqlite 0.1.9 → 0.1.11

package/src/ontology-storage/materializations.ts

@@ -65,6 +65,7 @@ import {
   type SqliteOntologyCommitRow,
   type SqliteOntologySourceRow,
 } from "./shared"
+import { activateSourceRoots } from "./source-roots"
 
 export class SqliteOntologyMaterializationStorage implements OntologyMaterializationStorage {
   private readonly sessions: SqliteMaterializationSessions
@@ -192,6 +193,7 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
         sourceId: replacement.sourceId,
         candidateMaterializationId: replacement.candidateMaterializationId,
         previousMaterializationId: replacement.previousMaterializationId,
+        incremental: replacement.incremental,
         kind: "object",
         pageRows: input.pageRows,
       })) {
@@ -236,6 +238,7 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
       sourceId: replacement.sourceId,
       candidateMaterializationId: replacement.candidateMaterializationId,
       previousMaterializationId: replacement.previousMaterializationId,
+      incremental: replacement.incremental,
       kind: "link",
       pageRows: input.pageRows,
     })) {
@@ -250,7 +253,8 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
         replacement.sourceId,
         replacement.candidateMaterializationId,
         materializationIds,
-        linkIdentities
+        linkIdentities,
+        replacement.incremental
       )
       yield { objects: [], links }
     }
@@ -359,6 +363,7 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
       sourceId: input.source.projectionId,
       candidateMaterializationId: input.candidateMaterializationId,
       previousMaterializationId: previous?.materialization_id ?? null,
+      incremental: candidate.base_materialization_id !== null,
       projectionKind: candidate.projection_kind,
       objectStreamStarted: false,
       objectStreamCompleted: false,
@@ -627,6 +632,7 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
       invalidCorrelation("Source activation does not match its ready candidate identity.")
     }
     this.assertSource(activation.expected, commit.projectId)
+    activateSourceRoots(this.db, commit.projectId, candidate, activation)
     const previous = this.getActiveSource(commit.projectId, activation.source.projectionId)
     if (previous) {
       assertPinnedDatasetWatermark(

package/src/ontology-storage/outbox.ts

@@ -43,19 +43,37 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
         )
       }
 
+      // Probe at most one page plus a sentinel through the eligibility index. Sparse retries
+      // avoid scanning delayed rows; overflowing backlogs use publication order without a sort.
+      const eligible = this.db
+        .query(
+          `SELECT CAST(rowid AS TEXT) AS rowId
+           FROM ontology_outbox INDEXED BY idx_ontology_outbox_claim
+           WHERE project_id = ? AND published_at IS NULL AND available_at <= ?
+             AND (lease_expires_at IS NULL OR lease_expires_at <= ?)
+           LIMIT ?`
+        )
+        .all(input.projectId, input.now, input.now, input.limit + 1) as { rowId: string }[]
+      if (eligible.length === 0) return []
+      const selected =
+        eligible.length <= input.limit
+          ? eligible
+          : (this.db
+              .query(
+                `SELECT CAST(rowid AS TEXT) AS rowId
+                 FROM ontology_outbox INDEXED BY idx_ontology_outbox_publication_order
+                 WHERE project_id = ? AND published_at IS NULL AND available_at <= ?
+                   AND (lease_expires_at IS NULL OR lease_expires_at <= ?)
+                 ORDER BY created_at, commit_id, commit_ordinal
+                 LIMIT ?`
+              )
+              .all(input.projectId, input.now, input.now, input.limit) as { rowId: string }[])
       const rows = this.db
         .query(
           `
             UPDATE ontology_outbox
             SET attempts = attempts + 1, lease_id = ?, lease_expires_at = ?
-            WHERE rowid IN (
-              SELECT rowid
-              FROM ontology_outbox
-              WHERE project_id = ? AND published_at IS NULL AND available_at <= ?
-                AND (lease_expires_at IS NULL OR lease_expires_at <= ?)
-              ORDER BY created_at, commit_id, commit_ordinal
-              LIMIT ?
-            )
+            WHERE rowid IN (SELECT CAST(value AS INTEGER) FROM json_each(?))
             RETURNING envelope, available_at, attempts, lease_id, lease_expires_at,
               published_at, last_failure, created_at
           `
@@ -63,10 +81,7 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
         .all(
           input.leaseId,
           input.leaseExpiresAt,
-          input.projectId,
-          input.now,
-          input.now,
-          input.limit
+          JSON.stringify(selected.map((row) => row.rowId))
         ) as SqliteOntologyOutboxRow[]
       return rows
         .map((row) => outboxRecord(row))
@@ -98,8 +113,9 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
             WITH requested(id) AS MATERIALIZED (
               SELECT CAST(value AS TEXT) FROM json_each(?)
             ), leased(id) AS MATERIALIZED (
-              SELECT outbox.id FROM ontology_outbox AS outbox
-              JOIN requested USING (id)
+              -- Drive validation from the bounded ID list even before ANALYZE.
+              SELECT outbox.id FROM requested
+              CROSS JOIN ontology_outbox AS outbox USING (id)
               WHERE outbox.project_id = ? AND outbox.lease_id = ?
                 AND outbox.lease_expires_at IS NOT NULL
             ), eligible(id) AS (
@@ -139,8 +155,9 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
             WITH requested(id) AS MATERIALIZED (
               SELECT CAST(value AS TEXT) FROM json_each(?)
             ), leased(id) AS MATERIALIZED (
-              SELECT outbox.id FROM ontology_outbox AS outbox
-              JOIN requested USING (id)
+              -- Drive validation from the bounded ID list even before ANALYZE.
+              SELECT outbox.id FROM requested
+              CROSS JOIN ontology_outbox AS outbox USING (id)
               WHERE outbox.project_id = ? AND outbox.lease_id = ?
                 AND outbox.lease_expires_at IS NOT NULL
             ), eligible(id) AS (

package/src/ontology-storage/shared.ts

@@ -62,6 +62,8 @@ export interface SqliteOntologySourceRow {
   readonly ownership_hash: string
   readonly ontology_revision: string
   readonly root_count: number | null
+  readonly base_materialization_id: string | null
+  readonly base_commit_id: string | null
   readonly assertion_count: number | null
   readonly created_at: string
   readonly ready_at: string | null
@@ -125,6 +127,14 @@ export function sourceRecord(row: SqliteOntologySourceRow): OntologySourceRecord
     projectionRevision: row.projection_revision,
     ownershipHash: row.ownership_hash,
     ontologyRevision: row.ontology_revision,
+    ...(row.base_materialization_id === null
+      ? {}
+      : {
+          base: {
+            materializationId: row.base_materialization_id,
+            lastCommitId: row.base_commit_id!,
+          },
+        }),
     rootCount: row.root_count,
     assertionCount: row.assertion_count,
     createdAt: row.created_at,

package/src/ontology-storage/source-cleanup.ts

@@ -0,0 +1,59 @@
+import type { Database } from "bun:sqlite"
+import type {
+  CleanupTerminalSourceMaterializationsInput,
+  CleanupTerminalSourceMaterializationsResult,
+} from "@sixb/core/storage"
+
+export function cleanupSourceVersions(
+  db: Database,
+  input: CleanupTerminalSourceMaterializationsInput
+): CleanupTerminalSourceMaterializationsResult {
+  let remaining = input.limit
+  let rowsDeleted = db
+    .query(`DELETE FROM ontology_source_rows WHERE rowid IN (
+    SELECT rows.rowid FROM ontology_source_roots AS roots
+    JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
+    CROSS JOIN ontology_source_rows AS rows
+      ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
+      AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
+    WHERE roots.project_id = ? AND roots.active = 0 AND roots.retired_at < ?
+      AND sources.status IN ('superseded', 'abandoned')
+    ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
+    LIMIT ?
+  )`)
+    .run(input.projectId, input.terminalBefore, remaining).changes
+  remaining -= rowsDeleted
+  if (remaining > 0) {
+    const removed = db
+      .query(`DELETE FROM ontology_source_roots WHERE rowid IN (
+      SELECT roots.rowid FROM ontology_source_roots AS roots
+      JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
+      WHERE roots.project_id = ? AND roots.active = 0 AND roots.retired_at < ?
+        AND sources.status IN ('superseded', 'abandoned')
+        AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
+          WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
+            AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
+      ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key LIMIT ?
+    )`)
+      .run(input.projectId, input.terminalBefore, remaining).changes
+    rowsDeleted += removed
+    remaining -= removed
+  }
+  const materializationsDeleted =
+    remaining === 0
+      ? 0
+      : db
+          .query(`DELETE FROM ontology_sources WHERE rowid IN (
+    SELECT sources.rowid FROM ontology_sources AS sources
+    WHERE project_id = ? AND status IN ('superseded', 'abandoned') AND terminal_at < ?
+      AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
+        WHERE roots.project_id = sources.project_id AND roots.source_id = sources.source_id
+          AND roots.materialization_id = sources.materialization_id)
+      AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
+        WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
+          AND rows.materialization_id = sources.materialization_id)
+    ORDER BY terminal_at, source_id, materialization_id LIMIT ?
+  )`)
+          .run(input.projectId, input.terminalBefore, remaining).changes
+  return { rowsDeleted, materializationsDeleted }
+}

package/src/ontology-storage/source-roots.ts

@@ -0,0 +1,230 @@
+import type { Database } from "bun:sqlite"
+import type { ProjectionEntityRef } from "@sixb/core/internal/materialization"
+import {
+  MaterializationConflictError,
+  MaterializationValidationError,
+  projectionEntityKey,
+} from "@sixb/core/internal/materialization"
+import { sourceStageRoots, utf8SortKey } from "@sixb/core/internal/ontology-storage-provider"
+import type { SourceActivationWrite, StageSourceRowsInput } from "@sixb/core/storage"
+import type { SqliteOntologySourceAssertionRow } from "./shared"
+import { canonicalJson, type SqliteOntologySourceRow } from "./shared"
+
+export function replacementSourceRows(input: {
+  readonly projectId: string
+  readonly sourceId: string
+  readonly materializationId: string
+  readonly incremental: boolean
+}): { readonly sql: string; readonly values: readonly string[] } {
+  const roots = input.incremental
+    ? `ontology_source_roots AS changed
+    CROSS JOIN ontology_source_roots AS roots ON roots.project_id = changed.project_id
+      AND roots.source_id = changed.source_id AND roots.root_sort_key = changed.root_sort_key AND roots.active = 1`
+    : "ontology_source_roots AS roots"
+  const selected = input.incremental ? "changed" : "roots"
+  return {
+    sql: `SELECT * FROM ontology_source_rows WHERE project_id = ? AND source_id = ? AND materialization_id = ?
+      UNION ALL
+      SELECT rows.* FROM ${roots}
+      CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id
+        AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
+        AND rows.root_sort_key = roots.root_sort_key
+      WHERE ${selected}.project_id = ? AND ${selected}.source_id = ?
+        AND ${input.incremental ? "changed.materialization_id = ?" : "roots.active = 1"}`,
+    values: [
+      input.projectId,
+      input.sourceId,
+      input.materializationId,
+      input.projectId,
+      input.sourceId,
+      ...(input.incremental ? [input.materializationId] : []),
+    ],
+  }
+}
+
+export function stageSourceRoots(
+  db: Database,
+  manifest: SqliteOntologySourceRow,
+  input: StageSourceRowsInput
+): void {
+  const roots = sourceStageRoots(manifest.projection_kind, input)
+  if (!manifest.base_materialization_id && roots.some((root) => root.deleted)) {
+    throw new MaterializationValidationError("Root deletions require a source delta base.")
+  }
+  const existing = db.query(`SELECT staging_ordinal, deleted FROM ontology_source_roots
+    WHERE project_id = ? AND source_id = ? AND materialization_id = ? AND root_sort_key = ?`)
+  const insert = db.query(`INSERT INTO ontology_source_roots (
+    project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
+  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING RETURNING root_sort_key`)
+  for (const root of roots) {
+    const inserted = insert.get(
+      input.projectId,
+      input.source.projectionId,
+      input.materializationId,
+      root.sortKey,
+      root.root.kind,
+      root.rootKey,
+      canonicalJson(root.root),
+      root.stagingOrdinal,
+      Number(root.deleted)
+    )
+    if (inserted) continue
+    const previous = existing.get(
+      input.projectId,
+      input.source.projectionId,
+      input.materializationId,
+      root.sortKey
+    ) as { staging_ordinal: number; deleted: number } | null
+    if (
+      !previous ||
+      previous.staging_ordinal !== root.stagingOrdinal ||
+      Boolean(previous.deleted) !== root.deleted
+    ) {
+      throw new MaterializationValidationError(
+        "Source materialization repeats root or stream ordinal with different content."
+      )
+    }
+  }
+}
+
+export function activateSourceRoots(
+  db: Database,
+  projectId: string,
+  candidate: SqliteOntologySourceRow,
+  activation: SourceActivationWrite
+): void {
+  if (
+    candidate.base_materialization_id !== null &&
+    (candidate.base_materialization_id !== activation.expected.activeMaterializationId ||
+      candidate.base_commit_id !== activation.expected.lastCommitId)
+  )
+    throw new MaterializationConflictError(
+      "projection-fence",
+      "Source delta base changed before activation."
+    )
+
+  if (candidate.base_materialization_id !== null && candidate.root_count === 0) return
+  const scope = [projectId, activation.source.projectionId, activation.materializationId] as const
+  db.query(`UPDATE ontology_source_roots AS roots SET active = 0, retired_at = ?
+    WHERE project_id = ? AND source_id = ? AND active = 1
+      ${
+        candidate.base_materialization_id === null
+          ? ""
+          : `AND root_sort_key IN (
+        SELECT root_sort_key FROM ontology_source_roots
+        WHERE project_id = ? AND source_id = ? AND materialization_id = ?)`
+      }`).run(
+    activation.updatedAt,
+    projectId,
+    activation.source.projectionId,
+    ...(candidate.base_materialization_id === null ? [] : [...scope])
+  )
+  db.query(`UPDATE ontology_source_roots SET active = CASE WHEN deleted = 1 THEN 0 ELSE 1 END,
+      retired_at = CASE WHEN deleted = 1 THEN ? ELSE NULL END
+    WHERE project_id = ? AND source_id = ? AND materialization_id = ?`).run(
+    activation.updatedAt,
+    ...scope
+  )
+}
+
+export function assertSourceRootCoverage(db: Database, manifest: SqliteOntologySourceRow): void {
+  const invalid = db
+    .query(`SELECT 1 FROM ontology_source_roots AS roots
+    WHERE project_id = ? AND source_id = ? AND materialization_id = ?
+      AND roots.deleted = EXISTS (
+        SELECT 1 FROM ontology_source_rows AS rows
+        WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
+          AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
+      ) LIMIT 1`)
+    .get(manifest.project_id, manifest.source_id, manifest.materialization_id)
+  if (invalid)
+    throw new MaterializationValidationError(
+      "Source roots must have complete assertions or an explicit deletion, never both."
+    )
+}
+
+export function activeSourceRows(
+  db: Database,
+  projectId: string,
+  refs: readonly ProjectionEntityRef[]
+): SqliteOntologySourceAssertionRow[] {
+  const keys = refs.flatMap((entity) => {
+    const entitySortKey = utf8SortKey(projectionEntityKey(entity))
+    const roots =
+      entity.kind === "object"
+        ? [entitySortKey]
+        : [
+            entitySortKey,
+            utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source })),
+          ]
+    return roots.map((rootSortKey) => ({ rootSortKey, entitySortKey }))
+  })
+  return db
+    .query(`WITH requested AS (
+    SELECT DISTINCT json_extract(value, '$.rootSortKey') AS root_sort_key,
+      json_extract(value, '$.entitySortKey') AS entity_sort_key FROM json_each(?)
+  ) SELECT rows.* FROM requested
+    CROSS JOIN ontology_source_roots AS roots ON roots.project_id = ?
+      AND roots.active = 1 AND roots.root_sort_key = requested.root_sort_key
+    CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
+      AND rows.materialization_id = roots.materialization_id AND rows.entity_sort_key = requested.entity_sort_key
+      AND rows.root_sort_key = roots.root_sort_key`)
+    .all(canonicalJson(keys), projectId) as SqliteOntologySourceAssertionRow[]
+}
+
+export function replacementAssertionRows(
+  db: Database,
+  input: {
+    projectId: string
+    sourceId: string
+    materializationId: string
+    incremental: boolean
+    includePrevious: boolean
+    refs: readonly ProjectionEntityRef[]
+  }
+): SqliteOntologySourceAssertionRow[] {
+  const keys = input.refs.map((ref) => ({
+    entityKind: ref.kind,
+    entityKey: projectionEntityKey(ref),
+    rootKeys:
+      ref.kind === "object"
+        ? [utf8SortKey(projectionEntityKey(ref))]
+        : [
+            utf8SortKey(projectionEntityKey(ref)),
+            utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source })),
+          ],
+  }))
+  const previous = !input.includePrevious
+    ? ""
+    : `UNION ALL
+    SELECT rows.* FROM requested_entities AS requested
+    CROSS JOIN ontology_source_roots AS roots ON roots.project_id = ? AND roots.source_id = ?
+      AND roots.active = 1 AND roots.root_sort_key IN (SELECT value FROM json_each(requested.root_keys))
+    CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
+      AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
+      AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
+    ${
+      !input.incremental
+        ? ""
+        : `WHERE EXISTS (SELECT 1 FROM ontology_source_roots AS changed
+      WHERE changed.project_id = roots.project_id AND changed.source_id = roots.source_id
+        AND changed.materialization_id = ? AND changed.root_sort_key = roots.root_sort_key)`
+    }`
+  return db
+    .query(`WITH requested_entities AS (
+    SELECT DISTINCT json_extract(value, '$.entityKind') AS entity_kind,
+      json_extract(value, '$.entityKey') AS entity_key, json_extract(value, '$.rootKeys') AS root_keys FROM json_each(?)
+  ) SELECT rows.* FROM requested_entities AS requested
+    CROSS JOIN ontology_source_rows AS rows ON rows.project_id = ? AND rows.source_id = ?
+      AND rows.materialization_id = ? AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
+    ${previous}`)
+    .all(
+      canonicalJson(keys),
+      input.projectId,
+      input.sourceId,
+      input.materializationId,
+      ...(input.includePrevious
+        ? [input.projectId, input.sourceId, ...(input.incremental ? [input.materializationId] : [])]
+        : [])
+    ) as SqliteOntologySourceAssertionRow[]
+}

package/src/ontology-storage/sources.ts

@@ -47,6 +47,8 @@ import {
   sourceAssertion,
   sourceRecord,
 } from "./shared"
+import { cleanupSourceVersions } from "./source-cleanup"
+import { assertSourceRootCoverage, stageSourceRoots } from "./source-roots"
 
 export class SqliteOntologySourceStorage implements OntologySourceStorage {
   constructor(
@@ -98,9 +100,9 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
               dataset_id, dataset_version_id, dataset_version_created_at,
               projection_revision, ownership_hash, ontology_revision,
               root_count, assertion_count, created_at, ready_at, activated_at,
-              terminal_at, last_commit_id, updated_at
+              terminal_at, last_commit_id, updated_at, base_materialization_id, base_commit_id
             ) VALUES (?, ?, ?, ?, ?, 'replacement', 'staging', ?, ?, ?, ?, ?, ?, ?,
-              NULL, NULL, ?, NULL, NULL, NULL, NULL, ?)
+              NULL, NULL, ?, NULL, NULL, NULL, NULL, ?, ?, ?)
           `
         )
         .run(
@@ -117,7 +119,9 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
           input.ownershipHash,
           input.ontologyRevision,
           input.createdAt,
-          input.createdAt
+          input.createdAt,
+          input.base?.materializationId ?? null,
+          input.base?.lastCommitId ?? null
         )
       return sourceRecord(
         this.requireManifest(input.projectId, input.source.projectionId, input.materializationId)
@@ -144,6 +148,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
       const rows = sourceStageRows(manifest.projection_kind, input.rows)
       const existing = this.findStageRows(input, rows)
       const { pending, unchanged } = reconcileSourceStageRows(rows, existing)
+      stageSourceRoots(this.db, manifest, input)
       this.insertStageRows(input, pending)
       return { inserted: pending.length, unchanged }
     })
@@ -249,78 +254,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
       assertNonblank(input.projectId, "Terminal source cleanup project id")
       assertTimestamp(input.terminalBefore, "Terminal source cleanup cutoff", true)
       assertPositiveInteger(input.limit, "Terminal source cleanup limit")
-      const manifests = this.db
-        .query(
-          `
-            SELECT * FROM ontology_sources
-            WHERE project_id = ? AND status IN ('superseded', 'abandoned')
-              AND terminal_at < ?
-            ORDER BY terminal_at, source_id, materialization_id
-            LIMIT ?
-          `
-        )
-        .all(input.projectId, input.terminalBefore, input.limit) as SqliteOntologySourceRow[]
-      let remaining = input.limit
-      let rowsDeleted = 0
-      let materializationsDeleted = 0
-      for (const manifest of manifests) {
-        if (remaining === 0) break
-        const deleted = this.db
-          .query(
-            `
-              DELETE FROM ontology_source_rows
-              WHERE rowid IN (
-                SELECT rows.rowid
-                FROM ontology_source_rows AS rows
-                JOIN ontology_sources AS sources
-                  ON sources.project_id = rows.project_id
-                 AND sources.source_id = rows.source_id
-                 AND sources.materialization_id = rows.materialization_id
-                WHERE rows.project_id = ? AND rows.source_id = ? AND rows.materialization_id = ?
-                  AND sources.status IN ('superseded', 'abandoned')
-                  AND sources.terminal_at = ? AND sources.terminal_at < ?
-                ORDER BY rows.entity_sort_key
-                LIMIT ?
-              )
-            `
-          )
-          .run(
-            manifest.project_id,
-            manifest.source_id,
-            manifest.materialization_id,
-            manifest.terminal_at,
-            input.terminalBefore,
-            remaining
-          ).changes
-        rowsDeleted += deleted
-        remaining -= deleted
-        if (remaining === 0) break
-        const removed = this.db
-          .query(
-            `
-              DELETE FROM ontology_sources
-              WHERE project_id = ? AND source_id = ? AND materialization_id = ?
-                AND status IN ('superseded', 'abandoned')
-                AND terminal_at = ? AND terminal_at < ?
-                AND NOT EXISTS (
-                  SELECT 1 FROM ontology_source_rows AS rows
-                  WHERE rows.project_id = ontology_sources.project_id
-                    AND rows.source_id = ontology_sources.source_id
-                    AND rows.materialization_id = ontology_sources.materialization_id
-                )
-            `
-          )
-          .run(
-            manifest.project_id,
-            manifest.source_id,
-            manifest.materialization_id,
-            manifest.terminal_at,
-            input.terminalBefore
-          ).changes
-        materializationsDeleted += removed
-        remaining -= removed
-      }
-      return { rowsDeleted, materializationsDeleted }
+      return cleanupSourceVersions(this.db, input)
     })
   }
 
@@ -433,6 +367,10 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
         `Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`
       )
     }
+    this.db
+      .query(`UPDATE ontology_source_roots SET retired_at = ?
+      WHERE project_id = ? AND source_id = ? AND materialization_id = ? AND active = 0`)
+      .run(abandonedAt, manifest.project_id, manifest.source_id, manifest.materialization_id)
     return sourceRecord(
       this.requireManifest(manifest.project_id, manifest.source_id, manifest.materialization_id)
     )
@@ -446,14 +384,22 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
     const counts = this.db
       .query(
         `
-          SELECT COUNT(*) AS assertions, COUNT(DISTINCT root_key) AS roots,
+          SELECT (SELECT COUNT(*) FROM ontology_source_rows
+            WHERE project_id = ? AND source_id = ? AND materialization_id = ?) AS assertions, COUNT(*) AS roots,
             COUNT(DISTINCT staging_ordinal) AS ordinals,
             MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
-          FROM ontology_source_rows
+          FROM ontology_source_roots
           WHERE project_id = ? AND source_id = ? AND materialization_id = ?
         `
       )
-      .get(manifest.project_id, manifest.source_id, manifest.materialization_id) as {
+      .get(
+        manifest.project_id,
+        manifest.source_id,
+        manifest.materialization_id,
+        manifest.project_id,
+        manifest.source_id,
+        manifest.materialization_id
+      ) as {
       assertions: number
       roots: number
       ordinals: number
@@ -470,6 +416,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
         "Source ready counts do not match the staged roots and assertions."
       )
     }
+    assertSourceRootCoverage(this.db, manifest)
     const invalid = this.db
       .query(
         manifest.projection_kind === "link"

package/src/projection-run-storage.ts

@@ -114,7 +114,7 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
                 input_exhausted = ?, missing_target_object_type_id = NULL,
                 missing_target_object_id = NULL, missing_target_batch_ordinal = NULL,
                 missing_target_first_seen_at = NULL, source_rows_read = 0,
-                source_rows_skipped = 0, error = NULL
+                source_rows_skipped = 0, source_changes_read = NULL, error = NULL
               WHERE project_id = ? AND id = ? AND status = 'failed'
                 AND json_extract(error, '$.code') = 'queue.enqueue_failed'
             `
@@ -257,6 +257,7 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
             execution_token = NULL,
             source_rows_read = ?,
             source_rows_skipped = ?,
+            source_changes_read = ?,
             input_exhausted = ?,
             error = ?
           WHERE project_id = ? AND id = ? AND status = 'running' AND execution_token = ?
@@ -267,6 +268,7 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
           plan.finishedAt.toISOString(),
           plan.progress.sourceRowsRead,
           plan.progress.sourceRowsSkipped,
+          plan.progress.sourceChangesRead ?? null,
           plan.inputExhausted ? 1 : existingRow.input_exhausted,
           plan.error === undefined
             ? null
@@ -531,9 +533,10 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
     progress: ProjectionRunProgress,
     executionToken: string
   ): number {
-    const args: (string | number)[] = [
+    const args: (string | number | null)[] = [
       progress.sourceRowsRead,
       progress.sourceRowsSkipped,
+      progress.sourceChangesRead ?? null,
       existing.project_id,
       existing.id,
       executionToken,
@@ -544,7 +547,8 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
         UPDATE projection_runs
         SET
           source_rows_read = ?,
-          source_rows_skipped = ?
+          source_rows_skipped = ?,
+          source_changes_read = ?
         WHERE project_id = ? AND id = ? AND status = 'running'
           AND execution_token = ?
       `
@@ -631,6 +635,9 @@ function restoreProjectionRunRow(row: DatabaseRow): StoredProjectionRunRecord {
     progress: {
       sourceRowsRead: databaseSafeInteger(row.source_rows_read, "sourceRowsRead"),
       sourceRowsSkipped: databaseSafeInteger(row.source_rows_skipped, "sourceRowsSkipped"),
+      ...(row.source_changes_read == null
+        ? {}
+        : { sourceChangesRead: databaseSafeInteger(row.source_changes_read, "sourceChangesRead") }),
     },
     error: row.error ?? undefined,
   }
@@ -694,6 +701,7 @@ interface DatabaseRow {
   missing_target_first_seen_at: string | null
   source_rows_read: number
   source_rows_skipped: number
+  source_changes_read: number | null
   error: string | null
 }