@sixb/sqlite 0.1.14 → 0.1.15

package/src/migrations.ts

@@ -115,6 +115,12 @@ import nativeSessionsSql from "./migrations/049-native-sessions.sql" with { type
 import directoryGroupMembershipsSql from "./migrations/050-directory-group-memberships.sql" with {
   type: "text",
 }
+import projectionRunSupersessionSql from "./migrations/051-projection-run-supersession.sql" with {
+  type: "text",
+}
+import compactSourceStorageSql from "./migrations/052-compact-source-storage.sql" with {
+  type: "text",
+}
 
 const MIGRATIONS_TABLE_SQL = `
   CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -220,6 +226,26 @@ export const sqliteStorageMigrations = defineMigrations({
     sqliteSql("048-file-upload-sessions", fileUploadSessionsSql),
     sqliteSql("049-native-sessions", nativeSessionsSql),
     sqliteSql("050-directory-group-memberships", directoryGroupMembershipsSql),
+    sqliteSql("051-projection-run-supersession", projectionRunSupersessionSql),
+    sqliteStep(
+      "052-compact-source-storage",
+      (db) => {
+        // SQLite has no procedural block to check the copy in SQL: count around it instead.
+        const before = sourceStorageCounts(db)
+        db.run(compactSourceStorageSql)
+        const after = sourceStorageCounts(db)
+        if (
+          after.versions !== before.versions ||
+          after.roots !== before.roots ||
+          after.rows !== before.rows
+        ) {
+          throw new Error(
+            "[SixbSqliteStorage] Source versions, roots or rows were lost while compacting source storage."
+          )
+        }
+      },
+      { checksum: checksum(compactSourceStorageSql) }
+    ),
   ],
 })
 
@@ -378,6 +404,18 @@ function rowToMigrationRecord(row: unknown): MigrationRecord {
   }
 }
 
+function sourceStorageCounts(db: Database): {
+  readonly versions: number
+  readonly roots: number
+  readonly rows: number
+} {
+  return db
+    .query(`SELECT (SELECT count(*) FROM ontology_sources) AS versions,
+      (SELECT count(*) FROM ontology_source_roots) AS roots,
+      (SELECT count(*) FROM ontology_source_rows) AS rows`)
+    .get() as { readonly versions: number; readonly roots: number; readonly rows: number }
+}
+
 function checksum(value: string): string {
   return createHash("sha256").update(value).digest("hex")
 }

package/src/ontology-storage/materialization-state.ts

@@ -43,8 +43,15 @@ import {
   parseJson,
   type SqliteOntologySourceAssertionRow,
   type SqliteStoredOverrideRow,
+  sourceAssertion,
 } from "./shared"
-import { activeSourceRows, replacementAssertionRows, replacementSourceRows } from "./source-roots"
+import {
+  activeSourceRows,
+  LIVE_SOURCE_ROWS_JOIN,
+  replacementAssertionRows,
+  replacementSourceRows,
+  SOURCE_ASSERTION_COLUMNS,
+} from "./source-roots"
 
 export const SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work"
 export const SQLITE_REPLACEMENT_WORK_TABLE = "ontology_replacement_work"
@@ -438,19 +445,13 @@ export class SqliteMaterializationStateReader {
              json_extract(value, '$.linkId') AS link_id
            FROM json_each(?)
          )
-         SELECT rows.* FROM requested
+         SELECT ${SOURCE_ASSERTION_COLUMNS} FROM requested
          CROSS JOIN ontology_source_rows AS rows
-           ON rows.project_id = ?
-          AND rows.entity_kind = 'link'
+           ON rows.entity_kind = 'link'
           AND rows.source_type_id = requested.source_type_id
           AND rows.source_primary_id = requested.source_primary_id
           AND rows.link_id = requested.link_id
-         JOIN ontology_source_roots AS sources
-           ON sources.project_id = rows.project_id
-          AND sources.source_id = rows.source_id
-          AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-         WHERE sources.active = 1`
+         ${LIVE_SOURCE_ROWS_JOIN}`
       )
       .all(requestedJson, this.projectId) as SqliteOntologySourceAssertionRow[]
     const slotOverrideRows = this.db
@@ -570,12 +571,8 @@ export class SqliteMaterializationStateReader {
               SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
                 rows.target_type_id, rows.target_primary_id
               FROM ontology_source_rows AS rows
-              JOIN ontology_source_roots AS sources
-                ON sources.project_id = rows.project_id
-               AND sources.source_id = rows.source_id
-               AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-              WHERE rows.project_id = ? AND rows.entity_kind = 'link' AND sources.active = 1
+              ${LIVE_SOURCE_ROWS_JOIN}
+              WHERE rows.entity_kind = 'link'
             ), selected AS (
               SELECT DISTINCT links.*,
                 ${linkSortExpression("links")} AS sort_key
@@ -752,7 +749,7 @@ export class SqliteMaterializationStateReader {
   }
 
   private prepareReplacementObjects(input: ReplacementIdentityInput): void {
-    const rows = replacementSourceRows({
+    const rows = replacementSourceRows(this.db, {
       projectId: this.projectId,
       sourceId: input.sourceId,
       materializationId: input.candidateMaterializationId,
@@ -765,7 +762,7 @@ export class SqliteMaterializationStateReader {
             session_id, entity_kind, identity_key, sort_key, diff_required
           )
           SELECT ?, 'object', json_array(object_type_id, primary_id),
-            MIN(entity_sort_key), 1
+            ${objectSortExpression()}, 1
           FROM (${rows.sql}) AS rows
           WHERE entity_kind = 'object'
           GROUP BY object_type_id, primary_id
@@ -775,7 +772,7 @@ export class SqliteMaterializationStateReader {
   }
 
   private prepareReplacementLinks(input: ReplacementIdentityInput): void {
-    const rows = replacementSourceRows({
+    const rows = replacementSourceRows(this.db, {
       projectId: this.projectId,
       sourceId: input.sourceId,
       materializationId: input.candidateMaterializationId,
@@ -834,31 +831,21 @@ export class SqliteMaterializationStateReader {
             UNION
             SELECT rows.source_type_id, rows.source_primary_id AS source_id,
               rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
-            FROM ontology_source_rows AS rows
-            JOIN ontology_source_roots AS sources
-              ON sources.project_id = rows.project_id
-             AND sources.source_id = rows.source_id
-             AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-            JOIN incident_objects
-              ON incident_objects.object_type_id = rows.source_type_id
-             AND incident_objects.primary_id = rows.source_primary_id
-            WHERE rows.project_id = ? AND rows.entity_kind = 'link'
-              AND sources.active = 1
+            FROM incident_objects
+            CROSS JOIN ontology_source_rows AS rows
+              ON rows.entity_kind = 'link'
+             AND rows.source_type_id = incident_objects.object_type_id
+             AND rows.source_primary_id = incident_objects.primary_id
+            ${LIVE_SOURCE_ROWS_JOIN}
             UNION
             SELECT rows.source_type_id, rows.source_primary_id AS source_id,
               rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
-            FROM ontology_source_rows AS rows
-            JOIN ontology_source_roots AS sources
-              ON sources.project_id = rows.project_id
-             AND sources.source_id = rows.source_id
-             AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-            JOIN incident_objects
-              ON incident_objects.object_type_id = rows.target_type_id
-             AND incident_objects.primary_id = rows.target_primary_id
-            WHERE rows.project_id = ? AND rows.entity_kind = 'link'
-              AND sources.active = 1
+            FROM incident_objects
+            CROSS JOIN ontology_source_rows AS rows
+              ON rows.entity_kind = 'link'
+             AND rows.target_type_id = incident_objects.object_type_id
+             AND rows.target_primary_id = incident_objects.primary_id
+            ${LIVE_SOURCE_ROWS_JOIN}
           ), diff_links AS (
             SELECT * FROM replacement_links
             UNION SELECT * FROM incident_links
@@ -1062,9 +1049,7 @@ function storedSource(row: SqliteOntologySourceAssertionRow): StoredSourceAssert
   return {
     source: { projectionId: row.source_id },
     materializationId: row.materialization_id,
-    root: parseJson<StoredSourceAssertion["root"]>(row.root),
-    assertion: parseJson<StoredSourceAssertion["assertion"]>(row.assertion),
-    stagingOrdinal: row.staging_ordinal,
+    ...sourceAssertion(row),
   } as StoredSourceAssertion
 }
 
@@ -1130,6 +1115,11 @@ function storedPoint(row: TelemetryRow): StoredTelemetryPoint {
   }
 }
 
+function objectSortExpression(alias?: string): string {
+  const prefix = alias ? `${alias}.` : ""
+  return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}primary_id) AS BLOB)))`
+}
+
 export function linkSortExpression(alias?: string): string {
   const prefix = alias ? `${alias}.` : ""
   return `LOWER(HEX(CAST(json_array(${prefix}source_type_id, ${prefix}source_id, ${prefix}link_id, ${prefix}target_type_id, ${prefix}target_id) AS BLOB)))`

package/src/ontology-storage/shared.ts

@@ -1,7 +1,10 @@
 import type { Database } from "bun:sqlite"
 import { parseSixbFailure } from "@sixb/core/internal/errors"
-import type { ProjectionEntityRef } from "@sixb/core/internal/materialization"
 import { MaterializationConflictError } from "@sixb/core/internal/materialization"
+import {
+  sourceAssertionFromColumns,
+  sourceEntityFromKey,
+} from "@sixb/core/internal/ontology-storage-provider"
 import type {
   AssertSourceMaterializationExecutionInput,
   OntologyCommitRecord,
@@ -48,6 +51,7 @@ export interface SqliteOntologyCommitRow {
 }
 
 export interface SqliteOntologySourceRow {
+  readonly version_id: number
   readonly project_id: string
   readonly source_id: string
   readonly materialization_id: string
@@ -74,19 +78,21 @@ export interface SqliteOntologySourceRow {
   readonly updated_at: string
 }
 
+/** An assertion with its root and version, as `SOURCE_ASSERTION_COLUMNS` selects it. */
 export interface SqliteOntologySourceAssertionRow {
-  readonly project_id: string
   readonly source_id: string
   readonly materialization_id: string
-  readonly entity_kind: "object" | "link"
-  readonly entity_key: string
-  readonly entity_sort_key: string
-  readonly root_kind: "object" | "link"
   readonly root_key: string
-  readonly root_sort_key: string
   readonly staging_ordinal: number
-  readonly root: string
-  readonly assertion: string
+  readonly entity_kind: "object" | "link"
+  readonly object_type_id: string | null
+  readonly primary_id: string | null
+  readonly source_type_id: string | null
+  readonly source_primary_id: string | null
+  readonly link_id: string | null
+  readonly target_type_id: string | null
+  readonly target_primary_id: string | null
+  readonly payload: string | null
 }
 
 export interface SqliteStoredOverrideRow {
@@ -149,8 +155,11 @@ export function sourceRecord(row: SqliteOntologySourceRow): OntologySourceRecord
 
 export function sourceAssertion(row: SqliteOntologySourceAssertionRow): StageSourceAssertion {
   return {
-    root: parseJson<ProjectionEntityRef>(row.root),
-    assertion: parseJson<StageSourceAssertion["assertion"]>(row.assertion),
+    root: sourceEntityFromKey(row.root_key),
+    assertion: sourceAssertionFromColumns(
+      row,
+      row.payload === null ? null : parseJson<unknown>(row.payload)
+    ),
     stagingOrdinal: row.staging_ordinal,
   }
 }

package/src/ontology-storage/source-cleanup.ts

@@ -2,38 +2,79 @@ import type { Database } from "bun:sqlite"
 import type {
   CleanupTerminalSourceMaterializationsInput,
   CleanupTerminalSourceMaterializationsResult,
+  PurgeAbandonedSourceMaterializationsInput,
 } from "@sixb/core/storage"
 
+/**
+ * Superseded versions can still hold live roots (a delta keeps unchanged roots where they were),
+ * so they are cleaned root by root once retired. An abandoned candidate was never published:
+ * everything it holds goes, one whole version at a time, each delete on its own version prefix.
+ */
+export function purgeAbandonedSourceVersions(
+  db: Database,
+  input: PurgeAbandonedSourceMaterializationsInput
+): CleanupTerminalSourceMaterializationsResult {
+  let remaining = input.limit
+  let rowsDeleted = 0
+  let materializationsDeleted = 0
+  const nextCandidate = db.query(`
+    SELECT version_id FROM ontology_sources
+    WHERE project_id = ? AND status = 'abandoned'
+    ORDER BY terminal_at, source_id, materialization_id LIMIT 1
+  `)
+  const deleteRows = db.query(`DELETE FROM ontology_source_rows WHERE rowid IN (
+    SELECT rows.rowid FROM ontology_source_roots AS roots
+    CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
+    WHERE roots.version_id = ? LIMIT ?)`)
+  const deleteRoots = db.query(`DELETE FROM ontology_source_roots WHERE id IN (
+    SELECT id FROM ontology_source_roots WHERE version_id = ? LIMIT ?)`)
+  const deleteManifest = db.query(
+    "DELETE FROM ontology_sources WHERE version_id = ? AND status = 'abandoned'"
+  )
+  while (remaining > 0) {
+    const candidate = nextCandidate.get(input.projectId) as { readonly version_id: number } | null
+    if (!candidate) break
+    const rows = deleteRows.run(candidate.version_id, remaining).changes
+    rowsDeleted += rows
+    remaining -= rows
+    if (remaining === 0) break
+    const roots = deleteRoots.run(candidate.version_id, remaining).changes
+    rowsDeleted += roots
+    remaining -= roots
+    if (remaining === 0) break
+    const manifest = deleteManifest.run(candidate.version_id).changes
+    if (manifest === 0) break
+    materializationsDeleted += manifest
+    remaining -= manifest
+  }
+  return { rowsDeleted, materializationsDeleted }
+}
+
 export function cleanupSourceVersions(
   db: Database,
   input: CleanupTerminalSourceMaterializationsInput
 ): CleanupTerminalSourceMaterializationsResult {
   let remaining = input.limit
+  const retired = `roots.project_id = ? AND roots.retired_at < ? AND versions.status = 'superseded'`
   let rowsDeleted = db
     .query(`DELETE FROM ontology_source_rows WHERE rowid IN (
     SELECT rows.rowid FROM ontology_source_roots AS roots
-    JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
-    CROSS JOIN ontology_source_rows AS rows
-      ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-      AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
-    WHERE roots.project_id = ? AND roots.active = 0 AND roots.retired_at < ?
-      AND sources.status IN ('superseded', 'abandoned')
-    ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
+    JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+    CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
+    WHERE ${retired}
+    ORDER BY roots.retired_at, roots.id
     LIMIT ?
   )`)
     .run(input.projectId, input.terminalBefore, remaining).changes
   remaining -= rowsDeleted
   if (remaining > 0) {
     const removed = db
-      .query(`DELETE FROM ontology_source_roots WHERE rowid IN (
-      SELECT roots.rowid FROM ontology_source_roots AS roots
-      JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
-      WHERE roots.project_id = ? AND roots.active = 0 AND roots.retired_at < ?
-        AND sources.status IN ('superseded', 'abandoned')
-        AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
-          WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-            AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
-      ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key LIMIT ?
+      .query(`DELETE FROM ontology_source_roots WHERE id IN (
+      SELECT roots.id FROM ontology_source_roots AS roots
+      JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+      WHERE ${retired}
+        AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id)
+      ORDER BY roots.retired_at, roots.id LIMIT ?
     )`)
       .run(input.projectId, input.terminalBefore, remaining).changes
     rowsDeleted += removed
@@ -43,15 +84,11 @@ export function cleanupSourceVersions(
     remaining === 0
       ? 0
       : db
-          .query(`DELETE FROM ontology_sources WHERE rowid IN (
-    SELECT sources.rowid FROM ontology_sources AS sources
-    WHERE project_id = ? AND status IN ('superseded', 'abandoned') AND terminal_at < ?
+          .query(`DELETE FROM ontology_sources WHERE version_id IN (
+    SELECT versions.version_id FROM ontology_sources AS versions
+    WHERE project_id = ? AND status = 'superseded' AND terminal_at < ?
       AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
-        WHERE roots.project_id = sources.project_id AND roots.source_id = sources.source_id
-          AND roots.materialization_id = sources.materialization_id)
-      AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
-        WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
-          AND rows.materialization_id = sources.materialization_id)
+        WHERE roots.version_id = versions.version_id)
     ORDER BY terminal_at, source_id, materialization_id LIMIT ?
   )`)
           .run(input.projectId, input.terminalBefore, remaining).changes