@sixb/pg 0.1.14 → 0.1.16

package/src/migrations.ts

@@ -109,7 +109,14 @@ import nativeSessionsSql from "./migrations/049-native-sessions.sql" with { type
 import directoryGroupMembershipsSql from "./migrations/050-directory-group-memberships.sql" with {
   type: "text",
 }
-import type { SQL, SQLClient } from "./pg-client"
+import projectionRunSupersessionSql from "./migrations/051-projection-run-supersession.sql" with {
+  type: "text",
+}
+import compactSourceStorageSql from "./migrations/052-compact-source-storage.sql" with {
+  type: "text",
+}
+import type { ReservedSQL, SQL, SQLClient } from "./pg-client"
+import { runPgTransactionOn, undoOnFailure, withReservedPgConnection } from "./transactions"
 
 export interface PostgresMigrationContext {
   exec(sqlText: string): Promise<void>
@@ -189,7 +196,7 @@ export function quoteIdent(identifier: string): string {
 }
 
 function postgresMigrationSession(
-  sql: SQL,
+  sql: ReservedSQL,
   schemaName: string
 ): { context: PostgresMigrationContext; state: MigrationHistoryStore } {
   const schema = quoteIdent(schemaName)
@@ -256,19 +263,15 @@ function postgresMigrationSession(
       },
       async transaction(run) {
         // `sql` is the reserved connection holding the migration advisory lock. porsager's
-        // reserved connection has no `.begin`, so drive the transaction manually on it and
-        // route exec()/markStarted()/markApplied() through the same connection via `active`.
+        // `.begin` would open the transaction on another pool connection, so drive it on this one
+        // and route exec()/markStarted()/markApplied() through it via `active`.
         const previous = active
         active = sql
-        await sql.unsafe("BEGIN")
         try {
-          await sql.unsafe(`SET LOCAL search_path TO ${schema}`)
-          const result = await run()
-          await sql.unsafe("COMMIT")
-          return result
-        } catch (error) {
-          await sql.unsafe("ROLLBACK")
-          throw error
+          return await runPgTransactionOn(sql, async () => {
+            await sql.unsafe(`SET LOCAL search_path TO ${schema}`)
+            return run()
+          })
         } finally {
           active = previous
         }
@@ -315,23 +318,21 @@ async function withPostgresMigrationLock<T>(
     readonly schemaName: string
     readonly migrations: MigrationSet<PostgresMigrationContext>
   },
-  run: (sql: SQL) => Promise<T>
+  run: (sql: ReservedSQL) => Promise<T>
 ): Promise<T> {
-  const sql = await params.sql.reserve()
   const [first, second] = advisoryLockParts(
     `storage:migration:${params.schemaName}:${params.migrations.adapterId}`
   )
 
-  try {
+  // A session-level lock outlives any transaction, so it is released before the connection goes
+  // back to the pool — except when the connection was lost, which already released it.
+  return withReservedPgConnection(params.sql, async (sql) => {
+    const unlock = () => sql`SELECT pg_advisory_unlock(${first}, ${second})`
     await sql`SELECT pg_advisory_lock(${first}, ${second})`
-    try {
-      return await run(sql)
-    } finally {
-      await sql`SELECT pg_advisory_unlock(${first}, ${second})`
-    }
-  } finally {
-    sql.release()
-  }
+    const result = await undoOnFailure(() => run(sql), unlock)
+    await unlock()
+    return result
+  })
 }
 
 function advisoryLockParts(key: string): readonly [number, number] {
@@ -412,6 +413,8 @@ export const postgresStorageMigrations = defineMigrations<PostgresMigrationConte
     pgSql("048-file-upload-sessions", fileUploadSessionsSql),
     pgSql("049-native-sessions", nativeSessionsSql),
     pgSql("050-directory-group-memberships", directoryGroupMembershipsSql),
+    pgSql("051-projection-run-supersession", projectionRunSupersessionSql),
+    pgSql("052-compact-source-storage", compactSourceStorageSql),
   ],
 })
 

package/src/objects/query-compiler.ts

@@ -110,7 +110,7 @@ export function compilePgObjectQuery(
   if (hasVectorProfile(query) && !isVectorProfileQuery(query)) {
     throw new ObjectQueryExecutionError(
       "unsupported_vector_composition",
-      "Vector search supports one profile with filters before ranking and limit/project after."
+      "Vector search supports one profile with filters, traversals or sets before ranking and limit/project after."
     )
   }
   const source = options.source ?? DEFAULT_OBJECT_QUERY_SOURCE

package/src/ontology-storage/materialization-state.ts

@@ -38,15 +38,21 @@ import type {
 } from "@sixb/core/storage"
 import type { SQLClient } from "../pg-client"
 import {
-  databaseSafeInteger,
   jsonParameter,
   linkRefFromColumns,
   objectRefFromColumns,
   type PgOntologySourceAssertionRow,
   type PgStoredOverrideRow,
+  sourceAssertion,
   toIsoString,
 } from "./shared"
-import { activeSourceRows, replacementAssertionRows, replacementSourceRows } from "./source-roots"
+import {
+  activeSourceRows,
+  liveSourceRowsJoin,
+  replacementAssertionRows,
+  replacementSourceRows,
+  sourceAssertionColumns,
+} from "./source-roots"
 
 export const PG_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work"
 export const PG_REPLACEMENT_WORK_TABLE = "ontology_replacement_work"
@@ -410,20 +416,14 @@ export class PgMaterializationStateReader {
               scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
             )
         )
-        SELECT rows.*
+        SELECT ${sourceAssertionColumns(this.sql)}
         FROM ontology_source_rows AS rows
         JOIN requested
           ON requested.source_type_id = rows.source_type_id
          AND requested.source_id = rows.source_primary_id
          AND requested.link_id = rows.link_id
-        JOIN ontology_source_roots AS sources
-          ON sources.project_id = rows.project_id
-         AND sources.source_id = rows.source_id
-         AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
-        WHERE rows.project_id = ${this.projectId}
-          AND rows.entity_kind = 'link'
-          AND sources.active
+        ${liveSourceRowsJoin(this.sql, this.projectId)}
+        WHERE rows.entity_kind = 'link'
       `,
       this.sql<LinkSlotOverrideRow[]>`
         WITH requested AS (
@@ -582,32 +582,20 @@ export class PgMaterializationStateReader {
           SELECT rows.source_type_id, rows.source_primary_id AS source_id,
             rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
           FROM ontology_source_rows AS rows
-          JOIN ontology_source_roots AS sources
-            ON sources.project_id = rows.project_id
-           AND sources.source_id = rows.source_id
-           AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
+          ${liveSourceRowsJoin(this.sql, this.projectId)}
           JOIN requested
             ON requested.object_type_id = rows.source_type_id
            AND requested.primary_id = rows.source_primary_id
-          WHERE rows.project_id = ${this.projectId}
-            AND rows.entity_kind = 'link'
-            AND sources.active
+          WHERE rows.entity_kind = 'link'
           UNION
           SELECT rows.source_type_id, rows.source_primary_id AS source_id,
             rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
           FROM ontology_source_rows AS rows
-          JOIN ontology_source_roots AS sources
-            ON sources.project_id = rows.project_id
-           AND sources.source_id = rows.source_id
-           AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
+          ${liveSourceRowsJoin(this.sql, this.projectId)}
           JOIN requested
             ON requested.object_type_id = rows.target_type_id
            AND requested.primary_id = rows.target_primary_id
-          WHERE rows.project_id = ${this.projectId}
-            AND rows.entity_kind = 'link'
-            AND sources.active
+          WHERE rows.entity_kind = 'link'
         ), selected AS (
           SELECT DISTINCT links.*,
             ${this.sql.unsafe(linkSortExpression("links"))} AS sort_key
@@ -912,32 +900,20 @@ export class PgMaterializationStateReader {
         SELECT rows.source_type_id, rows.source_primary_id AS source_id,
           rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
         FROM ontology_source_rows AS rows
-        JOIN ontology_source_roots AS sources
-          ON sources.project_id = rows.project_id
-         AND sources.source_id = rows.source_id
-         AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
+        ${liveSourceRowsJoin(this.sql, this.projectId)}
         JOIN incident_objects
           ON incident_objects.object_type_id = rows.source_type_id
          AND incident_objects.primary_id = rows.source_primary_id
-        WHERE rows.project_id = ${this.projectId}
-          AND rows.entity_kind = 'link'
-          AND sources.active
+        WHERE rows.entity_kind = 'link'
         UNION
         SELECT rows.source_type_id, rows.source_primary_id AS source_id,
           rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
         FROM ontology_source_rows AS rows
-        JOIN ontology_source_roots AS sources
-          ON sources.project_id = rows.project_id
-         AND sources.source_id = rows.source_id
-         AND sources.materialization_id = rows.materialization_id
-         AND sources.root_sort_key = rows.root_sort_key
+        ${liveSourceRowsJoin(this.sql, this.projectId)}
         JOIN incident_objects
           ON incident_objects.object_type_id = rows.target_type_id
          AND incident_objects.primary_id = rows.target_primary_id
-        WHERE rows.project_id = ${this.projectId}
-          AND rows.entity_kind = 'link'
-          AND sources.active
+        WHERE rows.entity_kind = 'link'
       ), diff_links AS (
         SELECT * FROM replacement_links
         UNION SELECT * FROM incident_links
@@ -1104,9 +1080,7 @@ function storedSource(row: PgOntologySourceAssertionRow): StoredSourceAssertion
   return {
     source: { projectionId: row.source_id },
     materializationId: row.materialization_id,
-    root: structuredClone(row.root) as StoredSourceAssertion["root"],
-    assertion: structuredClone(row.assertion) as StoredSourceAssertion["assertion"],
-    stagingOrdinal: databaseSafeInteger(row.staging_ordinal, "Source staging ordinal"),
+    ...sourceAssertion(row),
   } as StoredSourceAssertion
 }
 

package/src/ontology-storage/shared.ts

@@ -1,6 +1,9 @@
 import { parseSixbFailure } from "@sixb/core/internal/errors"
-import type { ProjectionEntityRef } from "@sixb/core/internal/materialization"
 import { MaterializationConflictError } from "@sixb/core/internal/materialization"
+import {
+  sourceAssertionFromColumns,
+  sourceEntityFromKey,
+} from "@sixb/core/internal/ontology-storage-provider"
 import type {
   AssertSourceMaterializationExecutionInput,
   OntologyCommitRecord,
@@ -50,6 +53,7 @@ export interface PgOntologyCommitRow {
 }
 
 export interface PgOntologySourceRow {
+  readonly version_id: string
   readonly project_id: string
   readonly source_id: string
   readonly materialization_id: string
@@ -76,19 +80,21 @@ export interface PgOntologySourceRow {
   readonly updated_at: Date | string
 }
 
+/** An assertion with its root and version, as `sourceAssertionColumns` selects it. */
 export interface PgOntologySourceAssertionRow {
-  readonly project_id: string
   readonly source_id: string
   readonly materialization_id: string
-  readonly entity_kind: "object" | "link"
-  readonly entity_key: unknown
-  readonly entity_sort_key: string
-  readonly root_kind: "object" | "link"
-  readonly root_key: unknown
-  readonly root_sort_key: string
+  readonly root_key: string
   readonly staging_ordinal: number | string
-  readonly root: unknown
-  readonly assertion: unknown
+  readonly entity_kind: "object" | "link"
+  readonly object_type_id: string | null
+  readonly primary_id: string | null
+  readonly source_type_id: string | null
+  readonly source_primary_id: string | null
+  readonly link_id: string | null
+  readonly target_type_id: string | null
+  readonly target_primary_id: string | null
+  readonly payload: unknown
 }
 
 export interface PgStoredOverrideRow {
@@ -174,8 +180,8 @@ export function sourceRecord(row: PgOntologySourceRow): OntologySourceRecord {
 
 export function sourceAssertion(row: PgOntologySourceAssertionRow): StageSourceAssertion {
   return {
-    root: structuredClone(row.root) as ProjectionEntityRef,
-    assertion: structuredClone(row.assertion) as StageSourceAssertion["assertion"],
+    root: sourceEntityFromKey(row.root_key),
+    assertion: sourceAssertionFromColumns(row, row.payload),
     stagingOrdinal: databaseSafeInteger(row.staging_ordinal, "Source staging ordinal"),
   }
 }

package/src/ontology-storage/source-cleanup.ts

@@ -1,24 +1,74 @@
 import type {
   CleanupTerminalSourceMaterializationsInput,
   CleanupTerminalSourceMaterializationsResult,
+  PurgeAbandonedSourceMaterializationsInput,
 } from "@sixb/core/storage"
 import type { SQLClient } from "../pg-client"
 
+/**
+ * Superseded versions can still hold live roots (a delta keeps unchanged roots where they were),
+ * so they are cleaned root by root once retired. An abandoned candidate was never published:
+ * everything it holds goes, one whole version at a time, each delete on its own version prefix.
+ */
+export async function purgeAbandonedSourceVersions(
+  sql: SQLClient,
+  input: PurgeAbandonedSourceMaterializationsInput
+): Promise<CleanupTerminalSourceMaterializationsResult> {
+  let remaining = input.limit
+  let rowsDeleted = 0
+  let materializationsDeleted = 0
+  while (remaining > 0) {
+    const [candidate] = await sql<{ readonly version_id: string }[]>`
+      SELECT version_id FROM ontology_sources
+      WHERE project_id = ${input.projectId} AND status = 'abandoned'
+      ORDER BY terminal_at, source_id, materialization_id
+      LIMIT 1 FOR UPDATE SKIP LOCKED
+    `
+    if (!candidate) break
+    const rows = await sql`
+      DELETE FROM ontology_source_rows WHERE ctid IN (
+        SELECT rows.ctid FROM ontology_source_roots AS roots
+        JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
+        WHERE roots.version_id = ${candidate.version_id} LIMIT ${remaining}
+      )
+    `
+    rowsDeleted += rows.count
+    remaining -= rows.count
+    if (remaining === 0) break
+    const roots = await sql`
+      DELETE FROM ontology_source_roots WHERE ctid IN (
+        SELECT ctid FROM ontology_source_roots
+        WHERE version_id = ${candidate.version_id} LIMIT ${remaining}
+      )
+    `
+    rowsDeleted += roots.count
+    remaining -= roots.count
+    if (remaining === 0) break
+    const manifest = await sql`
+      DELETE FROM ontology_sources
+      WHERE version_id = ${candidate.version_id} AND status = 'abandoned'
+    `
+    if (manifest.count === 0) break
+    materializationsDeleted += manifest.count
+    remaining -= manifest.count
+  }
+  return { rowsDeleted, materializationsDeleted }
+}
+
 export async function cleanupSourceVersions(
   sql: SQLClient,
   input: CleanupTerminalSourceMaterializationsInput
 ): Promise<CleanupTerminalSourceMaterializationsResult> {
   let remaining = input.limit
+  const retired = sql`roots.project_id = ${input.projectId}
+    AND roots.retired_at < ${input.terminalBefore} AND versions.status = 'superseded'`
   const removedRows = await sql`
     WITH selected AS (
       SELECT rows.ctid FROM ontology_source_roots AS roots
-      JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
-      JOIN ontology_source_rows AS rows
-        ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-        AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
-      WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
-        AND sources.status IN ('superseded', 'abandoned')
-      ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
+      JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+      JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
+      WHERE ${retired}
+      ORDER BY roots.retired_at, roots.id
       LIMIT ${remaining} FOR UPDATE OF rows SKIP LOCKED
     ) DELETE FROM ontology_source_rows AS rows USING selected WHERE rows.ctid = selected.ctid RETURNING 1
   `
@@ -28,13 +78,10 @@ export async function cleanupSourceVersions(
     const removed = await sql`
       WITH selected AS (
         SELECT roots.ctid FROM ontology_source_roots AS roots
-        JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
-        WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
-          AND sources.status IN ('superseded', 'abandoned')
-          AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
-            WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-              AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
-        ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key
+        JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+        WHERE ${retired}
+          AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id)
+        ORDER BY roots.retired_at, roots.id
         LIMIT ${remaining} FOR UPDATE OF roots SKIP LOCKED
       ) DELETE FROM ontology_source_roots AS roots USING selected WHERE roots.ctid = selected.ctid RETURNING 1
     `
@@ -46,16 +93,13 @@ export async function cleanupSourceVersions(
       ? []
       : await sql`
     WITH selected AS (
-      SELECT sources.ctid FROM ontology_sources AS sources
-      WHERE project_id = ${input.projectId} AND status IN ('superseded', 'abandoned') AND terminal_at < ${input.terminalBefore}
+      SELECT versions.ctid FROM ontology_sources AS versions
+      WHERE project_id = ${input.projectId} AND status = 'superseded'
+        AND terminal_at < ${input.terminalBefore}
         AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
-          WHERE roots.project_id = sources.project_id AND roots.source_id = sources.source_id
-            AND roots.materialization_id = sources.materialization_id)
-        AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
-          WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
-            AND rows.materialization_id = sources.materialization_id)
+          WHERE roots.version_id = versions.version_id)
       ORDER BY terminal_at, source_id, materialization_id LIMIT ${remaining} FOR UPDATE SKIP LOCKED
-    ) DELETE FROM ontology_sources AS sources USING selected WHERE sources.ctid = selected.ctid RETURNING 1
+    ) DELETE FROM ontology_sources AS versions USING selected WHERE versions.ctid = selected.ctid RETURNING 1
   `
   return { rowsDeleted, materializationsDeleted: removedManifests.length }
 }