@sixb/pg 0.1.13 → 0.1.15

package/src/ontology-storage/source-roots.ts

@@ -4,12 +4,91 @@ import {
   MaterializationValidationError,
   projectionEntityKey,
 } from "@sixb/core/internal/materialization"
-import { sourceStageRoots, utf8SortKey } from "@sixb/core/internal/ontology-storage-provider"
+import {
+  sourceEntityColumns,
+  sourceStageRoots,
+} from "@sixb/core/internal/ontology-storage-provider"
 import type { SourceActivationWrite, StageSourceRowsInput } from "@sixb/core/storage"
 import type { SQLClient } from "../pg-client"
 import type { PgOntologySourceAssertionRow } from "./shared"
 import { jsonParameter, type PgOntologySourceRow } from "./shared"
 
+/**
+ * Source storage in three levels: a version (`ontology_sources`, one per run), its roots (one per
+ * logical dataset row, keyed by the canonical entity key) and their rows (what each root asserts).
+ *
+ * A root is live while its version is published (active or superseded), it is not retired and it
+ * is not a deletion. Activation retires the live roots its candidate replaces and flips the
+ * version; the candidate's own roots are never rewritten. One live root per source and key holds
+ * because activation, under the source fence, retires every root the candidate replaces.
+ */
+const PUBLISHED = "('active', 'superseded')"
+
+/** The columns `sourceAssertion` reads, over `versions`, `roots` and `rows`. */
+export function sourceAssertionColumns(sql: SQLClient) {
+  return sql.unsafe(`versions.source_id, versions.materialization_id, roots.root_key,
+    roots.staging_ordinal, rows.entity_kind, rows.object_type_id, rows.primary_id,
+    rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
+    rows.target_primary_id, rows.payload`)
+}
+
+/** Keeps `rows` to those of live roots in `projectId`, joined as `roots` and `versions`. */
+export function liveSourceRowsJoin(sql: SQLClient, projectId: string) {
+  return sql`
+    JOIN ontology_source_roots AS roots ON roots.id = rows.root_id
+      AND roots.project_id = ${projectId} AND roots.retired_at IS NULL AND NOT roots.deleted
+    JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+      AND versions.status IN ${sql.unsafe(PUBLISHED)}
+  `
+}
+
+function candidateVersion(
+  sql: SQLClient,
+  input: {
+    readonly projectId: string
+    readonly sourceId: string
+    readonly materializationId: string
+  }
+) {
+  return sql`(SELECT version_id FROM ontology_sources WHERE project_id = ${input.projectId}
+    AND source_id = ${input.sourceId} AND materialization_id = ${input.materializationId})`
+}
+
+/** The typed identity of `rows` equals the one requested. */
+const ENTITY_MATCH = `rows.entity_kind = requested.entity_kind
+  AND rows.object_type_id IS NOT DISTINCT FROM requested.object_type_id
+  AND rows.primary_id IS NOT DISTINCT FROM requested.primary_id
+  AND rows.source_type_id IS NOT DISTINCT FROM requested.source_type_id
+  AND rows.source_primary_id IS NOT DISTINCT FROM requested.source_primary_id
+  AND rows.link_id IS NOT DISTINCT FROM requested.link_id
+  AND rows.target_type_id IS NOT DISTINCT FROM requested.target_type_id
+  AND rows.target_primary_id IS NOT DISTINCT FROM requested.target_primary_id`
+
+const REQUESTED_ENTITY = `entity_kind TEXT, object_type_id TEXT, primary_id TEXT,
+  source_type_id TEXT, source_primary_id TEXT, link_id TEXT, target_type_id TEXT,
+  target_primary_id TEXT`
+
+/** An entity's own key, and for a link the key of the object root that may assert it. */
+function requestedEntity(ref: ProjectionEntityRef) {
+  const columns = sourceEntityColumns(ref)
+  const key = projectionEntityKey(ref)
+  return {
+    entity_kind: ref.kind,
+    object_type_id: columns.objectTypeId,
+    primary_id: columns.primaryId,
+    source_type_id: columns.sourceTypeId,
+    source_primary_id: columns.sourcePrimaryId,
+    link_id: columns.linkId,
+    target_type_id: columns.targetTypeId,
+    target_primary_id: columns.targetPrimaryId,
+    root_keys:
+      ref.kind === "object"
+        ? [key]
+        : [key, projectionEntityKey({ kind: "object", ref: ref.ref.source })],
+  }
+}
+
+/** The candidate's rows, plus the rows of the live roots it may replace. */
 export function replacementSourceRows(
   sql: SQLClient,
   input: {
@@ -19,27 +98,32 @@ export function replacementSourceRows(
     readonly incremental: boolean
   }
 ) {
+  const live = sql`roots.retired_at IS NULL AND NOT roots.deleted
+    AND versions.source_id = ${input.sourceId} AND versions.status IN ${sql.unsafe(PUBLISHED)}`
   const roots = input.incremental
     ? sql`
-    SELECT roots.* FROM ontology_source_roots AS changed
+    SELECT live.id FROM ontology_source_roots AS changed
     CROSS JOIN LATERAL (
-      SELECT roots.* FROM ontology_source_roots AS roots
-      WHERE roots.project_id = changed.project_id AND roots.source_id = changed.source_id
-        AND roots.root_sort_key = changed.root_sort_key AND roots.active OFFSET 0
-    ) AS roots
-    WHERE changed.project_id = ${input.projectId} AND changed.source_id = ${input.sourceId}
-      AND changed.materialization_id = ${input.materializationId}
+      SELECT roots.id FROM ontology_source_roots AS roots
+      JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+      WHERE roots.project_id = ${input.projectId} AND roots.root_key = changed.root_key AND ${live}
+      OFFSET 0
+    ) AS live
+    WHERE changed.version_id = ${candidateVersion(sql, input)}
+  `
+    : sql`
+    SELECT roots.id FROM ontology_sources AS versions
+    JOIN ontology_source_roots AS roots ON roots.version_id = versions.version_id
+    WHERE versions.project_id = ${input.projectId} AND ${live}
   `
-    : sql`SELECT * FROM ontology_source_roots WHERE project_id = ${input.projectId} AND source_id = ${input.sourceId} AND active`
   return sql`
-    SELECT * FROM ontology_source_rows WHERE project_id = ${input.projectId}
-      AND source_id = ${input.sourceId} AND materialization_id = ${input.materializationId}
+    SELECT rows.* FROM ontology_source_roots AS roots
+    JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
+    WHERE roots.version_id = ${candidateVersion(sql, input)}
     UNION ALL
     SELECT rows.* FROM (${roots}) AS roots
     CROSS JOIN LATERAL (
-      SELECT rows.* FROM ontology_source_rows AS rows WHERE rows.project_id = roots.project_id
-        AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
-        AND rows.root_sort_key = roots.root_sort_key OFFSET 0
+      SELECT rows.* FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id OFFSET 0
     ) AS rows
   `
 }
@@ -57,38 +141,31 @@ export async function stageSourceRoots(
   const values = jsonParameter(
     sql,
     roots.map((root) => ({
-      root_sort_key: root.sortKey,
-      root_kind: root.root.kind,
-      root_key: JSON.parse(root.rootKey),
-      root: root.root,
+      root_key: root.rootKey,
       staging_ordinal: root.stagingOrdinal,
       deleted: root.deleted,
     }))
   )
-  const inserted = await sql<{ root_sort_key: string }[]>`
-    INSERT INTO ontology_source_roots (
-      project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
-    ) SELECT ${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
-      root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
-    FROM jsonb_to_recordset(${values}) AS input(
-      root_sort_key TEXT, root_kind TEXT, root_key JSONB, root JSONB, staging_ordinal BIGINT, deleted BOOLEAN)
-    ON CONFLICT DO NOTHING RETURNING root_sort_key
+  const inserted = await sql<{ root_key: string }[]>`
+    INSERT INTO ontology_source_roots (version_id, project_id, root_key, staging_ordinal, deleted)
+    SELECT ${manifest.version_id}, ${input.projectId}, root_key, staging_ordinal, deleted
+    FROM jsonb_to_recordset(${values}) AS input(root_key TEXT, staging_ordinal BIGINT, deleted BOOLEAN)
+    ON CONFLICT DO NOTHING RETURNING root_key
   `
   if (inserted.length === roots.length) return
-  const insertedKeys = new Set(inserted.map((row) => row.root_sort_key))
-  const skipped = roots.filter((root) => !insertedKeys.has(root.sortKey))
+  const insertedKeys = new Set(inserted.map((row) => row.root_key))
+  const skipped = roots.filter((root) => !insertedKeys.has(root.rootKey))
   const stored = await sql<
-    { root_sort_key: string; staging_ordinal: string | number; deleted: boolean }[]
+    { root_key: string; staging_ordinal: string | number; deleted: boolean }[]
   >`
-    SELECT root_sort_key, staging_ordinal, deleted FROM ontology_source_roots
-    WHERE project_id = ${input.projectId} AND source_id = ${input.source.projectionId}
-      AND materialization_id = ${input.materializationId}
-      AND root_sort_key = ANY(${sql.array(skipped.map((root) => root.sortKey))}::text[])
+    SELECT root_key, staging_ordinal, deleted FROM ontology_source_roots
+    WHERE version_id = ${manifest.version_id}
+      AND root_key = ANY(${sql.array(skipped.map((root) => root.rootKey))}::text[])
   `
-  const existing = new Map(stored.map((root) => [root.root_sort_key, root]))
+  const existing = new Map(stored.map((root) => [root.root_key, root]))
   if (
     skipped.some((root) => {
-      const previous = existing.get(root.sortKey)
+      const previous = existing.get(root.rootKey)
       return (
         !previous ||
         Number(previous.staging_ordinal) !== root.stagingOrdinal ||
@@ -101,6 +178,7 @@ export async function stageSourceRoots(
     )
 }
 
+/** Retires what the candidate replaces; its own roots go live when its version turns active. */
 export async function activateSourceRoots(
   sql: SQLClient,
   projectId: string,
@@ -117,24 +195,27 @@ export async function activateSourceRoots(
       "Source delta base changed before activation."
     )
   if (candidate.base_materialization_id !== null && Number(candidate.root_count) === 0) return
-  const affected =
-    candidate.base_materialization_id === null
-      ? sql``
-      : sql`AND root_sort_key IN (
-    SELECT root_sort_key FROM ontology_source_roots
-    WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
-      AND materialization_id = ${activation.materializationId}
-  )`
-  await sql`
-    UPDATE ontology_source_roots AS roots SET active = FALSE, retired_at = ${activation.updatedAt}
-    WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId} AND active
-    ${affected}
-  `
+  const live = sql`roots.retired_at IS NULL AND NOT roots.deleted
+    AND versions.version_id = roots.version_id AND versions.project_id = ${projectId}
+    AND versions.source_id = ${activation.source.projectionId}
+    AND versions.status IN ${sql.unsafe(PUBLISHED)}`
+  if (candidate.base_materialization_id === null) {
+    await sql`
+      UPDATE ontology_source_roots AS roots SET retired_at = ${activation.updatedAt}
+      FROM ontology_sources AS versions WHERE ${live}
+    `
+  } else {
+    await sql`
+      UPDATE ontology_source_roots AS roots SET retired_at = ${activation.updatedAt}
+      FROM ontology_source_roots AS changed, ontology_sources AS versions
+      WHERE changed.version_id = ${candidate.version_id} AND roots.project_id = ${projectId}
+        AND roots.root_key = changed.root_key AND ${live}
+    `
+  }
+  // Deletions are never live; retiring them lets cleanup remove them with their version.
   await sql`
-    UPDATE ontology_source_roots SET active = NOT deleted,
-      retired_at = CASE WHEN deleted THEN ${activation.updatedAt}::timestamptz ELSE NULL END
-    WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
-      AND materialization_id = ${activation.materializationId}
+    UPDATE ontology_source_roots SET retired_at = ${activation.updatedAt}
+    WHERE version_id = ${candidate.version_id} AND deleted
   `
 }
 
@@ -144,13 +225,11 @@ export async function assertSourceRootCoverage(
 ): Promise<void> {
   const [invalid] = await sql`
     SELECT 1 FROM ontology_source_roots AS roots
-    WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
-      AND materialization_id = ${manifest.materialization_id}
+    WHERE roots.version_id = ${manifest.version_id}
       AND roots.deleted = EXISTS (
-        SELECT 1 FROM ontology_source_rows AS rows
-        WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-          AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
-      ) LIMIT 1
+        SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id
+      )
+    LIMIT 1
   `
   if (invalid)
     throw new MaterializationValidationError(
@@ -158,35 +237,29 @@ export async function assertSourceRootCoverage(
     )
 }
 
+/** The live assertions of `refs`, whichever source asserts them. */
 export function activeSourceRows(
   sql: SQLClient,
   projectId: string,
   refs: readonly ProjectionEntityRef[]
 ) {
-  const keys = refs.flatMap((entity) => {
-    const entitySortKey = utf8SortKey(projectionEntityKey(entity))
-    const roots =
-      entity.kind === "object"
-        ? [entitySortKey]
-        : [
-            entitySortKey,
-            utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source })),
-          ]
-    return roots.map((rootSortKey) => ({
-      root_sort_key: rootSortKey,
-      entity_sort_key: entitySortKey,
-    }))
+  const requested = refs.flatMap((ref) => {
+    const { root_keys, ...entity } = requestedEntity(ref)
+    return root_keys.map((root_key) => ({ ...entity, root_key }))
   })
   return sql<PgOntologySourceAssertionRow[]>`
-    SELECT rows.* FROM jsonb_to_recordset(${jsonParameter(sql, keys)}) AS requested(root_sort_key TEXT, entity_sort_key TEXT)
+    SELECT ${sourceAssertionColumns(sql)}
+    FROM jsonb_to_recordset(${jsonParameter(sql, requested)})
+      AS requested(root_key TEXT, ${sql.unsafe(REQUESTED_ENTITY)})
     JOIN ontology_source_roots AS roots ON roots.project_id = ${projectId}
-      AND roots.active AND roots.root_sort_key = requested.root_sort_key
-    JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-      AND rows.materialization_id = roots.materialization_id AND rows.entity_sort_key = requested.entity_sort_key
-      AND rows.root_sort_key = roots.root_sort_key
+      AND roots.root_key = requested.root_key AND roots.retired_at IS NULL AND NOT roots.deleted
+    JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+      AND versions.status IN ${sql.unsafe(PUBLISHED)}
+    JOIN ontology_source_rows AS rows ON rows.root_id = roots.id AND ${sql.unsafe(ENTITY_MATCH)}
   `
 }
 
+/** The candidate's assertions of `refs`, and optionally the live ones they would replace. */
 export function replacementAssertionRows(
   sql: SQLClient,
   input: {
@@ -198,52 +271,41 @@ export function replacementAssertionRows(
     refs: readonly ProjectionEntityRef[]
   }
 ) {
-  const keys = input.refs.map((ref) => ({
-    entity_kind: ref.kind,
-    entity_key: JSON.parse(projectionEntityKey(ref)),
-    root_keys:
-      ref.kind === "object"
-        ? [utf8SortKey(projectionEntityKey(ref))]
-        : [
-            utf8SortKey(projectionEntityKey(ref)),
-            utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source })),
-          ],
-  }))
+  const version = candidateVersion(sql, input)
   const previous = !input.includePrevious
     ? sql``
     : sql`UNION ALL
-    SELECT rows.* FROM (
+    SELECT ${sourceAssertionColumns(sql)} FROM (
       SELECT roots.* FROM ontology_source_roots AS roots
-    WHERE roots.project_id = ${input.projectId} AND roots.source_id = ${input.sourceId}
-      AND roots.active AND roots.root_sort_key = ANY(requested.root_keys)
+      JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+      WHERE roots.project_id = ${input.projectId} AND roots.root_key = ANY(requested.root_keys)
+        AND roots.retired_at IS NULL AND NOT roots.deleted
+        AND versions.source_id = ${input.sourceId} AND versions.status IN ${sql.unsafe(PUBLISHED)}
       ${
         !input.incremental
           ? sql``
           : sql`AND EXISTS (SELECT 1 FROM ontology_source_roots AS changed
-        WHERE changed.project_id = roots.project_id AND changed.source_id = roots.source_id
-          AND changed.materialization_id = ${input.materializationId} AND changed.root_sort_key = roots.root_sort_key)`
+        WHERE changed.version_id = ${version} AND changed.root_key = roots.root_key)`
       }
-
       OFFSET 0
     ) AS roots
+    JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
     CROSS JOIN LATERAL (
       SELECT rows.* FROM ontology_source_rows AS rows
-      WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
-        AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
-        AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
+      WHERE rows.root_id = roots.id AND ${sql.unsafe(ENTITY_MATCH)}
       OFFSET 0
     ) AS rows
   `
   // Keep each JSON request as an indexed identity lookup even with fresh/stale statistics.
   // Flattening this LATERAL caused a cold 10k-root run to spend seconds on each state page.
   return sql<PgOntologySourceAssertionRow[]>`
-    SELECT selected.* FROM jsonb_to_recordset(${jsonParameter(sql, keys)})
-      AS requested(entity_kind TEXT, entity_key JSONB, root_keys TEXT[])
+    SELECT selected.* FROM jsonb_to_recordset(${jsonParameter(sql, input.refs.map(requestedEntity))})
+      AS requested(${sql.unsafe(REQUESTED_ENTITY)}, root_keys TEXT[])
     CROSS JOIN LATERAL (
-      SELECT rows.* FROM ontology_source_rows AS rows
-      WHERE rows.project_id = ${input.projectId} AND rows.source_id = ${input.sourceId}
-        AND rows.materialization_id = ${input.materializationId}
-        AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
+      SELECT ${sourceAssertionColumns(sql)} FROM ontology_source_roots AS roots
+      JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
+      JOIN ontology_source_rows AS rows ON rows.root_id = roots.id AND ${sql.unsafe(ENTITY_MATCH)}
+      WHERE roots.version_id = ${version} AND roots.root_key = ANY(requested.root_keys)
       ${previous}
       OFFSET 0
     ) AS selected