@sixb/core 0.1.16 → 0.1.17

package/src/storage/ontology/provider-work.ts

@@ -1,28 +1,51 @@
-import { stableJsonStringify } from "../../json"
-import { MaterializationValidationError } from "../../materialization/errors"
+import {
+  MaterializationConflictError,
+  MaterializationValidationError,
+} from "../../materialization/errors"
+import { eventAttribution, materializationEvent } from "../../materialization/event-envelopes"
+import type {
+  OntologyMaterializationEventAttribution,
+  OntologyMaterializationEventCommit,
+} from "../../materialization/events"
 import { createEventId, materializationEventKindOrdinal } from "../../materialization/identity"
-import type { OntologyLinkRef, OntologyObjectRef } from "../../materialization/model"
+import type {
+  EffectiveChangeCounts,
+  OntologyLinkRef,
+  OntologyObjectRef,
+} from "../../materialization/model"
 import {
   linkRefKey,
   linkRefSortKey,
   linkScopeSortKey,
   objectRefKey,
+  projectionEntityKey,
   telemetryPointKey,
 } from "../../materialization/refs"
 import type { OntologyCommitWrite } from "./commits"
 import {
+  type ExactEffectiveLinkDelete,
+  type ExactEffectiveLinkWrite,
+  type ExactEffectiveObjectDelete,
+  type ExactEffectiveObjectWrite,
+  type ExactLinkOverrideDelete,
+  type ExactLinkOverrideWrite,
+  type ExactLinkSlotOverrideDelete,
+  type ExactLinkSlotOverrideWrite,
+  type ExactObjectOverrideDelete,
+  type ExactObjectOverrideWrite,
   type ExactTimeseriesPointWrite,
   type MaterializationCardinalityOccupantWorkRecord,
   type MaterializationEventWorkRecord,
-  type MaterializationPlanChunk,
   type MaterializationPlanHeader,
   type MaterializationPlanWorkItem,
   type MaterializationPlanWorkRecord,
   type MaterializationWorkRecord,
   materializationApplyPhase,
 } from "./materializations"
-import type { OntologyOutboxWrite } from "./outbox"
+import type { ClaimedOntologyOutboxRow, OntologyMaterializationEventDraft } from "./outbox"
+import { assertMaterializationHeader } from "./provider-header-validation"
 import { assertTimestamp, invalidCorrelation } from "./provider-validation"
+import type { PlannedReplacementIdentity, StageReplacementPlanInput } from "./replacement-plans"
 
 export { invalidCorrelation } from "./provider-validation"
 
@@ -48,40 +71,355 @@ export function materializationChunkRows(chunk: MaterializationPlanChunk): numbe
   )
 }
 
-export function materializationPlanItems(
-  chunk: MaterializationPlanChunk
-): MaterializationPlanWorkItem[] {
-  return [
-    ...chunk.overrides.objects.upserts.map((value) => ({
-      kind: "object-override-upsert" as const,
-      value,
-    })),
-    ...chunk.overrides.objects.deletes.map((value) => ({
-      kind: "object-override-delete" as const,
-      value,
-    })),
-    ...chunk.overrides.links.edges.upserts.map((value) => ({
-      kind: "link-override-upsert" as const,
-      value,
-    })),
-    ...chunk.overrides.links.edges.deletes.map((value) => ({
-      kind: "link-override-delete" as const,
-      value,
-    })),
-    ...chunk.overrides.links.slots.upserts.map((value) => ({
-      kind: "link-slot-override-upsert" as const,
-      value,
-    })),
-    ...chunk.overrides.links.slots.deletes.map((value) => ({
-      kind: "link-slot-override-delete" as const,
-      value,
-    })),
-    ...chunk.effective.linkDeletes.map((value) => ({ kind: "link-delete" as const, value })),
-    ...chunk.effective.objectDeletes.map((value) => ({ kind: "object-delete" as const, value })),
-    ...chunk.effective.objectUpserts.map((value) => ({ kind: "object-upsert" as const, value })),
-    ...chunk.effective.linkUpserts.map((value) => ({ kind: "link-upsert" as const, value })),
-    ...chunk.timeseries.pointUpserts.map((value) => ({ kind: "point-upsert" as const, value })),
-  ]
+export interface ExactOverrideWrites {
+  readonly objects: {
+    readonly upserts: readonly ExactObjectOverrideWrite[]
+    readonly deletes: readonly ExactObjectOverrideDelete[]
+  }
+  readonly links: {
+    readonly edges: {
+      readonly upserts: readonly ExactLinkOverrideWrite[]
+      readonly deletes: readonly ExactLinkOverrideDelete[]
+    }
+    readonly slots: {
+      readonly upserts: readonly ExactLinkSlotOverrideWrite[]
+      readonly deletes: readonly ExactLinkSlotOverrideDelete[]
+    }
+  }
+}
+
+export interface ExactEffectiveWrites {
+  readonly objectUpserts: readonly ExactEffectiveObjectWrite[]
+  readonly objectDeletes: readonly ExactEffectiveObjectDelete[]
+  readonly linkUpserts: readonly ExactEffectiveLinkWrite[]
+  readonly linkDeletes: readonly ExactEffectiveLinkDelete[]
+}
+
+export interface ExactTimeseriesWrites {
+  readonly pointUpserts: readonly ExactTimeseriesPointWrite[]
+}
+
+/** Plan items and outbox writes of one bounded provider write, grouped by table. */
+export interface MaterializationPlanChunk {
+  readonly overrides: ExactOverrideWrites
+  readonly effective: ExactEffectiveWrites
+  readonly timeseries: ExactTimeseriesWrites
+  readonly outbox: readonly OntologyOutboxWrite[]
+}
+
+/** Groups staged plan items, already in apply order, and outbox writes into one provider write. */
+export function materializationPlanChunk(
+  items: readonly MaterializationPlanWorkItem[],
+  outbox: readonly OntologyOutboxWrite[] = []
+): MaterializationPlanChunk {
+  const chunk = {
+    overrides: {
+      objects: {
+        upserts: [] as ExactObjectOverrideWrite[],
+        deletes: [] as ExactObjectOverrideDelete[],
+      },
+      links: {
+        edges: {
+          upserts: [] as ExactLinkOverrideWrite[],
+          deletes: [] as ExactLinkOverrideDelete[],
+        },
+        slots: {
+          upserts: [] as ExactLinkSlotOverrideWrite[],
+          deletes: [] as ExactLinkSlotOverrideDelete[],
+        },
+      },
+    },
+    effective: {
+      objectUpserts: [] as ExactEffectiveObjectWrite[],
+      objectDeletes: [] as ExactEffectiveObjectDelete[],
+      linkUpserts: [] as ExactEffectiveLinkWrite[],
+      linkDeletes: [] as ExactEffectiveLinkDelete[],
+    },
+    timeseries: { pointUpserts: [] as ExactTimeseriesPointWrite[] },
+    outbox,
+  }
+  for (const item of items) {
+    switch (item.kind) {
+      case "object-override-upsert":
+        chunk.overrides.objects.upserts.push(item.value)
+        break
+      case "object-override-delete":
+        chunk.overrides.objects.deletes.push(item.value)
+        break
+      case "link-override-upsert":
+        chunk.overrides.links.edges.upserts.push(item.value)
+        break
+      case "link-override-delete":
+        chunk.overrides.links.edges.deletes.push(item.value)
+        break
+      case "link-slot-override-upsert":
+        chunk.overrides.links.slots.upserts.push(item.value)
+        break
+      case "link-slot-override-delete":
+        chunk.overrides.links.slots.deletes.push(item.value)
+        break
+      case "object-upsert":
+        chunk.effective.objectUpserts.push(item.value)
+        break
+      case "object-delete":
+        chunk.effective.objectDeletes.push(item.value)
+        break
+      case "link-upsert":
+        chunk.effective.linkUpserts.push(item.value)
+        break
+      case "link-delete":
+        chunk.effective.linkDeletes.push(item.value)
+        break
+      case "point-upsert":
+        chunk.timeseries.pointUpserts.push(item.value)
+        break
+    }
+  }
+  return chunk
+}
+
+/**
+ * One outbox row: a staged event draft at its commit ordinal. The draft is all a provider stores
+ * of the event; the commit's own row holds the context every event of that commit shares.
+ */
+export interface OntologyOutboxWrite {
+  readonly id: string
+  readonly commitOrdinal: number
+  readonly event: OntologyMaterializationEventDraft
+  /** The commit time: an event is due, and ordered, from the moment its commit is. */
+  readonly createdAt: string
+}
+
+/** The outbox write of a staged event draft at its commit ordinal. */
+export function materializationOutboxWrite(
+  commit: Pick<OntologyCommitWrite, "projectId" | "id" | "committedAt">,
+  event: OntologyMaterializationEventDraft,
+  commitOrdinal: number
+): OntologyOutboxWrite {
+  return {
+    id: createEventId(commit.projectId, commit.id, commitOrdinal),
+    commitOrdinal,
+    event,
+    createdAt: commit.committedAt,
+  }
+}
+
+/** A claimed outbox row as a provider reads it back: its draft and its commit's id. */
+export interface ClaimedOutboxDraft extends Omit<ClaimedOntologyOutboxRow, "envelope"> {
+  readonly id: string
+  readonly commitId: string
+  readonly commitOrdinal: number
+  readonly event: OntologyMaterializationEventDraft
+}
+
+/** The commit an event belongs to and the execution that commit ran under. */
+export interface OutboxEventContext {
+  readonly commit: OntologyMaterializationEventCommit
+  readonly execution: Parameters<typeof eventAttribution>[0]
+}
+
+/**
+ * Claimed rows with their events rebuilt from their commits and executions. Both outlive their
+ * events, so a missing one is corrupt storage, not a retryable state.
+ */
+export function claimedOutboxRows(
+  rows: readonly ClaimedOutboxDraft[],
+  contextOf: (commitId: string) => OutboxEventContext | undefined
+): ClaimedOntologyOutboxRow[] {
+  const attributions = new Map<string, OntologyMaterializationEventAttribution>()
+  return rows.map(({ id, commitId, commitOrdinal, event, ...row }) => {
+    const context = contextOf(commitId)
+    if (!context) {
+      throw new Error(`[Sixb] Ontology outbox event '${id}' has no commit '${commitId}'.`)
+    }
+    let attribution = attributions.get(commitId)
+    if (!attribution) {
+      attribution = eventAttribution(context.execution)
+      attributions.set(commitId, attribution)
+    }
+    return {
+      ...row,
+      envelope: materializationEvent(event, { id, commitOrdinal }, context.commit, attribution),
+    }
+  })
+}
+
+/** One planned replacement identity, checked and summarized for a provider to store. */
+export interface PreparedReplacementIdentity {
+  readonly kind: "object" | "link"
+  /** `objectRefKey` or `linkRefKey`, as classification records and provider tables key it. */
+  readonly key: string
+  readonly entityKey: string
+  readonly records: readonly MaterializationWorkRecord[]
+  readonly classified: boolean
+  /** The effective change its work makes, null when it makes none. */
+  readonly change: "created" | "updated" | "deleted" | null
+}
+
+/**
+ * Validates the commit a replacement plan carries: once the plan is open, the very commit it was
+ * opened with, time included.
+ */
+export function assertReplacementPlanCommit(
+  commit: OntologyCommitWrite,
+  opened?: { readonly commitId: string; readonly committedAt: string }
+): void {
+  assertMaterializationHeader({ commit, expected: NO_EXPECTATIONS })
+  if (opened && (commit.id !== opened.commitId || commit.committedAt !== opened.committedAt)) {
+    throw new MaterializationValidationError(
+      "Replacement plan work must carry the commit its plan was opened with."
+    )
+  }
+}
+
+const NO_EXPECTATIONS = { sources: [], objects: [], links: [], linkScopes: [], points: [] }
+
+/**
+ * The identities of one `stage` call, checked against the commit the plan was opened with. Each is
+ * planned once: one listed twice is rejected.
+ */
+export function prepareReplacementIdentities(
+  input: Pick<StageReplacementPlanInput, "commit" | "planned">,
+  opened: { readonly commitId: string; readonly committedAt: string }
+): PreparedReplacementIdentity[] {
+  assertReplacementPlanCommit(input.commit, opened)
+  const header = { commit: input.commit, expected: NO_EXPECTATIONS }
+  const keys = new Set<string>()
+  return input.planned.map((planned) => {
+    const prepared = prepareReplacementIdentity(planned, header)
+    if (keys.has(prepared.entityKey)) {
+      throw new MaterializationValidationError(
+        `Replacement identity ${prepared.entityKey} is staged twice.`
+      )
+    }
+    keys.add(prepared.entityKey)
+    return prepared
+  })
+}
+
+function prepareReplacementIdentity(
+  planned: PlannedReplacementIdentity,
+  header: MaterializationPlanHeader
+): PreparedReplacementIdentity {
+  const { identity } = planned
+  const identityKind = identity.kind
+  const key = identity.kind === "object" ? objectRefKey(identity.ref) : linkRefKey(identity.ref)
+  const keys = new Set<string>()
+  let classified = false
+  let change: PreparedReplacementIdentity["change"] = null
+  for (const record of planned.records) {
+    assertWorkRecord(record, header)
+    if (keys.has(record.recordKey)) {
+      throw new MaterializationConflictError(
+        "effective-state",
+        `Duplicate materialization work key '${record.recordKey}'.`
+      )
+    }
+    keys.add(record.recordKey)
+    if (record.kind === "classification") {
+      // Coverage is checked on the identity, so its classification must be its own.
+      if (record.entityKind !== identityKind || record.identityKey !== key) {
+        throw new MaterializationValidationError("A replacement identity can only classify itself.")
+      }
+      classified = true
+    }
+    if (record.kind !== "plan") continue
+    const { item } = record
+    if (item.kind === "object-upsert" || item.kind === "link-upsert") {
+      change = item.value.expected.exists ? "updated" : "created"
+    } else if (item.kind === "object-delete" || item.kind === "link-delete") {
+      change = "deleted"
+    }
+  }
+  return {
+    kind: identityKind,
+    key,
+    entityKey: projectionEntityKey(identity),
+    records: planned.records,
+    classified,
+    change,
+  }
+}
+
+/** The change counts a replacement commit reports, derived from its planned work. */
+export function projectionChangeCounts(
+  records: Iterable<MaterializationWorkRecord>
+): EffectiveChangeCounts {
+  let objects = 0
+  let links = 0
+  const counts = {
+    objectsCreated: 0,
+    objectsUpdated: 0,
+    objectsDeleted: 0,
+    linksCreated: 0,
+    linksUpdated: 0,
+    linksDeleted: 0,
+  }
+  for (const record of records) {
+    if (record.kind === "classification") {
+      if (record.entityKind === "object") objects += 1
+      if (record.entityKind === "link") links += 1
+      continue
+    }
+    if (record.kind !== "plan") continue
+    const { item } = record
+    if (item.kind === "object-upsert") {
+      if (item.value.expected.exists) counts.objectsUpdated += 1
+      else counts.objectsCreated += 1
+    } else if (item.kind === "object-delete") counts.objectsDeleted += 1
+    else if (item.kind === "link-upsert") {
+      if (item.value.expected.exists) counts.linksUpdated += 1
+      else counts.linksCreated += 1
+    } else if (item.kind === "link-delete") counts.linksDeleted += 1
+  }
+  return {
+    ...counts,
+    objectsUnchanged:
+      objects - counts.objectsCreated - counts.objectsUpdated - counts.objectsDeleted,
+    linksUnchanged: links - counts.linksCreated - counts.linksUpdated - counts.linksDeleted,
+  }
+}
+
+export interface CardinalityValidator {
+  /** Accepts staged cardinality records in canonical order (`compareCardinalityWork`). */
+  accept(record: MaterializationCardinalityOccupantWorkRecord): void
+}
+
+/**
+ * Rejects a cardinality-one scope with two occupants. The candidate view is the projection source
+ * alone; the effective view is the resolved scope after this commit.
+ */
+export function createCardinalityValidator(): CardinalityValidator {
+  let view: MaterializationCardinalityOccupantWorkRecord["view"] | null = null
+  let scope: string | null = null
+  let occupant: string | null = null
+  return {
+    accept(record) {
+      if (record.view !== view || record.scopeSortKey !== scope) {
+        view = record.view
+        scope = record.scopeSortKey
+        occupant = null
+      }
+      if (!record.occupied) return
+      if (occupant !== null && occupant !== record.linkSortKey) {
+        throw cardinalityViolation(record.view, record.ref.source.objectTypeId, record.ref.linkId)
+      }
+      occupant = record.linkSortKey
+    },
+  }
+}
+
+/** The user-facing error for a cardinality-one scope with two occupants. */
+export function cardinalityViolation(
+  view: MaterializationCardinalityOccupantWorkRecord["view"],
+  sourceTypeId: string,
+  linkId: string
+): MaterializationValidationError {
+  return new MaterializationValidationError(
+    view === "candidate"
+      ? `Projection source scope '${sourceTypeId}.${linkId}' has cardinality one.`
+      : `Link scope '${sourceTypeId}.${linkId}' has cardinality one.`
+  )
 }
 
 export function assertWorkRecord(
@@ -103,12 +441,7 @@ export function assertWorkRecord(
       !/^[0-9a-f]+$/.test(record.sortKey) ||
       !Number.isSafeInteger(record.eventKindRank) ||
       record.eventKindRank < 0 ||
-      record.eventKindRank !== materializationEventKindOrdinal(record.draft.type) ||
-      record.draft.projectId !== header.commit.projectId ||
-      record.draft.commitId !== header.commit.id ||
-      record.draft.occurredAt !== header.commit.committedAt ||
-      stableJsonStringify(record.draft.origin) !== stableJsonStringify(header.commit.origin) ||
-      !sameAttribution(record.draft, header.commit)
+      record.eventKindRank !== materializationEventKindOrdinal(record.draft.type)
     ) {
       throw new MaterializationValidationError("Materialization event work is invalid.")
     }
@@ -153,13 +486,14 @@ export function comparePlanWork(
 ): number {
   return (
     left.applyPhase - right.applyPhase ||
-    planKindOrder(left.item.kind) - planKindOrder(right.item.kind) ||
+    materializationPlanKindRank(left.item.kind) - materializationPlanKindRank(right.item.kind) ||
     left.sortKey.localeCompare(right.sortKey) ||
     left.recordKey.localeCompare(right.recordKey)
   )
 }
 
-function planKindOrder(kind: MaterializationPlanWorkItem["kind"]): number {
+/** Canonical order of plan item kinds within an apply phase; providers store it with the work. */
+export function materializationPlanKindRank(kind: MaterializationPlanWorkItem["kind"]): number {
   switch (kind) {
     case "object-override-upsert":
       return 0
@@ -219,16 +553,6 @@ function planPhaseMatches(record: MaterializationPlanWorkRecord): boolean {
   return record.applyPhase === materializationApplyPhase(record.item.kind)
 }
 
-export function assertPlanChunkCorrelations(
-  chunk: MaterializationPlanChunk,
-  commit: MaterializationPlanHeader["commit"]
-): void {
-  // Physical writes correlate through the same per-item rule the work stream validates, so the two
-  // paths stay in lockstep. Outbox writes are not plan items and keep their dedicated correlation.
-  for (const item of materializationPlanItems(chunk)) assertPlanItemCorrelation(item, commit)
-  for (const value of chunk.outbox) assertOutboxCorrelation(value, commit)
-}
-
 function assertPlanItemCorrelation(
   item: MaterializationPlanWorkItem,
   commit: MaterializationPlanHeader["commit"]
@@ -300,27 +624,6 @@ function assertPointWriteCorrelation(
   }
 }
 
-function assertOutboxCorrelation(
-  value: OntologyOutboxWrite,
-  commit: MaterializationPlanHeader["commit"]
-): void {
-  const { envelope } = value
-  if (
-    envelope.projectId !== commit.projectId ||
-    envelope.commitId !== commit.id ||
-    envelope.occurredAt !== commit.committedAt ||
-    value.availableAt !== commit.committedAt ||
-    value.createdAt !== commit.committedAt ||
-    stableJsonStringify(envelope.origin) !== stableJsonStringify(commit.origin) ||
-    !sameAttribution(envelope, commit) ||
-    !Number.isSafeInteger(envelope.commitOrdinal) ||
-    envelope.commitOrdinal < 0 ||
-    envelope.id !== createEventId(commit.projectId, commit.id, envelope.commitOrdinal)
-  ) {
-    invalidCorrelation("Outbox event does not correlate with its materialization commit.")
-  }
-}
-
 function assertCommitWriteCorrelation(
   lastCommitId: string,
   updatedAt: string,
@@ -347,15 +650,3 @@ function assertLinkRefEqual(left: OntologyLinkRef, right: OntologyLinkRef, label
     invalidCorrelation(`${label} row and expected references differ.`)
   }
 }
-
-/** Events carry exactly the attribution of the commit that produced them. */
-function sameAttribution(
-  event: Pick<OntologyCommitWrite, "requestedBy" | "executor">,
-  commit: Pick<OntologyCommitWrite, "requestedBy" | "executor">
-): boolean {
-  return (
-    stableJsonStringify(event.requestedBy ?? null) ===
-      stableJsonStringify(commit.requestedBy ?? null) &&
-    stableJsonStringify(event.executor) === stableJsonStringify(commit.executor)
-  )
-}