@sixb/core 0.1.16 → 0.1.18

package/src/storage/ontology/in-memory/index.ts

@@ -7,14 +7,17 @@ import type { AssertSourceMaterializationExecution } from "../sources"
 import { InMemoryOntologyCommitStorage } from "./commits"
 import { InMemoryOntologyMaterializationStorage } from "./materializations"
 import { InMemoryOntologyOutboxStorage } from "./outbox"
+import { InMemoryOntologyReplacementPlanStorage } from "./replacement-plans"
 import {
   cloneOntologyState,
   createInMemoryOntologyState,
+  type InMemoryExecutionReader,
   type InMemoryOntologyState,
   type InMemoryOntologyStorageTestHooks,
   restoreOntologyState,
 } from "./shared-state"
 import { InMemoryOntologySourceStorage } from "./sources"
+import { InMemoryMaterializationStateReader } from "./state-reader"
 import {
   type InMemoryOntologyStorageTestingAdapter,
   registerInMemoryOntologyStorageTestingAdapter,
@@ -27,7 +30,7 @@ interface InMemoryOntologyStorageOptions {
   readonly getTransactionToken: () => object | null
   readonly getMaterializationLifecycle: () => ProviderMaterializationTransactionLifecycle | null
   readonly assertSourceMaterializationExecution: AssertSourceMaterializationExecution
-  readonly executionExists: (projectId: string, executionId: string) => Promise<boolean>
+  readonly readExecution: InMemoryExecutionReader
 }
 
 import { InMemoryVectorIndexingStorage } from "./vector-indexing"
@@ -40,6 +43,7 @@ export class InMemoryOntologyStorage implements OntologyStorage {
   readonly commits: InMemoryOntologyCommitStorage
   readonly sources: InMemoryOntologySourceStorage
   readonly materializations: InMemoryOntologyMaterializationStorage
+  readonly replacementPlans: InMemoryOntologyReplacementPlanStorage
   readonly outbox: InMemoryOntologyOutboxStorage
   private testHooks: InMemoryOntologyStorageTestHooks = {}
   private readonly testingAdapter: InMemoryOntologyStorageTestingAdapter = {
@@ -72,20 +76,35 @@ export class InMemoryOntologyStorage implements OntologyStorage {
       options.runRootOperation,
       options.assertSourceMaterializationExecution
     )
-    this.outbox = new InMemoryOntologyOutboxStorage(this.state, options.runRootOperation)
+    this.outbox = new InMemoryOntologyOutboxStorage(
+      this.state,
+      options.runRootOperation,
+      options.readExecution
+    )
+    const hooks: InMemoryOntologyStorageTestHooks = {
+      beforeRead: (boundary) => this.testHooks.beforeRead?.(boundary),
+      beforeWrite: (boundary, ordinal) => this.testHooks.beforeWrite?.(boundary, ordinal),
+      observeBuffer: (boundary, rows) => this.testHooks.observeBuffer?.(boundary, rows),
+      observeWork: (records) => this.testHooks.observeWork?.(records),
+    }
+    const reader = new InMemoryMaterializationStateReader(this.state, objects, timeseries, hooks)
     this.materializations = new InMemoryOntologyMaterializationStorage(
       this.state,
       objects,
       timeseries,
+      reader,
       options.getTransactionToken,
       options.getMaterializationLifecycle,
-      options.executionExists,
-      {
-        beforeRead: (boundary) => this.testHooks.beforeRead?.(boundary),
-        beforeWrite: (boundary, ordinal) => this.testHooks.beforeWrite?.(boundary, ordinal),
-        observeBuffer: (boundary, rows) => this.testHooks.observeBuffer?.(boundary, rows),
-        observeWork: (records) => this.testHooks.observeWork?.(records),
-      }
+      options.readExecution,
+      hooks
+    )
+    this.replacementPlans = new InMemoryOntologyReplacementPlanStorage(
+      this.state,
+      objects,
+      reader,
+      options.runRootOperation,
+      options.assertSourceMaterializationExecution,
+      hooks
     )
     registerInMemoryOntologyStorageTestingAdapter(this, this.testingAdapter)
   }

package/src/storage/ontology/in-memory/materializations-finalization.ts

@@ -8,7 +8,7 @@ import type {
   PinnedDatasetVersion,
   TelemetryCommitResult,
 } from "../../../materialization/model"
-import { linkRefSortKey } from "../../../materialization/refs"
+import { linkRefKey, linkRefSortKey, objectRefKey } from "../../../materialization/refs"
 import {
   getInMemoryObjectMaterializerAdapter,
   type InMemoryObjectStorage,
@@ -34,12 +34,12 @@ export function assertFinalizationCorrelations(
     result.created !== true ||
     !Number.isSafeInteger(result.eventCount) ||
     result.eventCount < 0 ||
-    result.eventCount !== session.outboxEnvelopes.size
+    result.eventCount !== session.outboxWrites.size
   ) {
     invalidCorrelation("Materialization result does not correlate with its commit intent.")
   }
   for (let ordinal = 0; ordinal < result.eventCount; ordinal += 1) {
-    if (!session.outboxEnvelopes.has(ordinal)) {
+    if (!session.outboxWrites.has(ordinal)) {
       invalidCorrelation("Outbox event ordinals must be contiguous from zero.")
     }
   }
@@ -63,7 +63,7 @@ export function assertFinalizationCorrelations(
     assertSourceActivationCorrelation(activation, session, state)
   }
   if (sourceActivations[0]) {
-    assertReplacementFullyStreamed(session, sourceActivations[0])
+    assertPlanCoverage(session, sourceActivations[0])
   }
   assertFinalizedWork(session, state, objects)
   if (commit.intent.kind === "projection") {
@@ -317,16 +317,8 @@ function assertFinalizedWork(
   state: InMemoryOntologyState,
   objects: InMemoryObjectStorage
 ): void {
-  const expectedPlanItems = session.applyWork.map((record) => stableJsonStringify(record.item))
-  const appliedPlanItems = session.appliedPlanItems
-  if (expectedPlanItems.length > 0 && !session.workStreams.apply.completed) {
-    invalidCorrelation("Materialization plan work was not fully streamed.")
-  }
-  if (stableJsonStringify(appliedPlanItems) !== stableJsonStringify(expectedPlanItems)) {
-    invalidCorrelation("Materialization plan work was not applied exactly once.")
-  }
-  if (session.cardinalityWork.length > 0 && !session.workStreams.cardinality.completed) {
-    invalidCorrelation("Materialization cardinality work was not fully validated.")
+  if (!session.applied) {
+    invalidCorrelation("A materialization plan must apply before it finalizes.")
   }
   assertFinalCardinality(session.cardinalityWork, session.header.commit.projectId, objects)
   if (session.header.commit.intent.kind === "telemetry") {
@@ -339,28 +331,25 @@ function assertFinalizedWork(
   }
 
   const expectedEvents = [...session.eventWork].sort(compareEventWork)
-  if (expectedEvents.length > 0 && !session.workStreams.event.completed) {
-    invalidCorrelation("Materialization event work was not fully drained.")
-  }
-  if (session.outboxEnvelopes.size !== expectedEvents.length) {
+  if (session.outboxWrites.size !== expectedEvents.length) {
     invalidCorrelation("Materialization event work was not fully written to the outbox.")
   }
   for (let ordinal = 0; ordinal < expectedEvents.length; ordinal += 1) {
     const expected = expectedEvents[ordinal]
-    const actual = session.outboxEnvelopes.get(ordinal)
+    const actual = session.outboxWrites.get(ordinal)
     if (!expected || !actual) {
       invalidCorrelation("Materialization event work was not fully written to the outbox.")
     }
-    const { id, commitOrdinal, ...actualDraft } = actual
     if (
-      commitOrdinal !== ordinal ||
-      id !== createEventId(session.header.commit.projectId, session.header.commit.id, ordinal) ||
-      stableJsonStringify(actualDraft) !== stableJsonStringify(expected.draft)
+      actual.commitOrdinal !== ordinal ||
+      actual.id !==
+        createEventId(session.header.commit.projectId, session.header.commit.id, ordinal) ||
+      stableJsonStringify(actual.event) !== stableJsonStringify(expected.draft)
     ) {
       invalidCorrelation("Materialization outbox event does not match its staged event work.")
     }
     const persisted = state.outbox.get(outboxKey(session.header.commit.projectId, actual.id))
-    if (!persisted || stableJsonStringify(persisted.envelope) !== stableJsonStringify(actual)) {
+    if (!persisted || stableJsonStringify(persisted.event) !== stableJsonStringify(actual.event)) {
       invalidCorrelation("Materialization outbox event was not persisted.")
     }
   }
@@ -380,26 +369,15 @@ function assertFinalCardinality(
       readonly occupiedLinkKeys: Set<string>
     }
   >()
-  let currentScope: string | null = null
-  let occupiedCount = 0
+  // Apply already rejected a scope with two staged occupants; this checks what it wrote.
   for (const record of effectiveOccupants) {
-    if (record.scopeSortKey !== currentScope) {
-      currentScope = record.scopeSortKey
-      occupiedCount = 0
-    }
     const scope = scopes.get(record.scopeSortKey) ?? {
       source: structuredClone(record.ref.source),
       linkId: record.ref.linkId,
       occupiedLinkKeys: new Set<string>(),
     }
     scopes.set(record.scopeSortKey, scope)
-    if (record.occupied) {
-      occupiedCount += 1
-      scope.occupiedLinkKeys.add(record.linkSortKey)
-      if (occupiedCount > 1) {
-        invalidCorrelation("Materialization cardinality work violates cardinality-one.")
-      }
-    }
+    if (record.occupied) scope.occupiedLinkKeys.add(record.linkSortKey)
   }
 
   const adapter = getInMemoryObjectMaterializerAdapter(objects)
@@ -422,48 +400,37 @@ function assertFinalCardinality(
   }
 }
 
-function assertReplacementFullyStreamed(
+/** Every identity the plan decides with a diff is classified once, and nothing else is. */
+function assertPlanCoverage(
   session: SessionState,
   activation: import("../materializations").SourceActivationWrite
 ): void {
-  const replacement = session.replacement
+  const plan = session.plan
   if (
-    !replacement ||
-    replacement.sourceId !== activation.source.projectionId ||
-    replacement.candidateMaterializationId !== activation.materializationId ||
-    replacement.candidate.materializationId !== activation.materializationId ||
-    replacement.candidate.projectionKind !== activation.projectionKind ||
-    replacement.candidate.protocol !== activation.protocol
+    !plan ||
+    plan.sourceId !== activation.source.projectionId ||
+    plan.materializationId !== activation.materializationId ||
+    plan.projectionKind !== activation.projectionKind
   ) {
-    invalidCorrelation(
-      "Source activation does not match the replacement candidate opened by the session."
-    )
+    invalidCorrelation("Source activation does not match the replacement plan the session applies.")
   }
-  if (
-    activation.projectionKind === "object" &&
-    (!replacement.objectStreamCompleted || !replacement.linkStreamCompleted)
-  ) {
-    invalidCorrelation("Object projection replacement state was not fully streamed.")
-  }
-  if (activation.projectionKind === "link" && !replacement.linkStreamCompleted) {
-    invalidCorrelation("Link projection replacement state was not fully streamed.")
-  }
-  const expectedObjectKeys =
-    activation.projectionKind === "object" ? [...replacement.objects.keys()].sort() : []
-  const expectedLinkKeys = [...replacement.links.entries()]
-    .filter(([, value]) => value.diffRequired)
-    .map(([key]) => key)
-    .sort()
+  const expected = (kind: "object" | "link") =>
+    [...plan.identities.values()]
+      .filter((identity) => identity.entity.kind === kind && identity.diffRequired)
+      .map((identity) =>
+        identity.entity.kind === "object"
+          ? objectRefKey(identity.entity.ref)
+          : linkRefKey(identity.entity.ref)
+      )
+      .sort()
   if (
     stableJsonStringify(classificationKeys(session, "object")) !==
-      stableJsonStringify(expectedObjectKeys) ||
+      stableJsonStringify(expected("object")) ||
     stableJsonStringify(classificationKeys(session, "link")) !==
-      stableJsonStringify(expectedLinkKeys) ||
+      stableJsonStringify(expected("link")) ||
     classificationKeys(session, "point").length > 0
   ) {
-    invalidCorrelation(
-      "Projection replacement classification coverage does not match its streamed state."
-    )
+    invalidCorrelation("Projection replacement classification coverage does not match its plan.")
   }
 }
 

package/src/storage/ontology/in-memory/materializations-state.ts

@@ -4,7 +4,6 @@ import type {
   EffectiveObjectSnapshot,
   OntologyLinkRef,
 } from "../../../materialization/model"
-import { linkRefKey, linkRefSortKey, linkScopeSortKey } from "../../../materialization/refs"
 import type { ObjectLinkRow, ObjectRow } from "../../objects/types"
 import type { TimeseriesPoint } from "../../timeseries/types"
 import type {
@@ -19,7 +18,6 @@ import type {
   StoredSourceLinkAssertion,
   StoredSourceObjectAssertion,
 } from "../sources"
-import type { ReplacementSessionState } from "./materializations"
 import type {
   InMemoryOntologyState,
   InMemorySourceMaterialization,
@@ -158,23 +156,3 @@ export function storedSourceLink(
   if (!materializationId || !row || row.assertion.kind !== "link") return null
   return storedSource(sourceId, materializationId, row) as StoredSourceLinkAssertion
 }
-
-export function addReplacementLink(
-  replacement: ReplacementSessionState,
-  ref: OntologyLinkRef,
-  diffRequired: boolean
-): void {
-  const key = linkRefKey(ref)
-  const existing = replacement.links.get(key)
-  if (existing) {
-    existing.diffRequired ||= diffRequired
-    if (diffRequired) replacement.affectedScopes.add(linkScopeSortKey(ref.source, ref.linkId))
-    return
-  }
-  replacement.links.set(key, {
-    ref: structuredClone(ref),
-    sortKey: linkRefSortKey(ref),
-    diffRequired,
-  })
-  if (diffRequired) replacement.affectedScopes.add(linkScopeSortKey(ref.source, ref.linkId))
-}

package/src/storage/ontology/in-memory/materializations-work.ts

@@ -1,57 +1,12 @@
-import { stableJsonStringify } from "../../../json"
 import { MaterializationConflictError } from "../../../materialization/errors"
 import type {
-  MaterializationPlanChunk,
   StoredLinkOverride,
   StoredLinkSlotOverride,
   StoredObjectOverride,
 } from "../materializations"
-import { invalidCorrelation, materializationPlanItems } from "../provider-work"
-import type { SessionState } from "./materializations"
 
 export * from "../provider-work"
 
-export function assertChunkSequence(session: SessionState, chunk: MaterializationPlanChunk): void {
-  const planItems = materializationPlanItems(chunk)
-  const applyStream = session.workStreams.apply
-  const appliedStart = session.appliedPlanItems.length
-  if (
-    planItems.length > 0 &&
-    (!applyStream.started || appliedStart + planItems.length > applyStream.emittedCount)
-  ) {
-    invalidCorrelation("Materialization plan items cannot be applied before they are streamed.")
-  }
-  for (let offset = 0; offset < planItems.length; offset += 1) {
-    const expected = session.applyWork[appliedStart + offset]?.item
-    if (!expected || stableJsonStringify(planItems[offset]) !== stableJsonStringify(expected)) {
-      invalidCorrelation("Materialization plan items must be applied in exact streamed order.")
-    }
-  }
-
-  const eventStream = session.workStreams.event
-  const appliedOutboxCount = session.outboxEnvelopes.size
-  if (
-    chunk.outbox.length > 0 &&
-    (!eventStream.started || appliedOutboxCount + chunk.outbox.length > eventStream.emittedCount)
-  ) {
-    invalidCorrelation("Materialization events cannot be applied before they are streamed.")
-  }
-  for (let offset = 0; offset < chunk.outbox.length; offset += 1) {
-    const expected = session.eventWork[appliedOutboxCount + offset]
-    const actual = chunk.outbox[offset]?.envelope
-    if (!expected || !actual) {
-      invalidCorrelation("Materialization outbox events must follow exact streamed order.")
-    }
-    const { id: _id, commitOrdinal, ...actualDraft } = actual
-    if (
-      commitOrdinal !== appliedOutboxCount + offset ||
-      stableJsonStringify(actualDraft) !== stableJsonStringify(expected.draft)
-    ) {
-      invalidCorrelation("Materialization outbox events must follow exact streamed order.")
-    }
-  }
-}
-
 export function assertLastCommit(
   value: StoredObjectOverride | StoredLinkOverride | StoredLinkSlotOverride | undefined,
   expected: string | null,