@sixb/core 0.1.16 → 0.1.18

package/src/storage/ontology/in-memory/shared-state.ts

@@ -4,6 +4,7 @@ import type {
   OntologyMaterializationOrigin,
   ProjectionEntityRef,
 } from "../../../materialization/model"
+import type { ExecutionRecord } from "../../executions/types"
 import type { OntologyCommitOriginSelector, OntologyCommitRecord } from "../commits"
 import type {
   MaterializationWorkRecord,
@@ -11,7 +12,7 @@ import type {
   StoredLinkSlotOverride,
   StoredObjectOverride,
 } from "../materializations"
-import type { OntologyOutboxRecord } from "../outbox"
+import type { OntologyMaterializationEventDraft, OntologyOutboxRecord } from "../outbox"
 import type {
   OntologySourceRecord,
   StageSourceAssertion,
@@ -20,6 +21,7 @@ import type {
 } from "../sources"
 import type { VectorIndexingWork } from "../vector-indexing"
 import type { StoredObjectVector } from "../vectors"
+import type { InMemoryCommitTouches, InMemoryReplacementPlan } from "./replacement-plans"
 
 export interface InMemorySourceRoot {
   readonly root: ProjectionEntityRef
@@ -67,7 +69,25 @@ export interface InMemoryOntologyState {
   readonly objectOverrides: Map<string, InMemoryStoredObjectOverride>
   readonly linkOverrides: Map<string, InMemoryStoredLinkOverride>
   readonly linkSlotOverrides: Map<string, InMemoryStoredLinkSlotOverride>
-  readonly outbox: Map<string, OntologyOutboxRecord>
+  readonly outbox: Map<string, InMemoryOutboxRow>
+  readonly replacementPlans: Map<string, InMemoryReplacementPlan>
+  /** What each commit touched among the inputs plans read, keyed by its `commitsById` ordinal. */
+  readonly commitTouches: Map<number, InMemoryCommitTouches>
+}
+
+/** Reads one execution of the same storage, the record an ontology commit references. */
+export type InMemoryExecutionReader = (
+  projectId: string,
+  executionId: string
+) => Promise<ExecutionRecord | null>
+
+/** An outbox row as stored: the event's draft, rebuilt from its commit when claimed. */
+export interface InMemoryOutboxRow extends Omit<OntologyOutboxRecord, "envelope"> {
+  readonly projectId: string
+  readonly id: string
+  readonly commitId: string
+  readonly commitOrdinal: number
+  readonly event: OntologyMaterializationEventDraft
 }
 
 /** @internal Test-only failure injection for the in-memory ontology provider. */
@@ -94,6 +114,8 @@ export function createInMemoryOntologyState(): InMemoryOntologyState {
     linkOverrides: new Map(),
     linkSlotOverrides: new Map(),
     outbox: new Map(),
+    replacementPlans: new Map(),
+    commitTouches: new Map(),
   }
 }
 

package/src/storage/ontology/in-memory/source-roots.ts

@@ -1,8 +1,9 @@
 import { MaterializationConflictError } from "../../../materialization/errors"
-import { linkScopeSortKey, projectionEntityKey } from "../../../materialization/refs"
+import { linkRefKey, linkScopeSortKey, projectionEntityKey } from "../../../materialization/refs"
 import type { SourceActivationWrite } from "../materializations"
 import type { StoredSourceAssertion, StoredSourceLinkAssertion } from "../sources"
 import { storedSource } from "./materializations-state"
+import type { InMemoryCommitTouches } from "./replacement-plans"
 import {
   type InMemoryOntologyState,
   type InMemorySourceMaterialization,
@@ -37,10 +38,12 @@ export function* previousSourceRows(
   }
 }
 
+/** Activates the candidate's roots, touching each link a root it retires or activates asserts. */
 export function activateSourceRoots(
   state: InMemoryOntologyState,
   candidate: InMemorySourceMaterialization,
-  activation: SourceActivationWrite
+  activation: SourceActivationWrite,
+  touches: InMemoryCommitTouches
 ): void {
   if (
     candidate.base &&
@@ -75,6 +78,7 @@ export function activateSourceRoots(
       sources.delete(candidate.source.projectionId)
       if (sources.size === 0) state.activeSourceRows.delete(indexKey)
       if (row.assertion.kind === "link") {
+        touches.links.add(linkRefKey(row.assertion.ref))
         const scopeKey = projectEntityKey(
           candidate.projectId,
           linkScopeSortKey(row.assertion.ref.source, row.assertion.ref.linkId)
@@ -104,6 +108,7 @@ export function activateSourceRoots(
       sources.set(candidate.source.projectionId, row)
       state.activeSourceRows.set(indexKey, sources)
       if (row.assertion.kind === "link") {
+        touches.links.add(linkRefKey(row.assertion.ref))
         const scopeKey = projectEntityKey(
           candidate.projectId,
           linkScopeSortKey(row.assertion.ref.source, row.assertion.ref.linkId)

package/src/storage/ontology/in-memory/sources.ts

@@ -33,8 +33,6 @@ import type {
   StageSourceAssertion,
   StageSourceRowsInput,
   StageSourceRowsResult,
-  SummarizeTerminalSourceMaterializationsInput,
-  TerminalSourceMaterializationSummary,
 } from "../sources"
 import { findActiveSourceMaterialization } from "./materializations-state"
 import {
@@ -338,30 +336,6 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
     return { rowsDeleted, materializationsDeleted }
   }
 
-  async summarizeTerminal(
-    input: SummarizeTerminalSourceMaterializationsInput
-  ): Promise<TerminalSourceMaterializationSummary> {
-    return this.runRootOperation(() => {
-      assertNonblank(input.projectId, "Terminal source summary project id")
-      let count = 0
-      let oldestTerminalAt: string | null = null
-      for (const materialization of this.state.sourceMaterializations.values()) {
-        if (
-          materialization.projectId !== input.projectId ||
-          (materialization.status !== "superseded" && materialization.status !== "abandoned") ||
-          materialization.terminalAt === null
-        ) {
-          continue
-        }
-        count += 1
-        if (oldestTerminalAt === null || materialization.terminalAt < oldestTerminalAt) {
-          oldestTerminalAt = materialization.terminalAt
-        }
-      }
-      return { count, oldestTerminalAt }
-    })
-  }
-
   private async assertCurrentExecution(
     input:
       | BeginSourceMaterializationInput
@@ -446,14 +420,14 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
       terminalAt: abandonedAt,
       updatedAt: abandonedAt,
     }
-    this.state.sourceMaterializations.set(
-      sourceMaterializationKey(
-        materialization.projectId,
-        materialization.source.projectionId,
-        materialization.materializationId
-      ),
-      abandoned
+    const key = sourceMaterializationKey(
+      materialization.projectId,
+      materialization.source.projectionId,
+      materialization.materializationId
     )
+    this.state.sourceMaterializations.set(key, abandoned)
+    // Nothing applies an abandoned candidate's plan.
+    this.state.replacementPlans.delete(key)
     return sourceMaterializationRecord(abandoned)
   }
 

package/src/storage/ontology/in-memory/state-reader.ts

@@ -0,0 +1,245 @@
+import { MaterializationConflictError } from "../../../materialization/errors"
+import type { OntologyLinkRef, OntologyObjectRef } from "../../../materialization/model"
+import {
+  linkRefKey,
+  linkRefSortKey,
+  linkScopeKey,
+  linkScopeSortKey,
+  objectRefKey,
+  projectionEntityKey,
+} from "../../../materialization/refs"
+import {
+  getInMemoryObjectMaterializerAdapter,
+  type InMemoryObjectStorage,
+} from "../../objects/in-memory"
+import type { ObjectLinkRow } from "../../objects/types"
+import {
+  getInMemoryTimeseriesMaterializerAdapter,
+  type InMemoryTimeseriesStorage,
+} from "../../timeseries/store"
+import type {
+  MaterializationLinkScopeRevision,
+  MaterializationLinkScopeState,
+  MaterializationLinkState,
+  MaterializationObjectState,
+  StoredTelemetryPoint,
+} from "../materializations"
+import { appendScopeSnapshot, finishScopeAccumulator, startScopeAccumulator } from "../provider"
+import type {
+  StoredSourceAssertion,
+  StoredSourceLinkAssertion,
+  StoredSourceObjectAssertion,
+} from "../sources"
+import {
+  linkRef,
+  linkSnapshot,
+  objectSnapshot,
+  publicLinkOverride,
+  publicLinkSlotOverride,
+  publicObjectOverride,
+  storedPoint,
+} from "./materializations-state"
+import {
+  type InMemoryOntologyState,
+  type InMemoryOntologyStorageTestHooks,
+  projectEntityKey,
+} from "./shared-state"
+
+/** Reads one project's current ontology state for materialization sessions and plans. */
+export class InMemoryMaterializationStateReader {
+  constructor(
+    private readonly state: InMemoryOntologyState,
+    private readonly objects: InMemoryObjectStorage,
+    private readonly timeseries: InMemoryTimeseriesStorage,
+    private readonly hooks: InMemoryOntologyStorageTestHooks = {}
+  ) {}
+
+  async objectState(
+    projectId: string,
+    ref: OntologyObjectRef
+  ): Promise<MaterializationObjectState> {
+    const row = await this.objects.getByPrimaryId({
+      projectId,
+      objectTypeId: ref.objectTypeId,
+      primaryId: ref.primaryId,
+    })
+    return {
+      ref: structuredClone(ref),
+      source: this.findActiveObjectSource(projectId, ref),
+      override: structuredClone(
+        publicObjectOverride(
+          this.state.objectOverrides.get(projectEntityKey(projectId, objectRefKey(ref)))
+        )
+      ),
+      effective: row ? objectSnapshot(row) : null,
+      latestTelemetry: getInMemoryTimeseriesMaterializerAdapter(this.timeseries)
+        .listLatestForObject(projectId, ref.objectTypeId, ref.primaryId)
+        .map(storedPoint),
+    }
+  }
+
+  latestTelemetry(projectId: string, ref: OntologyObjectRef): StoredTelemetryPoint[] {
+    return getInMemoryTimeseriesMaterializerAdapter(this.timeseries)
+      .listLatestForObject(projectId, ref.objectTypeId, ref.primaryId)
+      .map(storedPoint)
+  }
+
+  async linkState(projectId: string, ref: OntologyLinkRef): Promise<MaterializationLinkState> {
+    const row = getInMemoryObjectMaterializerAdapter(this.objects).getExactLinkRow(projectId, {
+      sourceTypeId: ref.source.objectTypeId,
+      sourceId: ref.source.primaryId,
+      linkId: ref.linkId,
+      targetTypeId: ref.target.objectTypeId,
+      targetId: ref.target.primaryId,
+    })
+    return {
+      ref: structuredClone(ref),
+      source: this.findActiveLinkSource(projectId, ref),
+      override: structuredClone(
+        publicLinkOverride(
+          this.state.linkOverrides.get(projectEntityKey(projectId, linkRefKey(ref)))
+        )
+      ),
+      slotOverride: structuredClone(
+        publicLinkSlotOverride(
+          this.state.linkSlotOverrides.get(
+            projectEntityKey(projectId, linkScopeKey(ref.source, ref.linkId))
+          )
+        )
+      ),
+      effective: row ? linkSnapshot(row) : null,
+    }
+  }
+
+  linkSlotState(
+    projectId: string,
+    source: OntologyObjectRef,
+    linkId: string
+  ): MaterializationLinkScopeState {
+    const value = this.effectiveLinkScope(projectId, source, linkId)
+    return {
+      ...value,
+      sourceAssertion: this.findActiveLinkScopeSource(projectId, source, linkId),
+      override: structuredClone(
+        publicLinkSlotOverride(
+          this.state.linkSlotOverrides.get(
+            projectEntityKey(projectId, linkScopeKey(source, linkId))
+          )
+        )
+      ),
+    }
+  }
+
+  effectiveLinkScope(
+    projectId: string,
+    source: OntologyObjectRef,
+    linkId: string
+  ): MaterializationLinkScopeRevision & Pick<MaterializationLinkScopeState, "effective"> {
+    const rows: ObjectLinkRow[] = []
+    getInMemoryObjectMaterializerAdapter(this.objects).visitExactScopeLinks(
+      projectId,
+      source.objectTypeId,
+      source.primaryId,
+      linkId,
+      (row) => rows.push(row)
+    )
+    rows.sort((left, right) =>
+      linkRefSortKey(linkRef(left)).localeCompare(linkRefSortKey(linkRef(right)))
+    )
+    const accumulator = startScopeAccumulator(source, linkId)
+    for (let offset = 0; offset < rows.length; offset += 1_000) {
+      const page = rows.slice(offset, offset + 1_000)
+      this.hooks.observeBuffer?.("state.link-scope.page", page.length)
+      for (const row of page) {
+        appendScopeSnapshot(accumulator, linkSnapshot(row))
+      }
+    }
+    const effective = rows.length === 1 ? linkSnapshot(rows[0]!) : null
+    return {
+      ...finishScopeAccumulator(accumulator),
+      effective,
+    }
+  }
+
+  findActiveLinkScopeSource(
+    projectId: string,
+    source: OntologyObjectRef,
+    linkId: string
+  ): StoredSourceLinkAssertion | null {
+    const rows = this.state.activeSourceLinkScopes.get(
+      projectEntityKey(projectId, linkScopeSortKey(source, linkId))
+    )
+    if (rows && rows.size > 1)
+      throw new MaterializationConflictError(
+        "source-materialization",
+        `Multiple active source links assert cardinality-one scope '${source.objectTypeId}.${linkId}'.`
+      )
+    return structuredClone(rows?.values().next().value ?? null)
+  }
+
+  findActiveObjectSource(
+    projectId: string,
+    ref: OntologyObjectRef
+  ): StoredSourceObjectAssertion | null {
+    const found = this.findActiveSource(projectId, projectionEntityKey({ kind: "object", ref }))
+    return found?.assertion.kind === "object" ? (found as StoredSourceObjectAssertion) : null
+  }
+
+  findActiveLinkSource(projectId: string, ref: OntologyLinkRef): StoredSourceLinkAssertion | null {
+    const found = this.findActiveSource(projectId, projectionEntityKey({ kind: "link", ref }))
+    return found?.assertion.kind === "link" ? (found as StoredSourceLinkAssertion) : null
+  }
+
+  findActiveSource(projectId: string, entityKey: string): StoredSourceAssertion | null {
+    const rows = this.state.activeSourceRows.get(projectEntityKey(projectId, entityKey))
+    if (rows && rows.size > 1)
+      throw new MaterializationConflictError(
+        "source-materialization",
+        `Multiple active sources assert ${entityKey}.`
+      )
+    return structuredClone(rows?.values().next().value ?? null)
+  }
+
+  incidentLinkIndex(projectId: string): Map<string, readonly OntologyLinkRef[]> {
+    const mutable = new Map<string, Map<string, OntologyLinkRef>>()
+    const consider = (ref: OntologyLinkRef): void => {
+      const sortKey = linkRefSortKey(ref)
+      for (const endpoint of [ref.source, ref.target]) {
+        const key = objectRefKey(endpoint)
+        const links = mutable.get(key) ?? new Map<string, OntologyLinkRef>()
+        links.set(sortKey, structuredClone(ref))
+        mutable.set(key, links)
+      }
+    }
+    getInMemoryObjectMaterializerAdapter(this.objects).visitExactLinks(projectId, (row) =>
+      consider(linkRef(row))
+    )
+    for (const override of this.state.linkOverrides.values()) {
+      if (override.projectId === projectId) consider(override.ref)
+    }
+    for (const override of this.state.linkSlotOverrides.values()) {
+      if (override.projectId !== projectId) continue
+      consider({
+        source: override.ref.source,
+        linkId: override.ref.linkId,
+        target: override.value.target,
+      })
+    }
+    for (const [scopeKey, rows] of this.state.activeSourceLinkScopes) {
+      if (JSON.parse(scopeKey)[0] !== projectId) continue
+      for (const row of rows.values()) {
+        if (row.assertion.kind === "link") consider(row.assertion.ref)
+      }
+    }
+    const index = new Map<string, readonly OntologyLinkRef[]>()
+    for (const [objectKey, refs] of mutable) {
+      index.set(
+        objectKey,
+        [...refs.entries()]
+          .sort(([left], [right]) => left.localeCompare(right))
+          .map(([, ref]) => ref)
+      )
+    }
+    return index
+  }
+}

package/src/storage/ontology/index.ts

@@ -1,6 +1,7 @@
 import type { OntologyCommitStorage } from "./commits"
 import type { OntologyMaterializationStorage } from "./materializations"
 import type { OntologyOutboxStorage } from "./outbox"
+import type { OntologyReplacementPlanStorage } from "./replacement-plans"
 import type { OntologySourceStorage } from "./sources"
 import type { OntologyVectorIndexingStorage } from "./vector-indexing"
 import type { OntologyVectorStorage } from "./vectors"
@@ -22,22 +23,20 @@ export type {
   TelemetryOntologyCommitIntent,
 } from "./commits"
 export type {
-  ApplyMaterializationChunkInput,
+  AppliedMaterialization,
+  ApplyMaterializationInput,
   ApplyMaterializationResult,
   ExactEffectiveLinkDelete,
   ExactEffectiveLinkWrite,
   ExactEffectiveObjectDelete,
   ExactEffectiveObjectWrite,
-  ExactEffectiveWrites,
   ExactLinkOverrideDelete,
   ExactLinkOverrideWrite,
   ExactLinkSlotOverrideDelete,
   ExactLinkSlotOverrideWrite,
   ExactObjectOverrideDelete,
   ExactObjectOverrideWrite,
-  ExactOverrideWrites,
   ExactTimeseriesPointWrite,
-  ExactTimeseriesWrites,
   ExpectedSourceRevision,
   ExpectedTimeseriesPointRevision,
   FinalizeMaterializationInput,
@@ -53,7 +52,6 @@ export type {
   MaterializationObjectExistence,
   MaterializationObjectExistenceWorkRecord,
   MaterializationObjectState,
-  MaterializationPlanChunk,
   MaterializationPlanFinalization,
   MaterializationPlanHeader,
   MaterializationPlanWorkItem,
@@ -61,23 +59,21 @@ export type {
   MaterializationSession,
   MaterializationStatePage,
   MaterializationStateRequestChunk,
+  MaterializationVectorChange,
+  MaterializationVectorChangePage,
   MaterializationWorkEntityKind,
-  MaterializationWorkPage,
   MaterializationWorkRecord,
   OntologyMaterializationStorage,
-  ReadMaterializationObjectExistenceInput,
   SourceActivationWrite,
   SourceReplacementLinkState,
   SourceReplacementObjectState,
-  SourceReplacementStatePage,
   StageMaterializationWorkInput,
   StoredLinkOverride,
   StoredLinkSlotOverride,
   StoredObjectOverride,
   StoredTelemetryPoint,
   StreamMaterializationStateInput,
-  StreamMaterializationWorkInput,
-  StreamSourceReplacementStateInput,
+  StreamMaterializationVectorChangesInput,
 } from "./materializations"
 export type {
   ClaimedOntologyOutboxRow,
@@ -90,12 +86,23 @@ export type {
   OntologyOutboxRecord,
   OntologyOutboxStorage,
   OntologyOutboxSummary,
-  OntologyOutboxWrite,
   PurgePublishedOntologyOutboxInput,
   RescheduleOntologyOutboxLeaseInput,
   SummarizeOntologyOutboxInput,
 } from "./outbox"
 export { ONTOLOGY_OUTBOX_FAILURE_CODES } from "./outbox"
+export type {
+  OntologyReplacementPlanStorage,
+  OpenedReplacementPlan,
+  OpenReplacementPlanInput,
+  PlannedReplacementIdentity,
+  PurgeReplacementPlansInput,
+  ReplacementPlanRef,
+  ReplacementPlanStatePage,
+  ReplacementPlanStatus,
+  StageReplacementPlanInput,
+  StreamReplacementPlanStateInput,
+} from "./replacement-plans"
 export type {
   AbandonRunSourceMaterializationInput,
   AbandonSourceMaterializationCandidateInput,
@@ -120,8 +127,6 @@ export type {
   StoredSourceAssertion,
   StoredSourceLinkAssertion,
   StoredSourceObjectAssertion,
-  SummarizeTerminalSourceMaterializationsInput,
-  TerminalSourceMaterializationSummary,
 } from "./sources"
 
 export type { ObjectVectorState, OntologyVectorStorage, StoredObjectVector } from "./vectors"
@@ -132,6 +137,7 @@ export interface OntologyStorage {
   readonly commits: OntologyCommitStorage
   readonly sources: OntologySourceStorage
   readonly materializations: OntologyMaterializationStorage
+  readonly replacementPlans: OntologyReplacementPlanStorage
   readonly outbox: OntologyOutboxStorage
 }