@sixb/core 0.1.16 → 0.1.17

package/src/storage/migrations/types.ts

@@ -1,3 +1,5 @@
+import type { OntologyDefinitionCatalog } from "../../ontology/registry"
+import type { ObjectQueryPreparationResult } from "../objects/query-preparation"
 import type { Storage } from "../types"
 
 export interface MigrationStepInfo {
@@ -123,7 +125,13 @@ export interface MigrationCapableStorage extends Storage {
   readonly migrators: readonly StorageMigrator[]
 }
 
+export interface StorageMigrationOptions {
+  readonly projectId: string
+  readonly ontology: OntologyDefinitionCatalog
+}
+
 export interface StorageMigrationResult {
+  readonly queries?: ObjectQueryPreparationResult
   readonly status: "migrated" | "current" | "skipped"
   readonly reports: readonly MigrationReport[]
 }

package/src/storage/objects/in-memory/query.ts

@@ -117,6 +117,8 @@ export function evaluateObjectQuery(
   source: InMemoryReadSource
 ): QueryEvaluation {
   switch (query.kind) {
+    case "rerank":
+      throw new Error("[Sixb] Reranking must execute in the model runtime, not storage.")
     case "start":
       return evaluateStart(query.objectTypeId, source)
     case "refs":

package/src/storage/objects/in-memory/selected-reader.ts

@@ -105,6 +105,7 @@ export function createInMemorySelectedReader(
       const result = evaluateObjectQuery(stripOuterRowShape(input.query), universe)
       return visible(
         structuredClone({
+          total: result.entries.length,
           facets: buildFacetResults(
             result.entries.map((entry) => entry.row),
             input.facets

package/src/storage/objects/in-memory/storage.ts

@@ -305,6 +305,7 @@ export class InMemoryObjectStorage implements ObjectStorage {
       this.readSource(params.projectId)
     )
     return {
+      total: result.entries.length,
       facets: buildFacetResults(
         result.entries.map((entry) => entry.row),
         params.facets

package/src/storage/objects/query-preparation.ts

@@ -0,0 +1,77 @@
+import type { QueryScalarKind } from "../../objects/query/ir"
+import {
+  queryScalarKindForSchema,
+  resolvePropertyQueryCapabilities,
+  resolveQuerySchema,
+} from "../../ontology/query-capabilities"
+import type { OntologyDefinitionCatalog } from "../../ontology/registry"
+import type { ObjectQueryIndexDefinition } from "../../ontology/types"
+import type { Storage } from "../types"
+
+export interface ObjectQueryPreparationPlan {
+  readonly projectId: string
+  readonly objectTypes: readonly {
+    readonly objectTypeId: string
+    readonly properties: readonly {
+      readonly propertyId: string
+      readonly scalarKind?: QueryScalarKind
+      readonly filterable: boolean
+      readonly sortable: boolean
+      readonly text: boolean
+    }[]
+    readonly indexes: readonly ObjectQueryIndexDefinition[]
+  }[]
+}
+
+export interface ObjectQueryPreparationResult {
+  readonly status: "prepared" | "current"
+  readonly objectTypes: number
+  readonly indexes: number
+  readonly warnings: readonly string[]
+}
+
+/** Optional maintenance capability, separate from object reads and writes. */
+export interface QueryPreparationCapableStorage extends Storage {
+  /** Coordinate concurrent callers and skip completed plans. Backfills may block writes. */
+  prepareObjectQueries(plan: ObjectQueryPreparationPlan): Promise<ObjectQueryPreparationResult>
+}
+
+export function isQueryPreparationCapableStorage(
+  storage: Storage
+): storage is QueryPreparationCapableStorage {
+  return "prepareObjectQueries" in storage && typeof storage.prepareObjectQueries === "function"
+}
+
+/** Prepare declared query access paths after schema migration. Providers own physical layout;
+ * applications describe their ontology, not PostgreSQL expressions or generated columns.
+ * This is maintenance: backfills may block writes and must not run in a request handler. */
+export async function prepareObjectQueries(input: {
+  projectId: string
+  ontology: OntologyDefinitionCatalog
+  storage: Storage
+}): Promise<ObjectQueryPreparationResult> {
+  if (!isQueryPreparationCapableStorage(input.storage)) {
+    throw new Error("[Sixb] This storage provider does not support object query preparation")
+  }
+
+  const values = input.ontology.getValueTypesById()
+  const objectTypes = input.ontology.listObjectTypes().map((type) => ({
+    objectTypeId: type.id,
+    properties: type.properties
+      .filter((property) => property.mode !== "telemetry" && !property.primary)
+      .map((property) => {
+        const capabilities = resolvePropertyQueryCapabilities(property, values)
+        const schema = resolveQuerySchema(property.schema, values)
+        return {
+          propertyId: property.id,
+          scalarKind: schema ? queryScalarKindForSchema(schema) : undefined,
+          filterable: capabilities.operators.includes("eq"),
+          sortable: capabilities.sortable,
+          text: capabilities.text,
+        }
+      })
+      .filter((property) => property.filterable || property.sortable || property.text),
+    indexes: type.query?.indexes ?? [],
+  }))
+  return input.storage.prepareObjectQueries({ projectId: input.projectId, objectTypes })
+}

package/src/storage/objects/types.ts

@@ -261,6 +261,8 @@ export interface FacetObjectsInput {
 }
 
 export interface FacetObjectsResult {
+  /** Exact matching rows, including absent facet values and buckets beyond the limit. */
+  total: number
   facets: readonly ObjectFacetResult[]
 }
 

package/src/storage/ontology/commits.ts

@@ -1,5 +1,3 @@
-import type { AuthorizablePrincipal } from "../../execution/types"
-import type { EventExecutor } from "../../materialization/events"
 import type {
   EditCommitResult,
   OntologyMaterializationOrigin,
@@ -56,12 +54,9 @@ interface OntologyCommitFields {
   readonly id: string
   readonly idempotencyKey: string
   readonly requestHash: string
+  /** The execution the commit ran under: who asked for it and what wrote it live there. */
   readonly executionId: string
   readonly origin: OntologyMaterializationOrigin
-  /** Principal on whose behalf the execution ran, copied from `ExecutionRecord.requestedBy`. */
-  readonly requestedBy?: AuthorizablePrincipal
-  /** Workload that wrote the commit, copied from its execution. */
-  readonly executor: EventExecutor
   readonly ontologyRevision: string
   readonly projectionRevision?: string
   readonly ownershipHash?: string

package/src/storage/ontology/in-memory/index.ts

@@ -7,14 +7,17 @@ import type { AssertSourceMaterializationExecution } from "../sources"
 import { InMemoryOntologyCommitStorage } from "./commits"
 import { InMemoryOntologyMaterializationStorage } from "./materializations"
 import { InMemoryOntologyOutboxStorage } from "./outbox"
+import { InMemoryOntologyReplacementPlanStorage } from "./replacement-plans"
 import {
   cloneOntologyState,
   createInMemoryOntologyState,
+  type InMemoryExecutionReader,
   type InMemoryOntologyState,
   type InMemoryOntologyStorageTestHooks,
   restoreOntologyState,
 } from "./shared-state"
 import { InMemoryOntologySourceStorage } from "./sources"
+import { InMemoryMaterializationStateReader } from "./state-reader"
 import {
   type InMemoryOntologyStorageTestingAdapter,
   registerInMemoryOntologyStorageTestingAdapter,
@@ -27,7 +30,7 @@ interface InMemoryOntologyStorageOptions {
   readonly getTransactionToken: () => object | null
   readonly getMaterializationLifecycle: () => ProviderMaterializationTransactionLifecycle | null
   readonly assertSourceMaterializationExecution: AssertSourceMaterializationExecution
-  readonly executionExists: (projectId: string, executionId: string) => Promise<boolean>
+  readonly readExecution: InMemoryExecutionReader
 }
 
 import { InMemoryVectorIndexingStorage } from "./vector-indexing"
@@ -40,6 +43,7 @@ export class InMemoryOntologyStorage implements OntologyStorage {
   readonly commits: InMemoryOntologyCommitStorage
   readonly sources: InMemoryOntologySourceStorage
   readonly materializations: InMemoryOntologyMaterializationStorage
+  readonly replacementPlans: InMemoryOntologyReplacementPlanStorage
   readonly outbox: InMemoryOntologyOutboxStorage
   private testHooks: InMemoryOntologyStorageTestHooks = {}
   private readonly testingAdapter: InMemoryOntologyStorageTestingAdapter = {
@@ -72,20 +76,35 @@ export class InMemoryOntologyStorage implements OntologyStorage {
       options.runRootOperation,
       options.assertSourceMaterializationExecution
     )
-    this.outbox = new InMemoryOntologyOutboxStorage(this.state, options.runRootOperation)
+    this.outbox = new InMemoryOntologyOutboxStorage(
+      this.state,
+      options.runRootOperation,
+      options.readExecution
+    )
+    const hooks: InMemoryOntologyStorageTestHooks = {
+      beforeRead: (boundary) => this.testHooks.beforeRead?.(boundary),
+      beforeWrite: (boundary, ordinal) => this.testHooks.beforeWrite?.(boundary, ordinal),
+      observeBuffer: (boundary, rows) => this.testHooks.observeBuffer?.(boundary, rows),
+      observeWork: (records) => this.testHooks.observeWork?.(records),
+    }
+    const reader = new InMemoryMaterializationStateReader(this.state, objects, timeseries, hooks)
     this.materializations = new InMemoryOntologyMaterializationStorage(
       this.state,
       objects,
       timeseries,
+      reader,
       options.getTransactionToken,
       options.getMaterializationLifecycle,
-      options.executionExists,
-      {
-        beforeRead: (boundary) => this.testHooks.beforeRead?.(boundary),
-        beforeWrite: (boundary, ordinal) => this.testHooks.beforeWrite?.(boundary, ordinal),
-        observeBuffer: (boundary, rows) => this.testHooks.observeBuffer?.(boundary, rows),
-        observeWork: (records) => this.testHooks.observeWork?.(records),
-      }
+      options.readExecution,
+      hooks
+    )
+    this.replacementPlans = new InMemoryOntologyReplacementPlanStorage(
+      this.state,
+      objects,
+      reader,
+      options.runRootOperation,
+      options.assertSourceMaterializationExecution,
+      hooks
     )
     registerInMemoryOntologyStorageTestingAdapter(this, this.testingAdapter)
   }

package/src/storage/ontology/in-memory/materializations-finalization.ts

@@ -8,7 +8,7 @@ import type {
   PinnedDatasetVersion,
   TelemetryCommitResult,
 } from "../../../materialization/model"
-import { linkRefSortKey } from "../../../materialization/refs"
+import { linkRefKey, linkRefSortKey, objectRefKey } from "../../../materialization/refs"
 import {
   getInMemoryObjectMaterializerAdapter,
   type InMemoryObjectStorage,
@@ -34,12 +34,12 @@ export function assertFinalizationCorrelations(
     result.created !== true ||
     !Number.isSafeInteger(result.eventCount) ||
     result.eventCount < 0 ||
-    result.eventCount !== session.outboxEnvelopes.size
+    result.eventCount !== session.outboxWrites.size
   ) {
     invalidCorrelation("Materialization result does not correlate with its commit intent.")
   }
   for (let ordinal = 0; ordinal < result.eventCount; ordinal += 1) {
-    if (!session.outboxEnvelopes.has(ordinal)) {
+    if (!session.outboxWrites.has(ordinal)) {
       invalidCorrelation("Outbox event ordinals must be contiguous from zero.")
     }
   }
@@ -63,7 +63,7 @@ export function assertFinalizationCorrelations(
     assertSourceActivationCorrelation(activation, session, state)
   }
   if (sourceActivations[0]) {
-    assertReplacementFullyStreamed(session, sourceActivations[0])
+    assertPlanCoverage(session, sourceActivations[0])
   }
   assertFinalizedWork(session, state, objects)
   if (commit.intent.kind === "projection") {
@@ -317,16 +317,8 @@ function assertFinalizedWork(
   state: InMemoryOntologyState,
   objects: InMemoryObjectStorage
 ): void {
-  const expectedPlanItems = session.applyWork.map((record) => stableJsonStringify(record.item))
-  const appliedPlanItems = session.appliedPlanItems
-  if (expectedPlanItems.length > 0 && !session.workStreams.apply.completed) {
-    invalidCorrelation("Materialization plan work was not fully streamed.")
-  }
-  if (stableJsonStringify(appliedPlanItems) !== stableJsonStringify(expectedPlanItems)) {
-    invalidCorrelation("Materialization plan work was not applied exactly once.")
-  }
-  if (session.cardinalityWork.length > 0 && !session.workStreams.cardinality.completed) {
-    invalidCorrelation("Materialization cardinality work was not fully validated.")
+  if (!session.applied) {
+    invalidCorrelation("A materialization plan must apply before it finalizes.")
   }
   assertFinalCardinality(session.cardinalityWork, session.header.commit.projectId, objects)
   if (session.header.commit.intent.kind === "telemetry") {
@@ -339,28 +331,25 @@ function assertFinalizedWork(
   }
 
   const expectedEvents = [...session.eventWork].sort(compareEventWork)
-  if (expectedEvents.length > 0 && !session.workStreams.event.completed) {
-    invalidCorrelation("Materialization event work was not fully drained.")
-  }
-  if (session.outboxEnvelopes.size !== expectedEvents.length) {
+  if (session.outboxWrites.size !== expectedEvents.length) {
     invalidCorrelation("Materialization event work was not fully written to the outbox.")
   }
   for (let ordinal = 0; ordinal < expectedEvents.length; ordinal += 1) {
     const expected = expectedEvents[ordinal]
-    const actual = session.outboxEnvelopes.get(ordinal)
+    const actual = session.outboxWrites.get(ordinal)
     if (!expected || !actual) {
       invalidCorrelation("Materialization event work was not fully written to the outbox.")
     }
-    const { id, commitOrdinal, ...actualDraft } = actual
     if (
-      commitOrdinal !== ordinal ||
-      id !== createEventId(session.header.commit.projectId, session.header.commit.id, ordinal) ||
-      stableJsonStringify(actualDraft) !== stableJsonStringify(expected.draft)
+      actual.commitOrdinal !== ordinal ||
+      actual.id !==
+        createEventId(session.header.commit.projectId, session.header.commit.id, ordinal) ||
+      stableJsonStringify(actual.event) !== stableJsonStringify(expected.draft)
     ) {
       invalidCorrelation("Materialization outbox event does not match its staged event work.")
     }
     const persisted = state.outbox.get(outboxKey(session.header.commit.projectId, actual.id))
-    if (!persisted || stableJsonStringify(persisted.envelope) !== stableJsonStringify(actual)) {
+    if (!persisted || stableJsonStringify(persisted.event) !== stableJsonStringify(actual.event)) {
       invalidCorrelation("Materialization outbox event was not persisted.")
     }
   }
@@ -380,26 +369,15 @@ function assertFinalCardinality(
       readonly occupiedLinkKeys: Set<string>
     }
   >()
-  let currentScope: string | null = null
-  let occupiedCount = 0
+  // Apply already rejected a scope with two staged occupants; this checks what it wrote.
   for (const record of effectiveOccupants) {
-    if (record.scopeSortKey !== currentScope) {
-      currentScope = record.scopeSortKey
-      occupiedCount = 0
-    }
     const scope = scopes.get(record.scopeSortKey) ?? {
       source: structuredClone(record.ref.source),
       linkId: record.ref.linkId,
       occupiedLinkKeys: new Set<string>(),
     }
     scopes.set(record.scopeSortKey, scope)
-    if (record.occupied) {
-      occupiedCount += 1
-      scope.occupiedLinkKeys.add(record.linkSortKey)
-      if (occupiedCount > 1) {
-        invalidCorrelation("Materialization cardinality work violates cardinality-one.")
-      }
-    }
+    if (record.occupied) scope.occupiedLinkKeys.add(record.linkSortKey)
   }
 
   const adapter = getInMemoryObjectMaterializerAdapter(objects)
@@ -422,48 +400,37 @@ function assertFinalCardinality(
   }
 }
 
-function assertReplacementFullyStreamed(
+/** Every identity the plan decides with a diff is classified once, and nothing else is. */
+function assertPlanCoverage(
   session: SessionState,
   activation: import("../materializations").SourceActivationWrite
 ): void {
-  const replacement = session.replacement
+  const plan = session.plan
   if (
-    !replacement ||
-    replacement.sourceId !== activation.source.projectionId ||
-    replacement.candidateMaterializationId !== activation.materializationId ||
-    replacement.candidate.materializationId !== activation.materializationId ||
-    replacement.candidate.projectionKind !== activation.projectionKind ||
-    replacement.candidate.protocol !== activation.protocol
+    !plan ||
+    plan.sourceId !== activation.source.projectionId ||
+    plan.materializationId !== activation.materializationId ||
+    plan.projectionKind !== activation.projectionKind
   ) {
-    invalidCorrelation(
-      "Source activation does not match the replacement candidate opened by the session."
-    )
+    invalidCorrelation("Source activation does not match the replacement plan the session applies.")
   }
-  if (
-    activation.projectionKind === "object" &&
-    (!replacement.objectStreamCompleted || !replacement.linkStreamCompleted)
-  ) {
-    invalidCorrelation("Object projection replacement state was not fully streamed.")
-  }
-  if (activation.projectionKind === "link" && !replacement.linkStreamCompleted) {
-    invalidCorrelation("Link projection replacement state was not fully streamed.")
-  }
-  const expectedObjectKeys =
-    activation.projectionKind === "object" ? [...replacement.objects.keys()].sort() : []
-  const expectedLinkKeys = [...replacement.links.entries()]
-    .filter(([, value]) => value.diffRequired)
-    .map(([key]) => key)
-    .sort()
+  const expected = (kind: "object" | "link") =>
+    [...plan.identities.values()]
+      .filter((identity) => identity.entity.kind === kind && identity.diffRequired)
+      .map((identity) =>
+        identity.entity.kind === "object"
+          ? objectRefKey(identity.entity.ref)
+          : linkRefKey(identity.entity.ref)
+      )
+      .sort()
   if (
     stableJsonStringify(classificationKeys(session, "object")) !==
-      stableJsonStringify(expectedObjectKeys) ||
+      stableJsonStringify(expected("object")) ||
     stableJsonStringify(classificationKeys(session, "link")) !==
-      stableJsonStringify(expectedLinkKeys) ||
+      stableJsonStringify(expected("link")) ||
     classificationKeys(session, "point").length > 0
   ) {
-    invalidCorrelation(
-      "Projection replacement classification coverage does not match its streamed state."
-    )
+    invalidCorrelation("Projection replacement classification coverage does not match its plan.")
   }
 }
 

package/src/storage/ontology/in-memory/materializations-state.ts

@@ -4,7 +4,6 @@ import type {
   EffectiveObjectSnapshot,
   OntologyLinkRef,
 } from "../../../materialization/model"
-import { linkRefKey, linkRefSortKey, linkScopeSortKey } from "../../../materialization/refs"
 import type { ObjectLinkRow, ObjectRow } from "../../objects/types"
 import type { TimeseriesPoint } from "../../timeseries/types"
 import type {
@@ -19,7 +18,6 @@ import type {
   StoredSourceLinkAssertion,
   StoredSourceObjectAssertion,
 } from "../sources"
-import type { ReplacementSessionState } from "./materializations"
 import type {
   InMemoryOntologyState,
   InMemorySourceMaterialization,
@@ -158,23 +156,3 @@ export function storedSourceLink(
   if (!materializationId || !row || row.assertion.kind !== "link") return null
   return storedSource(sourceId, materializationId, row) as StoredSourceLinkAssertion
 }
-
-export function addReplacementLink(
-  replacement: ReplacementSessionState,
-  ref: OntologyLinkRef,
-  diffRequired: boolean
-): void {
-  const key = linkRefKey(ref)
-  const existing = replacement.links.get(key)
-  if (existing) {
-    existing.diffRequired ||= diffRequired
-    if (diffRequired) replacement.affectedScopes.add(linkScopeSortKey(ref.source, ref.linkId))
-    return
-  }
-  replacement.links.set(key, {
-    ref: structuredClone(ref),
-    sortKey: linkRefSortKey(ref),
-    diffRequired,
-  })
-  if (diffRequired) replacement.affectedScopes.add(linkScopeSortKey(ref.source, ref.linkId))
-}

package/src/storage/ontology/in-memory/materializations-work.ts

@@ -1,57 +1,12 @@
-import { stableJsonStringify } from "../../../json"
 import { MaterializationConflictError } from "../../../materialization/errors"
 import type {
-  MaterializationPlanChunk,
   StoredLinkOverride,
   StoredLinkSlotOverride,
   StoredObjectOverride,
 } from "../materializations"
-import { invalidCorrelation, materializationPlanItems } from "../provider-work"
-import type { SessionState } from "./materializations"
 
 export * from "../provider-work"
 
-export function assertChunkSequence(session: SessionState, chunk: MaterializationPlanChunk): void {
-  const planItems = materializationPlanItems(chunk)
-  const applyStream = session.workStreams.apply
-  const appliedStart = session.appliedPlanItems.length
-  if (
-    planItems.length > 0 &&
-    (!applyStream.started || appliedStart + planItems.length > applyStream.emittedCount)
-  ) {
-    invalidCorrelation("Materialization plan items cannot be applied before they are streamed.")
-  }
-  for (let offset = 0; offset < planItems.length; offset += 1) {
-    const expected = session.applyWork[appliedStart + offset]?.item
-    if (!expected || stableJsonStringify(planItems[offset]) !== stableJsonStringify(expected)) {
-      invalidCorrelation("Materialization plan items must be applied in exact streamed order.")
-    }
-  }
-
-  const eventStream = session.workStreams.event
-  const appliedOutboxCount = session.outboxEnvelopes.size
-  if (
-    chunk.outbox.length > 0 &&
-    (!eventStream.started || appliedOutboxCount + chunk.outbox.length > eventStream.emittedCount)
-  ) {
-    invalidCorrelation("Materialization events cannot be applied before they are streamed.")
-  }
-  for (let offset = 0; offset < chunk.outbox.length; offset += 1) {
-    const expected = session.eventWork[appliedOutboxCount + offset]
-    const actual = chunk.outbox[offset]?.envelope
-    if (!expected || !actual) {
-      invalidCorrelation("Materialization outbox events must follow exact streamed order.")
-    }
-    const { id: _id, commitOrdinal, ...actualDraft } = actual
-    if (
-      commitOrdinal !== appliedOutboxCount + offset ||
-      stableJsonStringify(actualDraft) !== stableJsonStringify(expected.draft)
-    ) {
-      invalidCorrelation("Materialization outbox events must follow exact streamed order.")
-    }
-  }
-}
-
 export function assertLastCommit(
   value: StoredObjectOverride | StoredLinkOverride | StoredLinkSlotOverride | undefined,
   expected: string | null,