@sixb/core 0.1.15 → 0.1.17

package/src/materializer/effective/build-events.ts

@@ -1,16 +1,11 @@
 import type {
   EffectiveLinkChange,
   EffectiveObjectChange,
-  OntologyMaterializationOrigin,
   TelemetryPointWrite,
 } from "../../materialization/model"
 import { linkRefSortKey, objectRefSortKey, telemetryPointSortKey } from "../../materialization/refs"
-import type {
-  OntologyMaterializationEvent,
-  OntologyMaterializationEventDraft,
-} from "../../storage/ontology"
-import type { MaterializerAttribution } from "../execution/scope"
-import { createEventId, materializationEventKindOrdinal } from "../shared/identity"
+import type { OntologyMaterializationEventDraft } from "../../storage/ontology"
+import { materializationEventKindOrdinal } from "../shared/identity"
 
 export interface OrderedMaterializationEventDraft {
   readonly kindRank: number
@@ -18,159 +13,83 @@ export interface OrderedMaterializationEventDraft {
   readonly draft: OntologyMaterializationEventDraft
 }
 
-export interface MaterializationEventDraftContext {
-  readonly projectId: string
-  readonly commitId: string
-  readonly committedAt: string
-  readonly correlationId: string
-  readonly origin: OntologyMaterializationOrigin
-  readonly attribution: MaterializerAttribution
-}
-
 export function buildObjectMaterializationEventDraft(
-  input: MaterializationEventDraftContext & { readonly change: EffectiveObjectChange }
+  change: EffectiveObjectChange
 ): OrderedMaterializationEventDraft {
-  const draft = buildObjectEventDraft(input, input.change)
-  return orderedDraft(objectRefSortKey(input.change.ref), draft)
+  return orderedDraft(objectRefSortKey(change.ref), buildObjectEventDraft(change))
 }
 
-function buildObjectEventDraft(
-  context: MaterializationEventDraftContext,
-  change: EffectiveObjectChange
-): OntologyMaterializationEventDraft {
-  const base = eventDraftBase(context)
-  const partitionKey = `${change.ref.objectTypeId}:${change.ref.primaryId}`
-
-  if (change.kind === "deleted") {
-    return {
-      ...base,
-      type: "object.deleted",
-      topic: "objects",
-      partitionKey,
-      payload: {
-        objectTypeId: change.ref.objectTypeId,
-        primaryId: change.ref.primaryId,
-        propertyChanges: change.propertyChanges,
-      },
-    }
-  }
-
-  const type = objectEventType(change)
-  return {
-    ...base,
-    type,
-    topic: "objects",
-    partitionKey,
-    payload: {
-      objectTypeId: change.ref.objectTypeId,
-      primaryId: change.ref.primaryId,
-      properties: change.after.properties,
-      propertyChanges: change.propertyChanges,
-    },
+function buildObjectEventDraft(change: EffectiveObjectChange): OntologyMaterializationEventDraft {
+  const identity = { objectTypeId: change.ref.objectTypeId, primaryId: change.ref.primaryId }
+  switch (change.kind) {
+    case "created":
+      return {
+        type: "object.created",
+        payload: { ...identity, properties: change.after.properties },
+      }
+    case "updated":
+      return {
+        type: "object.updated",
+        payload: {
+          ...identity,
+          properties: change.after.properties,
+          propertyChanges: change.propertyChanges,
+        },
+      }
+    case "deleted":
+      return {
+        type: "object.deleted",
+        payload: { ...identity, propertyChanges: change.propertyChanges },
+      }
   }
 }
 
-function objectEventType(
-  change: Exclude<EffectiveObjectChange, { readonly kind: "deleted" }>
-): "object.created" | "object.updated" {
-  if (change.kind === "created") return "object.created"
-  return "object.updated"
-}
-
 export function buildLinkMaterializationEventDraft(
-  input: MaterializationEventDraftContext & { readonly change: EffectiveLinkChange }
+  change: EffectiveLinkChange
 ): OrderedMaterializationEventDraft {
-  const draft = buildLinkEventDraft(input, input.change)
-  return orderedDraft(linkRefSortKey(input.change.ref), draft)
+  return orderedDraft(linkRefSortKey(change.ref), buildLinkEventDraft(change))
 }
 
-function buildLinkEventDraft(
-  context: MaterializationEventDraftContext,
-  change: EffectiveLinkChange
-): OntologyMaterializationEventDraft {
-  const base = eventDraftBase(context)
-  const partitionKey = `${change.ref.source.objectTypeId}:${change.ref.source.primaryId}:${change.ref.linkId}`
-  const commonPayload = {
+function buildLinkEventDraft(change: EffectiveLinkChange): OntologyMaterializationEventDraft {
+  const identity = {
     sourceTypeId: change.ref.source.objectTypeId,
     sourceId: change.ref.source.primaryId,
     linkId: change.ref.linkId,
     targetTypeId: change.ref.target.objectTypeId,
     targetId: change.ref.target.primaryId,
-    propertyChanges: change.propertyChanges,
   }
-
   if (change.kind === "deleted") {
     return {
-      ...base,
       type: "link.deleted",
-      topic: "links",
-      partitionKey,
-      payload: commonPayload,
+      payload: { ...identity, propertyChanges: change.propertyChanges },
     }
   }
-
-  const type = linkEventType(change)
-  if (change.after.properties === undefined) {
-    return { ...base, type, topic: "links", partitionKey, payload: commonPayload }
+  const properties =
+    change.after.properties === undefined ? {} : { properties: change.after.properties }
+  if (change.kind === "created") {
+    return { type: "link.created", payload: { ...identity, ...properties } }
   }
   return {
-    ...base,
-    type,
-    topic: "links",
-    partitionKey,
-    payload: { ...commonPayload, properties: change.after.properties },
+    type: "link.updated",
+    payload: { ...identity, ...properties, propertyChanges: change.propertyChanges },
   }
 }
 
-function linkEventType(
-  change: Exclude<EffectiveLinkChange, { readonly kind: "deleted" }>
-): "link.created" | "link.updated" {
-  if (change.kind === "created") return "link.created"
-  return "link.updated"
-}
-
 export function buildTelemetryMaterializationEventDraft(
-  input: MaterializationEventDraftContext & { readonly point: TelemetryPointWrite }
-): OrderedMaterializationEventDraft {
-  const draft = buildTelemetryEventDraft(input, input.point)
-  return orderedDraft(telemetryPointSortKey(input.point.series, input.point.at), draft)
-}
-
-function buildTelemetryEventDraft(
-  context: MaterializationEventDraftContext,
   point: TelemetryPointWrite
-): OntologyMaterializationEventDraft {
-  const base = eventDraftBase(context)
-  const partitionKey = `${point.series.object.objectTypeId}:${point.series.object.primaryId}:${point.series.propertyId}`
+): OrderedMaterializationEventDraft {
   const payload = {
     objectTypeId: point.series.object.objectTypeId,
     objectId: point.series.object.primaryId,
     propertyId: point.series.propertyId,
     value: point.value,
     at: point.at,
+    ...(point.unit === undefined ? {} : { unit: point.unit }),
   }
-  if (point.unit === undefined) {
-    return { ...base, type: "telemetry.appended", topic: "telemetry", partitionKey, payload }
-  }
-  return {
-    ...base,
+  return orderedDraft(telemetryPointSortKey(point.series, point.at), {
     type: "telemetry.appended",
-    topic: "telemetry",
-    partitionKey,
-    payload: { ...payload, unit: point.unit },
-  }
-}
-
-function eventDraftBase(context: MaterializationEventDraftContext) {
-  return {
-    schemaVersion: 1 as const,
-    projectId: context.projectId,
-    occurredAt: context.committedAt,
-    correlationId: context.correlationId,
-    origin: context.origin,
-    ...context.attribution,
-    commitId: context.commitId,
-  }
+    payload,
+  })
 }
 
 function orderedDraft(
@@ -183,16 +102,3 @@ function orderedDraft(
     draft,
   }
 }
-
-export function sequenceMaterializationEvent(
-  projectId: string,
-  commitId: string,
-  commitOrdinal: number,
-  draft: OntologyMaterializationEventDraft
-): OntologyMaterializationEvent {
-  const sequence = {
-    id: createEventId(projectId, commitId, commitOrdinal),
-    commitOrdinal,
-  }
-  return { ...draft, ...sequence }
-}

package/src/materializer/effective/vector-indexing.ts

@@ -3,16 +3,15 @@ import type { OntologyObjectRef } from "../../materialization/model"
 import { vectorConfiguration, vectorSources } from "../../objects/vectors/profile"
 import type { Storage } from "../../storage"
 import { objectBatchKey } from "../../storage/objects/keys"
-import type { MaterializationSession } from "../../storage/ontology"
+import type { MaterializationSession, MaterializationVectorChange } from "../../storage/ontology"
 import type { VectorIndexingRequest } from "../../storage/ontology/vector-indexing"
 import type { MaterializerContext } from "../context"
-import type { MaterializationPlanItem } from "../execution/plan-stream"
 
 /** Source assertions are not effective values: schedule only after conflict resolution. */
 export async function scheduleVectorChanges(
   context: Pick<MaterializerContext, "projectId" | "ontology" | "clock">,
   storage: Storage,
-  items: readonly MaterializationPlanItem[],
+  items: readonly MaterializationVectorChange[],
   session: MaterializationSession,
   projection = false
 ): Promise<void> {

package/src/materializer/effective/vectors.ts

@@ -1,14 +1,13 @@
 import { objectRefKey } from "../../materialization/refs"
 import { vectorSources } from "../../objects/vectors/profile"
-import type { MaterializationSession } from "../../storage/ontology"
+import type { MaterializationSession, MaterializationVectorChange } from "../../storage/ontology"
 import type { OntologyVectorStorage } from "../../storage/ontology/vectors"
-import type { MaterializationPlanItem } from "../execution/plan-stream"
 
 /** Invalidate a bounded materialization page, including stored profiles no longer declared. */
 export async function invalidateVectorChanges(
   projectId: string,
   vectors: OntologyVectorStorage | undefined,
-  items: readonly MaterializationPlanItem[],
+  items: readonly MaterializationVectorChange[],
   session: MaterializationSession
 ): Promise<void> {
   if (!vectors) return

package/src/materializer/execution/scope.ts

@@ -1,37 +1,22 @@
 import { resolveExecutionScopeAuthorization } from "../../execution/authorization"
 import { ensureExecutionRecord, executionRecordInputFromRuntime } from "../../execution/durable"
-import type {
-  AuthorizablePrincipal,
-  ExecutionScope,
-  TrustedPrimitiveKind,
-} from "../../execution/types"
+import type { ExecutionScope, TrustedPrimitiveKind } from "../../execution/types"
 import {
   MaterializationConflictError,
   MaterializationValidationError,
 } from "../../materialization/errors"
-import type { EventExecutor } from "../../materialization/events"
 import type {
   CreateExecutionInput,
   ExecutionRecord,
   ExecutionStorage,
 } from "../../storage/executions"
 
-/**
- * Who and what a commit and its events are attributed to. Both are copied from the execution, so
- * they are a pure function of `executionId` and never an input the caller chooses.
- */
-export interface MaterializerAttribution {
-  readonly requestedBy?: AuthorizablePrincipal
-  readonly executor: EventExecutor
-}
-
 /** Immutable execution metadata attached to one prepared Materializer command. */
 export interface MaterializerExecution {
   readonly scope: ExecutionScope
   readonly record: CreateExecutionInput
   readonly executionId: string
   readonly correlationId: string
-  readonly attribution: MaterializerAttribution
 }
 
 /** Validate the process-local scope before any Materializer read or write is attempted. */
@@ -49,28 +34,6 @@ export function prepareMaterializerExecution(
     }),
     executionId: scope.execution.id,
     correlationId: scope.execution.correlationId,
-    attribution: executionAttribution(scope),
-  }
-}
-
-function executionAttribution(scope: ExecutionScope): MaterializerAttribution {
-  const { requestedBy, executor } = scope.execution
-  const attributed = { executor: eventExecutor(executor) }
-  if (requestedBy === undefined) return attributed
-  return { ...attributed, requestedBy: { type: requestedBy.type, id: requestedBy.id } }
-}
-
-/** Drop the process-local Agent actor id: the durable execution does not record it. */
-function eventExecutor(executor: ExecutionScope["execution"]["executor"]): EventExecutor {
-  switch (executor.type) {
-    case "request":
-      return { type: "request", requestId: executor.requestId }
-    case "primitive":
-      return { type: "primitive", kind: executor.kind, id: executor.id, runId: executor.runId }
-    case "agent":
-      return { type: "agent", runId: executor.runId }
-    case "kernel":
-      return { type: "kernel", operation: structuredClone(executor.operation) }
   }
 }
 

package/src/materializer/execution/work-executor.ts

@@ -1,4 +1,3 @@
-import { MaterializationValidationError } from "../../materialization/errors"
 import { utf8JsonByteLength } from "../../materialization/refs"
 import type { Storage } from "../../storage"
 import type {
@@ -7,32 +6,13 @@ import type {
   OntologyMaterializationStorage,
 } from "../../storage/ontology"
 import type { MaterializerContext } from "../context"
-import { sequenceMaterializationEvent } from "../effective/build-events"
 import { scheduleVectorChanges } from "../effective/vector-indexing"
 import { invalidateVectorChanges } from "../effective/vectors"
 import { throwIfAborted } from "../shared/abort"
 import { chunkBySize } from "../shared/chunking"
-import type { TimedCommitIdentity } from "../shared/identity"
-import { type MaterializationPlanItem, planStream } from "./plan-stream"
-import { outboxItem } from "./work-records"
 
 type BatchingContext = Pick<MaterializerContext, "batching">
 
-type EventContext = Pick<MaterializerContext, "batching" | "projectId">
-
-export async function applyItems(
-  context: BatchingContext,
-  storage: OntologyMaterializationStorage,
-  session: MaterializationSession,
-  items: Iterable<MaterializationPlanItem>,
-  signal?: AbortSignal
-): Promise<void> {
-  for await (const chunk of planStream(items, context.batching)) {
-    throwIfAborted(signal)
-    await storage.applyChunk({ session, chunk })
-  }
-}
-
 export async function stageWorkBounded(
   context: BatchingContext,
   storage: OntologyMaterializationStorage,
@@ -48,108 +28,34 @@ export async function stageWorkBounded(
   }
 }
 
-export async function drainStagedWork(
+/**
+ * Hands the staged plan to storage, which applies it and writes its events. Vector profiles see
+ * the object changes first, while the objects still hold their previous values.
+ */
+export async function applyStagedWork(
   context: Pick<MaterializerContext, "batching" | "projectId" | "ontology" | "clock">,
   transactionStorage: Storage,
   session: MaterializationSession,
   signal?: AbortSignal,
   projection = false
-): Promise<void> {
+): Promise<number> {
   const ontologyStorage = transactionStorage.ontology
   const storage = ontologyStorage.materializations
-  let phase: number | null = null
-  let pending: MaterializationPlanItem[] = []
-  const flush = async () => {
-    if (pending.length === 0) return
-    await scheduleVectorChanges(context, transactionStorage, pending, session, projection)
-    await invalidateVectorChanges(context.projectId, ontologyStorage.vectors, pending, session)
-    await applyItems(context, storage, session, pending, signal)
-    pending = []
-  }
-  for await (const page of storage.streamWork({
-    session,
-    order: "apply",
-    pageRows: context.batching.planChunkRows,
-  })) {
-    for (const record of page.records) {
-      if (record.kind !== "plan") {
-        throw new MaterializationValidationError("Provider returned non-plan apply work.")
-      }
-      if (phase !== null && record.applyPhase !== phase) await flush()
-      phase = record.applyPhase
-      pending.push(record.item)
-      if (pending.length >= context.batching.planChunkRows) await flush()
-    }
-  }
-  await flush()
-}
-
-export async function validateStagedCardinality(
-  context: BatchingContext,
-  storage: OntologyMaterializationStorage,
-  session: MaterializationSession,
-  signal?: AbortSignal
-): Promise<void> {
-  let currentView: "candidate" | "effective" | null = null
-  let currentScope: string | null = null
-  let occupant: string | null = null
-  for await (const page of storage.streamWork({
-    session,
-    order: "cardinality",
-    pageRows: context.batching.statePageRows,
-  })) {
-    throwIfAborted(signal)
-    for (const record of page.records) {
-      if (record.kind !== "cardinality") {
-        throw new MaterializationValidationError(
-          "Provider returned non-cardinality cardinality work."
-        )
-      }
-      if (record.view !== currentView || record.scopeSortKey !== currentScope) {
-        currentView = record.view
-        currentScope = record.scopeSortKey
-        occupant = null
-      }
-      if (!record.occupied) continue
-      if (occupant && occupant !== record.linkSortKey) {
-        throw new MaterializationValidationError(
-          record.view === "candidate"
-            ? `Projection source scope '${record.ref.source.objectTypeId}.${record.ref.linkId}' has cardinality one.`
-            : `Link scope '${record.ref.source.objectTypeId}.${record.ref.linkId}' has cardinality one.`
-        )
-      }
-      occupant = record.linkSortKey
-    }
-  }
-}
-
-export async function drainStagedEvents(
-  context: EventContext,
-  storage: OntologyMaterializationStorage,
-  session: MaterializationSession,
-  identity: TimedCommitIdentity,
-  signal?: AbortSignal
-): Promise<number> {
-  let ordinal = 0
-  for await (const page of storage.streamWork({
-    session,
-    order: "event",
-    pageRows: context.batching.planChunkRows,
-  })) {
-    const items: MaterializationPlanItem[] = []
-    for (const record of page.records) {
-      if (record.kind !== "event") {
-        throw new MaterializationValidationError("Provider returned non-event event work.")
-      }
-      const event = sequenceMaterializationEvent(
-        context.projectId,
-        identity.commitId,
-        ordinal++,
-        record.draft
-      )
-      items.push(outboxItem(event, identity.committedAt))
+  if (ontologyStorage.vectorIndexing || ontologyStorage.vectors) {
+    for await (const page of storage.streamVectorChanges({
+      session,
+      objectTypeIds: context.ontology
+        .listObjectTypes()
+        .filter((type) => Object.keys(type.search?.vectors ?? {}).length > 0)
+        .map((type) => type.id),
+      pageRows: context.batching.planChunkRows,
+    })) {
+      throwIfAborted(signal)
+      await scheduleVectorChanges(context, transactionStorage, page.items, session, projection)
+      await invalidateVectorChanges(context.projectId, ontologyStorage.vectors, page.items, session)
     }
-    await applyItems(context, storage, session, items, signal)
   }
-  return ordinal
+  throwIfAborted(signal)
+  const { eventCount } = await storage.apply({ session })
+  return eventCount
 }

package/src/materializer/execution/work-records.ts

@@ -6,14 +6,11 @@ import type {
   MaterializationPlanWorkItem,
   MaterializationPlanWorkRecord,
   MaterializationWorkEntityKind,
-  OntologyMaterializationEvent,
-  OntologyOutboxWrite,
 } from "../../storage/ontology"
 import { materializationApplyPhase } from "../../storage/ontology/materializations"
 import type { WorkingLinkEdge, WorkingLinkSlot, WorkingObject } from "../edits/working-state"
 import type { OrderedMaterializationEventDraft } from "../effective/build-events"
 import type { TimedCommitIdentity } from "../shared/identity"
-import type { MaterializationPlanItem } from "./plan-stream"
 
 export function appendObjectOverrideWork(
   items: MaterializationPlanWorkItem[],
@@ -158,18 +155,6 @@ export function appendEffectiveLinkWork(
   }
 }
 
-export function outboxItem(
-  event: OntologyMaterializationEvent,
-  committedAt: string
-): MaterializationPlanItem {
-  const value: OntologyOutboxWrite = {
-    envelope: event,
-    availableAt: committedAt,
-    createdAt: committedAt,
-  }
-  return { kind: "outbox", value }
-}
-
 export function classificationWork(
   entityKind: MaterializationWorkEntityKind,
   identityKey: string,