@sixb/core 0.1.15 → 0.1.17

package/src/materialization/event-envelopes.ts

@@ -0,0 +1,168 @@
+import { compareStrings, type JsonValue } from "../json"
+import type { ExecutionRecord } from "../storage/executions/types"
+import type {
+  EventExecutor,
+  OntologyMaterializationEvent,
+  OntologyMaterializationEventAttribution,
+  OntologyMaterializationEventCommit,
+  OntologyMaterializationEventDraft,
+} from "./events"
+import type { OntologyMaterializationPropertyChangeMap } from "./model"
+
+/** Where an event sits in its commit: its id and its ordinal. */
+export interface OntologyMaterializationEventSequence {
+  readonly id: string
+  readonly commitOrdinal: number
+}
+
+/**
+ * The event consumers receive, rebuilt from the draft storage keeps, the commit it belongs to and
+ * the execution that commit ran under. It is the exact event core would have built in one piece,
+ * so storing drafts changes nothing a consumer can observe.
+ */
+export function materializationEvent(
+  draft: OntologyMaterializationEventDraft,
+  sequence: OntologyMaterializationEventSequence,
+  commit: OntologyMaterializationEventCommit,
+  attribution: OntologyMaterializationEventAttribution
+): OntologyMaterializationEvent {
+  const context = {
+    id: sequence.id,
+    schemaVersion: 1 as const,
+    projectId: commit.projectId,
+    occurredAt: commit.committedAt,
+    correlationId: attribution.correlationId,
+    origin: structuredClone(commit.origin),
+    ...(attribution.requestedBy === undefined
+      ? {}
+      : { requestedBy: { ...attribution.requestedBy } }),
+    executor: structuredClone(attribution.executor),
+    commitId: commit.id,
+    commitOrdinal: sequence.commitOrdinal,
+  }
+  switch (draft.type) {
+    case "object.created":
+      return {
+        ...context,
+        type: draft.type,
+        topic: "objects",
+        partitionKey: `${draft.payload.objectTypeId}:${draft.payload.primaryId}`,
+        payload: {
+          ...draft.payload,
+          propertyChanges: createdPropertyChanges(draft.payload.properties),
+        },
+      }
+    case "object.updated":
+      return {
+        ...context,
+        type: draft.type,
+        topic: "objects",
+        partitionKey: `${draft.payload.objectTypeId}:${draft.payload.primaryId}`,
+        payload: draft.payload,
+      }
+    case "object.deleted":
+      return {
+        ...context,
+        type: draft.type,
+        topic: "objects",
+        partitionKey: `${draft.payload.objectTypeId}:${draft.payload.primaryId}`,
+        payload: draft.payload,
+      }
+    case "link.created":
+      return {
+        ...context,
+        type: draft.type,
+        topic: "links",
+        partitionKey: `${draft.payload.sourceTypeId}:${draft.payload.sourceId}:${draft.payload.linkId}`,
+        payload: {
+          ...draft.payload,
+          propertyChanges: createdPropertyChanges(draft.payload.properties ?? {}),
+        },
+      }
+    case "link.updated":
+      return {
+        ...context,
+        type: draft.type,
+        topic: "links",
+        partitionKey: `${draft.payload.sourceTypeId}:${draft.payload.sourceId}:${draft.payload.linkId}`,
+        payload: draft.payload,
+      }
+    case "link.deleted":
+      return {
+        ...context,
+        type: draft.type,
+        topic: "links",
+        partitionKey: `${draft.payload.sourceTypeId}:${draft.payload.sourceId}:${draft.payload.linkId}`,
+        payload: draft.payload,
+      }
+    case "telemetry.appended":
+      return {
+        ...context,
+        type: draft.type,
+        topic: "telemetry",
+        partitionKey: `${draft.payload.objectTypeId}:${draft.payload.objectId}:${draft.payload.propertyId}`,
+        payload: draft.payload,
+      }
+  }
+}
+
+/**
+ * What the events of a commit say about who asked for it and what wrote it, read from the
+ * execution it ran under. A primitive's id lives in the execution's authority, the only place the
+ * durable record keeps it.
+ */
+export function eventAttribution(
+  execution: Pick<
+    ExecutionRecord,
+    "id" | "correlationId" | "requestedBy" | "executor" | "authorizationRef"
+  >
+): OntologyMaterializationEventAttribution {
+  const attribution = {
+    correlationId: execution.correlationId,
+    executor: eventExecutor(execution),
+  }
+  if (execution.requestedBy === undefined) return attribution
+  const { type, id } = execution.requestedBy
+  return { ...attribution, requestedBy: { type, id } }
+}
+
+function eventExecutor(
+  execution: Pick<ExecutionRecord, "id" | "executor" | "authorizationRef">
+): EventExecutor {
+  const { executor, authorizationRef } = execution
+  switch (executor.type) {
+    case "request":
+      return { type: "request", requestId: executor.requestId }
+    case "agent":
+      return { type: "agent", runId: executor.runId }
+    case "kernel":
+      return { type: "kernel", operation: structuredClone(executor.operation) }
+    case "primitive":
+      if (
+        authorizationRef.type !== "trustedPrimitive" ||
+        authorizationRef.primitive.kind !== executor.kind ||
+        authorizationRef.primitive.runId !== executor.runId
+      ) {
+        throw new Error(
+          `[Sixb] Execution '${execution.id}' runs a primitive without that primitive's authority.`
+        )
+      }
+      return {
+        type: "primitive",
+        kind: executor.kind,
+        id: authorizationRef.primitive.id,
+        runId: executor.runId,
+      }
+  }
+}
+
+/** A creation changes every property it sets, from nothing to its value, in key order. */
+function createdPropertyChanges(
+  properties: Readonly<Record<string, JsonValue>>
+): OntologyMaterializationPropertyChangeMap {
+  const changes: Record<string, { readonly operation: "created"; readonly after: JsonValue }> = {}
+  for (const id of Object.keys(properties).sort(compareStrings)) {
+    changes[id] = { operation: "created", after: properties[id] as JsonValue }
+  }
+  return changes
+}

package/src/materialization/events.ts

@@ -110,9 +110,45 @@ export type OntologyMaterializationEvent =
   | OntologyLinkMaterializationEvent
   | OntologyTelemetryMaterializationEvent
 
-type WithoutEventSequence<T> = T extends OntologyMaterializationEvent
-  ? Omit<T, "id" | "commitOrdinal">
+type OntologyMaterializationEventKind = OntologyMaterializationEvent["type"]
+
+type EventPayload<K extends OntologyMaterializationEventKind> =
+  OntologyMaterializationEvent extends infer E
+    ? E extends { readonly type: infer T; readonly payload: infer P }
+      ? K extends T
+        ? P
+        : never
+      : never
+    : never
+
+type EventDraftOf<K> = K extends OntologyMaterializationEventKind
+  ? {
+      readonly type: K
+      readonly payload: K extends "object.created" | "link.created"
+        ? Readonly<Omit<EventPayload<K>, "propertyChanges">>
+        : EventPayload<K>
+    }
   : never
 
-/** Complete core-authored event fact before contiguous commit sequencing is assigned. */
-export type OntologyMaterializationEventDraft = WithoutEventSequence<OntologyMaterializationEvent>
+/**
+ * An event as core stages it and storage keeps it: what changed, and nothing it can be rebuilt
+ * from. `materializationEvent` adds back the context its commit and execution share with every
+ * other event of that commit, the partition key and topic its type and payload fix, and the
+ * property changes of a creation, which only restate its properties.
+ */
+export type OntologyMaterializationEventDraft = EventDraftOf<OntologyMaterializationEventKind>
+
+/** The commit fields every event of that commit carries. */
+export interface OntologyMaterializationEventCommit {
+  readonly projectId: string
+  readonly id: string
+  readonly committedAt: string
+  readonly origin: OntologyMaterializationOrigin
+}
+
+/** Who asked for a commit and what wrote it, as its events carry them: read from its execution. */
+export interface OntologyMaterializationEventAttribution {
+  readonly correlationId: string
+  readonly requestedBy?: AuthorizablePrincipal
+  readonly executor: EventExecutor
+}

package/src/materialization/index.ts

@@ -17,7 +17,15 @@ export {
   MaterializationObjectNotFoundError,
   MaterializationValidationError,
 } from "./errors"
-export type { OntologyMaterializationEvent, OntologyMaterializationEventDraft } from "./events"
+export type { OntologyMaterializationEventSequence } from "./event-envelopes"
+export { eventAttribution, materializationEvent } from "./event-envelopes"
+export type {
+  OntologyMaterializationEvent,
+  OntologyMaterializationEventAttribution,
+  OntologyMaterializationEventCommit,
+  OntologyMaterializationEventDraft,
+} from "./events"
+export { createEventId } from "./identity"
 export type {
   BaseCommitResult,
   EditCommitResult,

package/src/materializer/README.md

@@ -32,26 +32,30 @@ normalize + validate intent
   -> resolve effective state
   -> diff against committed state
   -> stage deterministic work
-  -> apply work in dependency order
-  -> write outbox events with stable commit ordinals
+  -> storage applies the work in dependency order and writes its outbox events
   -> finalize ontology commit atomically
   -> advance a run resume checkpoint when required
 ```
 
-`staged work` is a transaction-local execution plan. It is not a source materialization and is
-never queried as Ontology state.
+Staged work is an execution plan, never queried as Ontology state. Edits and telemetry stage it on
+their session; a projection replacement stages it on the candidate's durable plan (below). Either
+way the Materializer plans and storage applies: it validates cardinality, writes every item in
+phase order and sequences the events, so planned rows are not read back to be sent again. Vector
+profiles see the changed objects first, while those still hold their previous values.
 
 ## Durable ownership
 
 ```text
 run storage       -> execution ownership, lifecycle, progress, and resume checkpoints
-ontology storage  -> sources, effective state, telemetry, commits, and outbox
+ontology storage  -> sources, replacement plans, effective state, telemetry, commits, and outbox
 Materializer      -> semantic planning and cross-store transaction orchestration
 ```
 
 Every ontology commit references its immutable execution through `executionId`. Storage providers enforce that reference. `origin` remains the semantic idempotency and run-correlation key: it says which runtime request, Action, projection replacement, or telemetry batch the commit represents; the execution says under which durable authority it ran. Neither field duplicates the other.
 
-A commit and each of its events also carry a copy of two execution facts, so consumers can answer "who changed this" without reading the execution: `requestedBy`, the principal on whose behalf the execution chain runs (absent for automatic, anonymous, or auth-disabled work), and `executor`, the request, primitive run, Agent run, or kernel operation that wrote. Both are derived from the bound scope in `execution/scope.ts`, which is already proven identical to the durable record, so they are a pure function of `executionId` and never caller input. Authority is deliberately not copied: it stays on the execution's `authorizationRef`. Header validation requires an Action or projection origin to name the exact primitive run in `executor`, and provider work requires every event to carry its commit's attribution.
+Every event of a commit also names who asked for it and what wrote it, so consumers can answer "who changed this" without reading the execution: `requestedBy`, the principal on whose behalf the execution chain runs (absent for automatic, anonymous, or auth-disabled work), `executor`, the request, primitive run, Agent run, or kernel operation that wrote, and `correlationId`. The commit does not copy them: they live once, on its immutable execution, and the outbox reads them there. Authority stays on the execution's `authorizationRef`; only a primitive's id is read from it. Before writing, the Materializer requires an Action or projection origin to name the exact primitive run of its execution (`assertTrustedPrimitiveMutationExecution`).
+
+Storage keeps an event once, as its draft — its type and payload, without the property changes a creation only restates. When the outbox claims a row, it rebuilds the full event from the draft, its commit and that commit's execution (`materializationEvent`). An event therefore cannot disagree with its commit or its execution, and a large publication writes their context once instead of once per event.
 
 Run records do not duplicate ontology commit ids or semantic commit history. Replacement projections need no resume checkpoint; projection telemetry stores only its next batch/row checkpoint on the run.
 
@@ -65,14 +69,54 @@ listing is reserved for history and observability.
 dataset entries
   -> staging source candidate
   -> ready source candidate
-  -> compare candidate + overrides + telemetry with effective state
-  -> plan and apply changes
-  -> atomically activate candidate and finalize commit
+  -> plan the candidate, outside any transaction, page by page
+  -> commit transaction: refresh the plan, apply it, activate the candidate, finalize the commit
 ```
 
 Source ingress is sealed before commit time is assigned. Activation and ontology commit
 finalization are atomic; the projection run stores no separate commit pointer.
 
+The plan is built outside the commit transaction, which refreshes it, then applies it. The plan is
+durable and belongs to the candidate:
+
+```text
+open      -> the candidate's identities: its entities and those of the roots it replaces
+plan      -> each page reads its state in one snapshot, records the revision of what it read,
+             and stages the resulting work; links are planned after objects, then extended to
+             the links of newly planned objects whose existence flips and to the members of
+             newly changed scopes
+refresh   -> in the commit transaction: a planned identity whose inputs a commit touched since
+             is planned again, with the links that commit brought into the plan; a few are
+             planned right there, more are given back to a round outside the transaction
+apply     -> only a plan that refresh found fresh; the commit reports its planned counts
+```
+
+An identity's revision covers everything its plan read: an object's effective row, override and
+latest telemetry; a link's effective row, edge and slot overrides, live source, and the existence
+of both endpoints (the whole revision of an endpoint the plan decides, whose planned existence
+follows it).
+
+Every commit records the entities whose plan inputs it changed, its touches: the objects whose row,
+override or telemetry it wrote, the links whose row or edge override it wrote or whose live source
+assertion its activation moved, and the scopes whose slot override it wrote. When no commit of the
+project landed since the plan last proved fresh, refresh checks nothing; otherwise it reads again
+only the revisions of identities those commits touched (a link also through its scope and both
+endpoints), so its cost follows the commits, not the plan. A plan older than the touches storage
+still keeps (PostgreSQL purge can race a plan's opening) is checked whole.
+
+A refresh that finds no more stale identities than `transactionReplanRows` (one state page) plans
+them inside the commit transaction, where nothing changes them again, and commits. More are given
+back to another round outside it. A commit that keeps finding many stale identities gives the
+delivery back after three rounds; the next delivery resumes the same plan, as does any redelivery
+of the run, unless the active source the plan replaces moved meanwhile: the plan then starts over.
+Storage keeps a plan while its candidate is ready and maintenance deletes it afterwards, before the
+candidate itself, along with the touches no ready plan still needs.
+
+The commit time is fixed when the plan opens: every planned row carries it as `updatedAt`, every
+planned event as `occurredAt`. A resumed plan keeps it, so a publication that commits later than
+it opened, after retries, stamps its writes with that earlier time, even over an edit committed in
+between. Commit order is the order of the ontology ledger and the outbox, never these timestamps.
+
 A candidate belongs to its run, not to the delivery that staged it:
 
 ```text
@@ -141,7 +185,8 @@ finish still commit atomically.
 - `effective/`: pure resolution, validation, diff, and event construction.
 - `execution/`: storage-neutral plan execution, replay, retry, and run correlation.
 - `shared/`: normalization, identity, batching, and chunking primitives.
-- `storage/ontology/`: durable ontology commit, source, materialization, and outbox contracts.
+- `storage/ontology/`: durable ontology commit, source, replacement plan, materialization, and
+  outbox contracts.
 
 Keep use-case entrypoints explicit. Share mechanics only after they have identical semantics; do
 not hide projection candidate lifecycle, edit continuation, or telemetry batching behind a generic

package/src/materializer/edits/commit.ts

@@ -21,7 +21,7 @@ import {
   type MaterializerExecution,
   prepareMaterializerExecution,
 } from "../execution/scope"
-import { drainStagedEvents, drainStagedWork } from "../execution/work-executor"
+import { applyStagedWork } from "../execution/work-executor"
 import type { MaterializerCommand } from "../materializer"
 import {
   createActionIdempotencyKey,
@@ -187,16 +187,10 @@ async function executeEditTransaction(
     storage.ontology.materializations,
     session,
     workingState,
-    editPlanContext(command)
-  )
-  await drainStagedWork(context, storage, session)
-  await commitVectorWrites(context, storage, command.input, command.identity.commitId, session)
-  const eventCount = await drainStagedEvents(
-    context,
-    storage.ontology.materializations,
-    session,
     command.identity
   )
+  const eventCount = await applyStagedWork(context, storage, session)
+  await commitVectorWrites(context, storage, command.input, command.identity.commitId, session)
 
   const result: EditCommitResult = {
     kind: "edit",
@@ -237,7 +231,6 @@ function buildEditCommit(
     requestHash: command.identity.requestHash,
     executionId: command.execution.executionId,
     origin: command.origin,
-    ...command.execution.attribution,
     ontologyRevision: context.projectionRegistry.ontologyRevision,
     intent: {
       kind: "edit",
@@ -362,15 +355,6 @@ function isRecoverableEditValidation(
   return error instanceof MaterializationValidationError
 }
 
-function editPlanContext(command: PreparedEditCommit) {
-  return {
-    identity: command.identity,
-    origin: command.origin,
-    correlationId: command.execution.correlationId,
-    attribution: command.execution.attribution,
-  }
-}
-
 async function finalizeEditMaterialization(
   storage: MaterializerStorage,
   session: MaterializationSession,

package/src/materializer/edits/plan.ts

@@ -1,8 +1,4 @@
-import type {
-  EffectiveLinkChange,
-  EffectiveObjectChange,
-  OntologyMaterializationOrigin,
-} from "../../materialization/model"
+import type { EffectiveLinkChange, EffectiveObjectChange } from "../../materialization/model"
 import {
   compareLinkRefs,
   compareObjectRefs,
@@ -26,7 +22,6 @@ import {
 } from "../effective/build-events"
 import { diffEffectiveLink, diffEffectiveLinkSlot, diffEffectiveObject } from "../effective/diff"
 import { validateEffectiveObject } from "../effective/validate"
-import type { MaterializerAttribution } from "../execution/scope"
 import { stageWorkBounded } from "../execution/work-executor"
 import {
   appendEffectiveLinkWork,
@@ -55,43 +50,29 @@ export interface EditPlanChanges {
   readonly links: readonly EffectiveLinkChange[]
 }
 
-interface EditPlanContext {
-  readonly identity: TimedCommitIdentity
-  readonly origin: OntologyMaterializationOrigin
-  readonly correlationId: string
-  readonly attribution: MaterializerAttribution
-}
-
 export async function stageEditPlan(
   context: MaterializerContext,
   storage: OntologyMaterializationStorage,
   session: MaterializationSession,
   state: EditWorkingState,
-  planContext: EditPlanContext
+  identity: TimedCommitIdentity
 ): Promise<EditPlanChanges> {
   validateWorkingCardinality(state.links.slots)
 
   const objectChanges: EffectiveObjectChange[] = []
   for (const working of sortedObjects(state.objects)) {
-    const change = await stageObject(context, storage, session, working, planContext)
+    const change = await stageObject(context, storage, session, working, identity)
     if (change) objectChanges.push(change)
   }
 
   const linkChanges: EffectiveLinkChange[] = []
   for (const working of sortedLinkEdges(state.links.edges)) {
-    const change = await stageLinkEdge(
-      context,
-      storage,
-      session,
-      state.objects,
-      working,
-      planContext
-    )
+    const change = await stageLinkEdge(context, storage, session, state.objects, working, identity)
     if (change) linkChanges.push(change)
   }
   for (const working of sortedLinkSlots(state.links.slots)) {
     linkChanges.push(
-      ...(await stageLinkSlot(context, storage, session, state.objects, working, planContext))
+      ...(await stageLinkSlot(context, storage, session, state.objects, working, identity))
     )
   }
 
@@ -109,16 +90,16 @@ async function stageLinkSlot(
   session: MaterializationSession,
   objects: EditWorkingState["objects"],
   working: WorkingLinkSlot,
-  planContext: EditPlanContext
+  identity: TimedCommitIdentity
 ): Promise<EffectiveLinkChange[]> {
   const overrideItems: MaterializationPlanWorkItem[] = []
-  appendLinkSlotOverrideWork(overrideItems, working, planContext.identity)
+  appendLinkSlotOverrideWork(overrideItems, working, identity)
   const resolved = resolveLinkSlot(context.ontology, working, objects)
   const changes = diffEffectiveLinkSlot({
     before: working.before,
     resolved,
-    commitId: planContext.identity.commitId,
-    committedAt: planContext.identity.committedAt,
+    commitId: identity.commitId,
+    committedAt: identity.committedAt,
   })
 
   const representative =
@@ -145,7 +126,7 @@ async function stageLinkSlot(
     appendEffectiveLinkWork(effectiveItems, change)
     const sortKey = linkRefSortKey(change.ref)
     work.push(...effectiveItems.map((item) => planWork(item, sortKey)))
-    work.push(eventWork(buildLinkEvent(context, planContext, change)))
+    work.push(eventWork(buildLinkMaterializationEventDraft(change)))
   }
   await stageWorkBounded(context, storage, session, work)
   return changes
@@ -156,18 +137,18 @@ async function stageObject(
   storage: OntologyMaterializationStorage,
   session: MaterializationSession,
   working: WorkingObject,
-  planContext: EditPlanContext
+  identity: TimedCommitIdentity
 ): Promise<EffectiveObjectChange | null> {
   const items: MaterializationPlanWorkItem[] = []
-  appendObjectOverrideWork(items, working, planContext.identity)
+  appendObjectOverrideWork(items, working, identity)
 
   const resolved = resolveObject(context.ontology, working)
   if (resolved) validateEffectiveObject(context.ontology, resolved.ref, resolved.properties)
   const change = diffEffectiveObject({
     before: working.before,
     resolved,
-    commitId: planContext.identity.commitId,
-    committedAt: planContext.identity.committedAt,
+    commitId: identity.commitId,
+    committedAt: identity.committedAt,
   })
   if (change) appendEffectiveObjectWork(items, change)
 
@@ -177,7 +158,7 @@ async function stageObject(
     ...items.map((item) => planWork(item, sortKey)),
   ]
   if (change) {
-    work.push(eventWork(buildObjectEvent(context, planContext, change)))
+    work.push(eventWork(buildObjectMaterializationEventDraft(change)))
   }
   await stageWorkBounded(context, storage, session, work)
   return change
@@ -189,17 +170,17 @@ async function stageLinkEdge(
   session: MaterializationSession,
   objects: EditWorkingState["objects"],
   working: WorkingLinkEdge,
-  planContext: EditPlanContext
+  identity: TimedCommitIdentity
 ): Promise<EffectiveLinkChange | null> {
   const items: MaterializationPlanWorkItem[] = []
-  appendLinkEdgeOverrideWork(items, working, planContext.identity)
+  appendLinkEdgeOverrideWork(items, working, identity)
 
   const resolved = resolveLinkEdge(context.ontology, working, objects)
   const change = diffEffectiveLink({
     before: working.before,
     resolved,
-    commitId: planContext.identity.commitId,
-    committedAt: planContext.identity.committedAt,
+    commitId: identity.commitId,
+    committedAt: identity.committedAt,
   })
   if (change) appendEffectiveLinkWork(items, change)
 
@@ -209,44 +190,12 @@ async function stageLinkEdge(
     ...items.map((item) => planWork(item, sortKey)),
   ]
   if (change) {
-    work.push(eventWork(buildLinkEvent(context, planContext, change)))
+    work.push(eventWork(buildLinkMaterializationEventDraft(change)))
   }
   await stageWorkBounded(context, storage, session, work)
   return change
 }
 
-function buildObjectEvent(
-  context: Pick<MaterializerContext, "projectId">,
-  planContext: EditPlanContext,
-  change: EffectiveObjectChange
-) {
-  const eventContext = materializationEventContext(context, planContext)
-  return buildObjectMaterializationEventDraft({ ...eventContext, change })
-}
-
-function buildLinkEvent(
-  context: Pick<MaterializerContext, "projectId">,
-  planContext: EditPlanContext,
-  change: EffectiveLinkChange
-) {
-  const eventContext = materializationEventContext(context, planContext)
-  return buildLinkMaterializationEventDraft({ ...eventContext, change })
-}
-
-function materializationEventContext(
-  context: Pick<MaterializerContext, "projectId">,
-  planContext: EditPlanContext
-) {
-  return {
-    projectId: context.projectId,
-    commitId: planContext.identity.commitId,
-    committedAt: planContext.identity.committedAt,
-    origin: planContext.origin,
-    correlationId: planContext.correlationId,
-    attribution: planContext.attribution,
-  }
-}
-
 function sortedObjects(objects: EditWorkingState["objects"]): WorkingObject[] {
   return [...objects.values()].sort((left, right) => compareObjectRefs(left.ref, right.ref))
 }