@sixb/core 0.1.16 → 0.1.18

package/src/materializer/execution/work-executor.ts

@@ -1,4 +1,3 @@
-import { MaterializationValidationError } from "../../materialization/errors"
 import { utf8JsonByteLength } from "../../materialization/refs"
 import type { Storage } from "../../storage"
 import type {
@@ -7,32 +6,13 @@ import type {
   OntologyMaterializationStorage,
 } from "../../storage/ontology"
 import type { MaterializerContext } from "../context"
-import { sequenceMaterializationEvent } from "../effective/build-events"
 import { scheduleVectorChanges } from "../effective/vector-indexing"
 import { invalidateVectorChanges } from "../effective/vectors"
 import { throwIfAborted } from "../shared/abort"
 import { chunkBySize } from "../shared/chunking"
-import type { TimedCommitIdentity } from "../shared/identity"
-import { type MaterializationPlanItem, planStream } from "./plan-stream"
-import { outboxItem } from "./work-records"
 
 type BatchingContext = Pick<MaterializerContext, "batching">
 
-type EventContext = Pick<MaterializerContext, "batching" | "projectId">
-
-export async function applyItems(
-  context: BatchingContext,
-  storage: OntologyMaterializationStorage,
-  session: MaterializationSession,
-  items: Iterable<MaterializationPlanItem>,
-  signal?: AbortSignal
-): Promise<void> {
-  for await (const chunk of planStream(items, context.batching)) {
-    throwIfAborted(signal)
-    await storage.applyChunk({ session, chunk })
-  }
-}
-
 export async function stageWorkBounded(
   context: BatchingContext,
   storage: OntologyMaterializationStorage,
@@ -48,108 +28,34 @@ export async function stageWorkBounded(
   }
 }
 
-export async function drainStagedWork(
+/**
+ * Hands the staged plan to storage, which applies it and writes its events. Vector profiles see
+ * the object changes first, while the objects still hold their previous values.
+ */
+export async function applyStagedWork(
   context: Pick<MaterializerContext, "batching" | "projectId" | "ontology" | "clock">,
   transactionStorage: Storage,
   session: MaterializationSession,
   signal?: AbortSignal,
   projection = false
-): Promise<void> {
+): Promise<number> {
   const ontologyStorage = transactionStorage.ontology
   const storage = ontologyStorage.materializations
-  let phase: number | null = null
-  let pending: MaterializationPlanItem[] = []
-  const flush = async () => {
-    if (pending.length === 0) return
-    await scheduleVectorChanges(context, transactionStorage, pending, session, projection)
-    await invalidateVectorChanges(context.projectId, ontologyStorage.vectors, pending, session)
-    await applyItems(context, storage, session, pending, signal)
-    pending = []
-  }
-  for await (const page of storage.streamWork({
-    session,
-    order: "apply",
-    pageRows: context.batching.planChunkRows,
-  })) {
-    for (const record of page.records) {
-      if (record.kind !== "plan") {
-        throw new MaterializationValidationError("Provider returned non-plan apply work.")
-      }
-      if (phase !== null && record.applyPhase !== phase) await flush()
-      phase = record.applyPhase
-      pending.push(record.item)
-      if (pending.length >= context.batching.planChunkRows) await flush()
-    }
-  }
-  await flush()
-}
-
-export async function validateStagedCardinality(
-  context: BatchingContext,
-  storage: OntologyMaterializationStorage,
-  session: MaterializationSession,
-  signal?: AbortSignal
-): Promise<void> {
-  let currentView: "candidate" | "effective" | null = null
-  let currentScope: string | null = null
-  let occupant: string | null = null
-  for await (const page of storage.streamWork({
-    session,
-    order: "cardinality",
-    pageRows: context.batching.statePageRows,
-  })) {
-    throwIfAborted(signal)
-    for (const record of page.records) {
-      if (record.kind !== "cardinality") {
-        throw new MaterializationValidationError(
-          "Provider returned non-cardinality cardinality work."
-        )
-      }
-      if (record.view !== currentView || record.scopeSortKey !== currentScope) {
-        currentView = record.view
-        currentScope = record.scopeSortKey
-        occupant = null
-      }
-      if (!record.occupied) continue
-      if (occupant && occupant !== record.linkSortKey) {
-        throw new MaterializationValidationError(
-          record.view === "candidate"
-            ? `Projection source scope '${record.ref.source.objectTypeId}.${record.ref.linkId}' has cardinality one.`
-            : `Link scope '${record.ref.source.objectTypeId}.${record.ref.linkId}' has cardinality one.`
-        )
-      }
-      occupant = record.linkSortKey
-    }
-  }
-}
-
-export async function drainStagedEvents(
-  context: EventContext,
-  storage: OntologyMaterializationStorage,
-  session: MaterializationSession,
-  identity: TimedCommitIdentity,
-  signal?: AbortSignal
-): Promise<number> {
-  let ordinal = 0
-  for await (const page of storage.streamWork({
-    session,
-    order: "event",
-    pageRows: context.batching.planChunkRows,
-  })) {
-    const items: MaterializationPlanItem[] = []
-    for (const record of page.records) {
-      if (record.kind !== "event") {
-        throw new MaterializationValidationError("Provider returned non-event event work.")
-      }
-      const event = sequenceMaterializationEvent(
-        context.projectId,
-        identity.commitId,
-        ordinal++,
-        record.draft
-      )
-      items.push(outboxItem(event, identity.committedAt))
+  if (ontologyStorage.vectorIndexing || ontologyStorage.vectors) {
+    for await (const page of storage.streamVectorChanges({
+      session,
+      objectTypeIds: context.ontology
+        .listObjectTypes()
+        .filter((type) => Object.keys(type.search?.vectors ?? {}).length > 0)
+        .map((type) => type.id),
+      pageRows: context.batching.planChunkRows,
+    })) {
+      throwIfAborted(signal)
+      await scheduleVectorChanges(context, transactionStorage, page.items, session, projection)
+      await invalidateVectorChanges(context.projectId, ontologyStorage.vectors, page.items, session)
     }
-    await applyItems(context, storage, session, items, signal)
   }
-  return ordinal
+  throwIfAborted(signal)
+  const { eventCount } = await storage.apply({ session })
+  return eventCount
 }

package/src/materializer/execution/work-records.ts

@@ -6,14 +6,11 @@ import type {
   MaterializationPlanWorkItem,
   MaterializationPlanWorkRecord,
   MaterializationWorkEntityKind,
-  OntologyMaterializationEvent,
-  OntologyOutboxWrite,
 } from "../../storage/ontology"
 import { materializationApplyPhase } from "../../storage/ontology/materializations"
 import type { WorkingLinkEdge, WorkingLinkSlot, WorkingObject } from "../edits/working-state"
 import type { OrderedMaterializationEventDraft } from "../effective/build-events"
 import type { TimedCommitIdentity } from "../shared/identity"
-import type { MaterializationPlanItem } from "./plan-stream"
 
 export function appendObjectOverrideWork(
   items: MaterializationPlanWorkItem[],
@@ -158,18 +155,6 @@ export function appendEffectiveLinkWork(
   }
 }
 
-export function outboxItem(
-  event: OntologyMaterializationEvent,
-  committedAt: string
-): MaterializationPlanItem {
-  const value: OntologyOutboxWrite = {
-    envelope: event,
-    availableAt: committedAt,
-    createdAt: committedAt,
-  }
-  return { kind: "outbox", value }
-}
-
 export function classificationWork(
   entityKind: MaterializationWorkEntityKind,
   identityKey: string,

package/src/materializer/projections/replace.ts

@@ -23,6 +23,8 @@ import type {
   OntologyMaterializationStorage,
   OntologySourceRecord,
   OntologyStorage,
+  ReplacementPlanRef,
+  ReplacementPlanStatus,
 } from "../../storage/ontology"
 import type { MaterializerContext, MaterializerStorage } from "../context"
 import { replayCommitRecord, withSerializationRetry } from "../execution/commit-lifecycle"
@@ -34,7 +36,7 @@ import {
   type MaterializerExecution,
   prepareMaterializerExecution,
 } from "../execution/scope"
-import { drainStagedEvents, drainStagedWork } from "../execution/work-executor"
+import { applyStagedWork } from "../execution/work-executor"
 import type { MaterializerCommand } from "../materializer"
 import { throwIfAborted } from "../shared/abort"
 import {
@@ -51,7 +53,7 @@ import {
 } from "../shared/normalize"
 import { createProjectionRunMaterializationIdentity } from "../shared/projection-run"
 import { createProjectionEntryValidator } from "./entry-validator"
-import { planProjectionReplacement } from "./replacement-plan"
+import { type ProjectionReplacementPlanInput, planProjectionReplacement } from "./replacement-plan"
 import {
   type StagedProjectionMaterialization,
   stageProjectionMaterialization,
@@ -324,25 +326,68 @@ async function stageCandidateEntries(
   return stageProjectionMaterialization(context, { ...input, signal })
 }
 
+/**
+ * Rounds of planning a commit tolerates before it gives the delivery back: each round plans again
+ * only what changed under the previous one, and the next delivery resumes the same plan. A commit
+ * whose refresh finds few stale identities plans them in its own transaction instead, so a round
+ * ends with a commit unless writes keep changing many of the plan's inputs.
+ */
+const MAX_PLAN_ROUNDS = 3
+
 async function commitProjectionCandidate(
   context: MaterializerContext,
   command: PreparedProjectionReplacement,
   ready: ReadyProjectionReplacement
 ): Promise<ProjectionCommitResult> {
-  return withSerializationRetry(context, () =>
-    context.storage.transaction(
-      (storage) => executeProjectionTransaction(context, storage, command, ready),
-      { isolation: "serializable" }
+  const plans = context.storage.ontology.replacementPlans
+  const plan: ReplacementPlanRef = {
+    projectId: context.projectId,
+    source: command.source,
+    materializationId: ready.materializationId,
+    execution: command.execution,
+  }
+  // A resumed plan keeps the commit time it was opened with, since its planned rows and events
+  // carry that time; its session begins with the commit at exactly that time.
+  const { committedAt } = await plans.open({
+    ...plan,
+    commit: projectionCommit(context.projectId, command, ready.identity),
+  })
+  const commit = projectionCommit(context.projectId, command, { ...ready.identity, committedAt })
+  for (let round = 1; ; round += 1) {
+    await planProjectionReplacement(context, plans, plan, planInput(command, commit))
+    const attempt = await withSerializationRetry(context, () =>
+      context.storage.transaction(
+        (storage) => executeProjectionTransaction(context, storage, command, ready, commit, plan),
+        { isolation: "serializable" }
+      )
     )
-  )
+    if (attempt.kind === "committed") return attempt.result
+    if (round === MAX_PLAN_ROUNDS) {
+      throw new MaterializationConflictError(
+        "effective-state",
+        `Projection '${command.source.projectionId}' inputs kept changing while it was planned ` +
+          `(${attempt.unplanned} changed during the last of ${MAX_PLAN_ROUNDS} rounds).`
+      )
+    }
+  }
 }
 
+/**
+ * One commit transaction. `stale` commits only the refresh: the identities whose inputs changed
+ * since they were planned are unplanned, for the next round to plan again outside the transaction.
+ */
+type ProjectionCommitAttempt =
+  | { readonly kind: "committed"; readonly result: ProjectionCommitResult }
+  | { readonly kind: "stale"; readonly unplanned: number }
+
 async function executeProjectionTransaction(
   context: MaterializerContext,
   storage: MaterializerStorage,
   command: PreparedProjectionReplacement,
-  ready: ReadyProjectionReplacement
-): Promise<ProjectionCommitResult> {
+  ready: ReadyProjectionReplacement,
+  commit: OntologyCommitWrite,
+  plan: ReplacementPlanRef
+): Promise<ProjectionCommitAttempt> {
   await ensureMaterializerExecution(storage.executions, command.scopeExecution)
   await assertProjectionExecution(storage, context.projectId, command)
   const replay = await replayCommitRecord(
@@ -351,7 +396,7 @@ async function executeProjectionTransaction(
     command.scopeExecution.executionId,
     storage
   )
-  if (replay) return projectionReplayResult(replay, command)
+  if (replay) return { kind: "committed", result: projectionReplayResult(replay, command) }
 
   if (command.base) {
     const active = await storage.ontology.sources.getActive({
@@ -361,10 +406,11 @@ async function executeProjectionTransaction(
     validateProjectionWatermark(active, command.datasetVersion)
     assertDeltaBase(active, command)
   }
-  const origin = projectionOrigin(command)
   throwIfAborted(command.signal)
+  const status = await refreshProjectionPlan(context, storage, command, commit, plan)
+  if (!status.fresh) return { kind: "stale", unplanned: status.unplanned }
   const session = await storage.ontology.materializations.begin({
-    commit: projectionCommit(context.projectId, command, ready.identity, origin),
+    commit,
     expected: {
       sources: [ready.expectedSource],
       objects: [],
@@ -372,33 +418,48 @@ async function executeProjectionTransaction(
       linkScopes: [],
       points: [],
     },
+    plan: { source: command.source, materializationId: ready.materializationId },
   })
-  const counts = await planReadyProjection(
-    context,
-    storage.ontology.materializations,
-    session,
-    command,
-    ready,
-    origin
-  )
-  await drainStagedWork(context, storage, session, command.signal, true)
-  const eventCount = await drainStagedEvents(
-    context,
-    storage.ontology.materializations,
-    session,
-    ready.identity,
-    command.signal
-  )
+  const eventCount = await applyStagedWork(context, storage, session, command.signal, true)
   const result: ProjectionCommitResult = {
     kind: "projection",
-    commitId: ready.identity.commitId,
+    commitId: commit.id,
     created: true,
     eventCount,
-    committedAt: ready.identity.committedAt,
-    counts,
+    committedAt: commit.committedAt,
+    counts: status.counts,
   }
   throwIfAborted(command.signal)
-  return finalizeProjectionMaterialization(storage, session, command, ready, result)
+  return {
+    kind: "committed",
+    result: await finalizeProjectionMaterialization(
+      storage,
+      session,
+      command,
+      ready,
+      commit,
+      result
+    ),
+  }
+}
+
+/**
+ * Refreshes the plan within the commit transaction. When it finds no more stale identities than
+ * the transaction may plan itself, they are planned here: nothing can change them again before the
+ * commit, so the second refresh finds the plan fresh.
+ */
+async function refreshProjectionPlan(
+  context: MaterializerContext,
+  storage: MaterializerStorage,
+  command: PreparedProjectionReplacement,
+  commit: OntologyCommitWrite,
+  plan: ReplacementPlanRef
+): Promise<ReplacementPlanStatus> {
+  const plans = storage.ontology.replacementPlans
+  const status = await plans.refresh(plan)
+  if (status.fresh || status.unplanned > context.batching.transactionReplanRows) return status
+  await planProjectionReplacement(context, plans, plan, planInput(command, commit))
+  return plans.refresh(plan)
 }
 
 async function finalizeProjectionMaterialization(
@@ -406,12 +467,13 @@ async function finalizeProjectionMaterialization(
   session: Parameters<OntologyMaterializationStorage["finalize"]>[0]["session"],
   command: PreparedProjectionReplacement,
   ready: ReadyProjectionReplacement,
+  commit: OntologyCommitWrite,
   result: ProjectionCommitResult
 ): Promise<ProjectionCommitResult> {
   const applied = await storage.ontology.materializations.finalize({
     session,
     finalization: {
-      sourceActivations: [projectionActivation(command, ready)],
+      sourceActivations: [projectionActivation(command, ready, commit)],
       result,
     },
   })
@@ -431,8 +493,7 @@ function projectionOrigin(command: PreparedProjectionReplacement): OntologyMater
 function projectionCommit(
   projectId: string,
   command: PreparedProjectionReplacement,
-  identity: TimedCommitIdentity,
-  origin: OntologyMaterializationOrigin
+  identity: TimedCommitIdentity
 ): OntologyCommitWrite {
   return {
     projectId,
@@ -440,8 +501,7 @@ function projectionCommit(
     idempotencyKey: identity.idempotencyKey,
     requestHash: identity.requestHash,
     executionId: command.scopeExecution.executionId,
-    origin,
-    ...command.scopeExecution.attribution,
+    origin: projectionOrigin(command),
     ontologyRevision: command.runIdentity.ontologyRevision,
     projectionRevision: command.runIdentity.projectionRevision,
     ownershipHash: command.runIdentity.ownershipHash,
@@ -450,35 +510,18 @@ function projectionCommit(
   }
 }
 
-async function planReadyProjection(
-  context: MaterializerContext,
-  storage: OntologyMaterializationStorage,
-  session: Parameters<OntologyMaterializationStorage["finalize"]>[0]["session"],
+function planInput(
   command: PreparedProjectionReplacement,
-  ready: ReadyProjectionReplacement,
-  origin: OntologyMaterializationOrigin
-) {
-  const input = {
-    source: command.source,
-    materializationId: ready.materializationId,
-    projectionKind: command.projectionKind,
-    identity: ready.identity,
-    origin,
-    correlationId: command.scopeExecution.correlationId,
-    attribution: command.scopeExecution.attribution,
-  }
-  if (command.signal === undefined) {
-    return planProjectionReplacement(context, storage, session, input)
-  }
-  return planProjectionReplacement(context, storage, session, {
-    ...input,
-    signal: command.signal,
-  })
+  commit: OntologyCommitWrite
+): ProjectionReplacementPlanInput {
+  const input = { projectionKind: command.projectionKind, commit }
+  return command.signal === undefined ? input : { ...input, signal: command.signal }
 }
 
 function projectionActivation(
   command: PreparedProjectionReplacement,
-  ready: ReadyProjectionReplacement
+  ready: ReadyProjectionReplacement,
+  commit: OntologyCommitWrite
 ) {
   return {
     source: command.source,
@@ -491,8 +534,8 @@ function projectionActivation(
     ownershipHash: command.runIdentity.ownershipHash,
     ontologyRevision: command.runIdentity.ontologyRevision,
     expected: ready.expectedSource,
-    lastCommitId: ready.identity.commitId,
-    updatedAt: ready.identity.committedAt,
+    lastCommitId: commit.id,
+    updatedAt: commit.committedAt,
   }
 }