@sixb/core 0.1.14 → 0.1.15

package/src/materialization/dataset-watermark.ts

@@ -2,16 +2,20 @@ import { MaterializationConflictError } from "./errors"
 import type { PinnedDatasetVersion } from "./model"
 
 export type PinnedDatasetWatermarkConflict =
-  | "dataset-mismatch"
   | "version-metadata-mismatch"
   | "older-version"
   | "ambiguous-version"
 
+/**
+ * Orders versions of one dataset. A projection rebound to a different dataset starts a new
+ * watermark: versions of two datasets have no order, and whether a run may read that dataset is
+ * decided by the registered definition, which rejects runs pinned to any other dataset.
+ */
 export function comparePinnedDatasetWatermarks(
   active: PinnedDatasetVersion,
   next: PinnedDatasetVersion
 ): PinnedDatasetWatermarkConflict | null {
-  if (active.datasetId !== next.datasetId) return "dataset-mismatch"
+  if (active.datasetId !== next.datasetId) return null
   if (active.versionId === next.versionId && active.createdAt !== next.createdAt) {
     return "version-metadata-mismatch"
   }
@@ -40,8 +44,6 @@ function pinnedDatasetWatermarkConflictMessage(
   subject: string
 ): string {
   switch (conflict) {
-    case "dataset-mismatch":
-      return `${subject} dataset does not match the active source dataset.`
     case "version-metadata-mismatch":
       return `${subject} reused an immutable dataset version id with different metadata.`
     case "older-version":

package/src/materialization/model.ts

@@ -217,6 +217,13 @@ export type ProjectionRunTerminalDecision =
       readonly error?: SixbFailure<ProjectionRunFailureCode>
       readonly inputExhausted?: never
     }
+  | {
+      readonly protocol: "replacement"
+      /** A run of the same projection pinned to a later dataset version makes this one stale. */
+      readonly status: "superseded"
+      readonly error?: never
+      readonly inputExhausted?: never
+    }
 
 /**
  * Queue-agnostic terminal decision for one fenced projection execution.

package/src/materializer/README.md

@@ -70,13 +70,33 @@ dataset entries
   -> atomically activate candidate and finalize commit
 ```
 
-Source ingress is sealed before commit time is assigned. A retry reuses the ready candidate and
-the same semantic commit identity. Activation and ontology commit finalization are atomic; the
-projection run stores no separate commit pointer.
+Source ingress is sealed before commit time is assigned. Activation and ontology commit
+finalization are atomic; the projection run stores no separate commit pointer.
 
-Projection runs finish only through `projections.finishRun(...)`. Its serializable transaction reads
-the authoritative ontology commit before replacement success, rejects failure after commit, and
-then applies the fenced run transition.
+A candidate belongs to its run, not to the delivery that staged it:
+
+```text
+delivery fails transiently -> candidate stays with the run
+next delivery              -> adopts it: ready is published as is, staging resumes at its last root
+run ends without success   -> finishRun abandons it in the same transaction
+maintenance                -> deletes abandoned candidates whole, without retention
+```
+
+An adopted candidate staged against another delta base is abandoned and replaced, since its roots
+describe a different change. Staging resumes by reading the entries again from the start, which is
+sound because they are deterministic for the pinned dataset version and definition.
+
+Projection runs finish only through `projections.finishRun(...)`, including a failure found before
+materialization began. Its serializable transaction reads the authoritative ontology commit before
+replacement success, rejects failure after commit, releases the run's candidate, and then applies
+the fenced run transition. Success must match the deployed definition; failure only has to hold
+the run's execution token, so a run pinned to a definition a deploy has since replaced can still
+end.
+
+A replacement run ends `superseded`, without materializing, when a run of the same projection
+pinned to a later dataset version is queued, running, or succeeded. The worker checks this on every
+delivery after its claim; `finishRun` checks it again under its transaction, since a newer run that
+failed in between replaces nothing, and releases the candidate like any other end without success.
 
 ## Managed edits and Actions
 

package/src/materializer/projections/finish-run.ts

@@ -12,7 +12,10 @@ import type { ProjectionDefinition, ResolvedProjection } from "../../projections
 import type { OntologyCommitRecord } from "../../storage/ontology"
 import type { MaterializerContext, MaterializerStorage } from "../context"
 import { withSerializationRetry } from "../execution/commit-lifecycle"
-import { lockProjectionRunForMaterialization } from "../execution/run-correlation"
+import {
+  type LockedProjectionExecution,
+  lockProjectionRunForMaterialization,
+} from "../execution/run-correlation"
 import {
   assertMaterializerRunExecution,
   assertTrustedPrimitiveMutationExecution,
@@ -31,12 +34,21 @@ import { createProjectionRunMaterializationIdentity } from "../shared/projection
 interface PreparedProjectionRunFinish {
   readonly projectId: string
   readonly input: ProjectionRunFinishInput
-  readonly identity: ProjectionMaterializationIdentity
-  readonly resolved: ResolvedProjection<ProjectionDefinition>
+  /**
+   * The deployed definition a success must match. A failure needs none: it only has to hold the
+   * run's execution token, and it must stay possible after a deploy changed or removed the
+   * projection, or the run and its candidate could never end.
+   */
+  readonly deployed: DeployedProjectionIdentity | null
   readonly finishedAt: Date
   readonly execution: MaterializerExecution
 }
 
+interface DeployedProjectionIdentity {
+  readonly identity: ProjectionMaterializationIdentity
+  readonly resolved: ResolvedProjection<ProjectionDefinition>
+}
+
 export async function finishProjectionRun(
   context: MaterializerContext,
   raw: MaterializerCommand<ProjectionRunFinishInput>
@@ -57,28 +69,35 @@ function prepareProjectionRunFinish(
   const source = normalizeProjectionSourceRef(raw.input.source)
   const datasetVersion = normalizePinnedDatasetVersion(raw.input.datasetVersion)
   const projectionExecution = normalizeProjectionExecution(raw.input.execution)
+  const input = { ...raw.input, source, datasetVersion, execution: projectionExecution }
+  return {
+    projectId: context.projectId,
+    input,
+    deployed: input.status === "succeeded" ? resolveDeployedProjection(context, input) : null,
+    finishedAt: new Date(raw.input.finishedAt ?? context.clock()),
+    execution: prepareMaterializerExecution(context.projectId, raw.scope),
+  }
+}
+
+function resolveDeployedProjection(
+  context: Pick<MaterializerContext, "projectionRegistry">,
+  input: ProjectionRunFinishInput
+): DeployedProjectionIdentity {
   const resolved =
-    raw.input.protocol === "replacement"
-      ? context.projectionRegistry.resolveSource(source.projectionId)
-      : context.projectionRegistry.resolveTelemetry(source.projectionId)
-  if (resolved.datasetId !== datasetVersion.datasetId) {
+    input.protocol === "replacement"
+      ? context.projectionRegistry.resolveSource(input.source.projectionId)
+      : context.projectionRegistry.resolveTelemetry(input.source.projectionId)
+  if (resolved.datasetId !== input.datasetVersion.datasetId) {
     throw new MaterializationValidationError(
       `Projection '${resolved.projectionId}' requires dataset '${resolved.datasetId}'.`
     )
   }
   const identity = createProjectionRunMaterializationIdentity({
     resolved,
-    datasetVersion,
+    datasetVersion: input.datasetVersion,
     ontologyRevision: context.projectionRegistry.ontologyRevision,
   })
-  return {
-    projectId: context.projectId,
-    input: { ...raw.input, source, datasetVersion, execution: projectionExecution },
-    identity,
-    resolved,
-    finishedAt: new Date(raw.input.finishedAt ?? context.clock()),
-    execution: prepareMaterializerExecution(context.projectId, raw.scope),
-  }
+  return { identity, resolved }
 }
 
 async function finishProjectionRunTransaction(
@@ -91,36 +110,115 @@ async function finishProjectionRunTransaction(
     id: command.input.source.projectionId,
     runId: command.input.execution.projectionRunId,
   })
-  const { projectionRuns, run } = await lockProjectionRunForMaterialization(storage, {
-    projectId: command.projectId,
-    projectionRunId: command.input.execution.projectionRunId,
-    executionToken: command.input.execution.executionToken,
-    identity: command.identity,
-    resolved: command.resolved,
-  })
+  const { projectionRuns, run } = command.deployed
+    ? await lockProjectionRunForMaterialization(storage, {
+        projectId: command.projectId,
+        projectionRunId: command.input.execution.projectionRunId,
+        executionToken: command.input.execution.executionToken,
+        identity: command.deployed.identity,
+        resolved: command.deployed.resolved,
+      })
+    : await lockProjectionRunAsStored(storage, command)
   assertMaterializerRunExecution(command.execution, run.executionId, `Projection run '${run.id}'`)
 
   if (command.input.protocol === "replacement") {
     await assertReplacementTerminalDecision(storage, command)
+    if (command.input.status === "superseded") {
+      await assertRunSuperseded(projectionRuns, command)
+    }
+    if (command.input.status !== "succeeded") {
+      // The candidate belongs to the run: once the run ends, nothing may adopt or activate it.
+      await storage.ontology.sources.abandon({
+        kind: "run",
+        projectId: command.projectId,
+        source: command.input.source,
+        execution: command.input.execution,
+        abandonedAt: command.finishedAt.toISOString(),
+      })
+    }
   }
 
   await projectionRuns.finish({
     id: command.input.execution.projectionRunId,
     projectId: command.projectId,
     executionToken: command.input.execution.executionToken,
-    identity: command.identity,
+    identity: run.identity,
     ...terminalDecision(command.input),
     finishedAt: command.finishedAt,
   })
 }
 
+/**
+ * Checked again here, under the finish transaction, rather than trusted from the caller's earlier
+ * read: the newer run may have failed since, and a run superseded by a failure would leave the
+ * projection with neither version.
+ */
+async function assertRunSuperseded(
+  projectionRuns: LockedProjectionExecution["projectionRuns"],
+  command: PreparedProjectionRunFinish
+): Promise<void> {
+  const newer = await projectionRuns.findSupersedingRun({
+    projectId: command.projectId,
+    id: command.input.execution.projectionRunId,
+  })
+  if (newer) return
+  throw new MaterializationConflictError(
+    "run-correlation",
+    `Projection run '${command.input.execution.projectionRunId}' cannot be superseded: no run of a later dataset version stands to replace it.`
+  )
+}
+
+/** Locks a run by the identity it was pinned with, for a decision that needs no definition. */
+async function lockProjectionRunAsStored(
+  storage: MaterializerStorage,
+  command: PreparedProjectionRunFinish
+): Promise<LockedProjectionExecution> {
+  const projectionRuns = storage.projectionRuns
+  if (!projectionRuns) {
+    throw new MaterializationValidationError(
+      "Storage transaction does not provide projection run capabilities."
+    )
+  }
+  const { input } = command
+  const stored = await projectionRuns.getById({
+    projectId: command.projectId,
+    id: input.execution.projectionRunId,
+  })
+  if (
+    !stored ||
+    stored.identity.projectionId !== input.source.projectionId ||
+    stored.identity.protocol !== input.protocol ||
+    !datasetVersionsEqual(stored.identity.datasetVersion, input.datasetVersion)
+  ) {
+    throw new MaterializationConflictError(
+      "run-correlation",
+      `Projection run '${input.execution.projectionRunId}' does not match its finish decision.`
+    )
+  }
+  const run = await projectionRuns.lockForMaterialization({
+    id: stored.id,
+    projectId: command.projectId,
+    executionToken: input.execution.executionToken,
+    identity: stored.identity,
+  })
+  return { projectionRuns, run }
+}
+
 function assertValidTerminalDecision(input: ProjectionRunFinishInput): void {
   if (input.protocol !== "replacement" && input.protocol !== "telemetry") {
     throw new MaterializationValidationError("Projection finish protocol is invalid.")
   }
-  if (input.status !== "succeeded" && input.status !== "failed" && input.status !== "cancelled") {
+  if (
+    input.status !== "succeeded" &&
+    input.status !== "failed" &&
+    input.status !== "cancelled" &&
+    input.status !== "superseded"
+  ) {
     throw new MaterializationValidationError("Projection finish status must be terminal.")
   }
+  if (input.status === "superseded" && input.protocol !== "replacement") {
+    throw new MaterializationValidationError("Only a replacement projection run can be superseded.")
+  }
   const inputExhausted = "inputExhausted" in input ? input.inputExhausted : undefined
   if (input.status === "succeeded" && input.protocol === "telemetry") {
     if (inputExhausted === true) return
@@ -136,6 +234,9 @@ function assertValidTerminalDecision(input: ProjectionRunFinishInput): void {
 }
 
 function terminalDecision(input: ProjectionRunFinishInput): ProjectionRunTerminalDecision {
+  if (input.status === "superseded") {
+    return { protocol: "replacement", status: "superseded" }
+  }
   if (input.status !== "succeeded") {
     return {
       protocol: input.protocol,
@@ -182,8 +283,10 @@ function assertReplacementCommitCorrelation(
   commit: OntologyCommitRecord,
   command: PreparedProjectionRunFinish
 ): void {
-  const { input, identity } = command
+  const { input } = command
+  const identity = command.deployed?.identity
   if (
+    !identity ||
     commit.origin.kind !== "projection" ||
     commit.executionId !== command.execution.executionId ||
     commit.intent.kind !== "projection" ||

package/src/materializer/projections/replace.ts

@@ -1,6 +1,5 @@
 import { assertPinnedDatasetWatermark } from "../../materialization/dataset-watermark"
 import {
-  MaterializationCancellationError,
   MaterializationConflictError,
   MaterializationValidationError,
 } from "../../materialization/errors"
@@ -18,6 +17,7 @@ import type {
 } from "../../materialization/model"
 import type { ProjectionRegistry } from "../../projections/registry"
 import type {
+  AdoptedSourceMaterialization,
   OntologyCommitRecord,
   OntologyCommitWrite,
   OntologyMaterializationStorage,
@@ -76,6 +76,8 @@ interface PreparedProjectionReplacement {
 interface ProjectionCandidate {
   readonly materializationId: string
   readonly createdAt: string
+  /** Set when a redelivery adopted the candidate a previous execution of this run staged. */
+  readonly adopted?: AdoptedSourceMaterialization
   readonly expectedSource: {
     readonly source: ProjectionSourceRef
     readonly activeMaterializationId: string | null
@@ -96,14 +98,11 @@ export async function replaceProjection(
   const replay = await admitProjectionExecution(context, command)
   if (replay) return replay
 
+  // A failed attempt leaves its candidate to the run: a redelivery adopts it, and the run's
+  // terminal transition (`finishRun`) abandons it.
   const candidate = await prepareProjectionCandidate(context, command)
-  try {
-    const ready = await stageProjectionCandidate(context, command, candidate)
-    return await commitProjectionCandidate(context, command, ready)
-  } catch (error) {
-    await abandonFailedCandidate(context, command, candidate, error)
-    throw error
-  }
+  const ready = await stageProjectionCandidate(context, command, candidate)
+  return commitProjectionCandidate(context, command, ready)
 }
 
 function prepareProjectionReplacement(
@@ -230,12 +229,11 @@ async function prepareProjectionCandidate(
   context: MaterializerContext,
   command: PreparedProjectionReplacement
 ): Promise<ProjectionCandidate> {
-  await context.storage.ontology.sources.abandon({
-    kind: "reclaim",
+  const adopted = await context.storage.ontology.sources.adopt({
     projectId: context.projectId,
     source: command.source,
     execution: command.execution,
-    abandonedAt: context.clock().toISOString(),
+    adoptedAt: context.clock().toISOString(),
   })
   const active = await context.storage.ontology.sources.getActive({
     projectId: context.projectId,
@@ -243,14 +241,35 @@ async function prepareProjectionCandidate(
   })
   validateProjectionWatermark(active, command.datasetVersion)
   assertDeltaBase(active, command)
+  const expectedSource = {
+    source: command.source,
+    activeMaterializationId: active?.materializationId ?? null,
+    lastCommitId: active?.lastCommitId ?? null,
+  }
+  if (adopted && sameSourceBase(adopted.record.base, command.base)) {
+    return {
+      materializationId: adopted.record.materializationId,
+      createdAt: adopted.record.createdAt,
+      adopted,
+      expectedSource,
+    }
+  }
+  if (adopted) {
+    // Staged against another delta base (the active source moved, or this attempt reads the
+    // whole version): its roots answer a different question, so it cannot be completed.
+    await context.storage.ontology.sources.abandon({
+      kind: "candidate",
+      projectId: context.projectId,
+      source: command.source,
+      materializationId: adopted.record.materializationId,
+      execution: command.execution,
+      abandonedAt: context.clock().toISOString(),
+    })
+  }
   return {
     materializationId: context.materializationId(),
     createdAt: context.clock().toISOString(),
-    expectedSource: {
-      source: command.source,
-      activeMaterializationId: active?.materializationId ?? null,
-      lastCommitId: active?.lastCommitId ?? null,
-    },
+    expectedSource,
   }
 }
 
@@ -259,6 +278,26 @@ async function stageProjectionCandidate(
   command: PreparedProjectionReplacement,
   candidate: ProjectionCandidate
 ): Promise<ReadyProjectionReplacement> {
+  const staged = await stageOrReuseCandidate(context, command, candidate)
+  return {
+    ...candidate,
+    staged,
+    // Commit time starts only after the source candidate is sealed ready.
+    identity: timestampCommitIdentity(command.identity, context.clock()),
+  }
+}
+
+async function stageOrReuseCandidate(
+  context: MaterializerContext,
+  command: PreparedProjectionReplacement,
+  candidate: ProjectionCandidate
+): Promise<StagedProjectionMaterialization> {
+  const adopted = candidate.adopted?.record
+  if (adopted?.status === "ready" && adopted.rootCount !== null) {
+    // The entries are left unread: the pinned version and definition that produced this candidate
+    // are the ones this run is bound to.
+    return { rootCount: adopted.rootCount, assertionCount: adopted.assertionCount ?? 0 }
+  }
   const input = {
     source: command.source,
     materializationId: candidate.materializationId,
@@ -270,15 +309,10 @@ async function stageProjectionCandidate(
     createdAt: candidate.createdAt,
     entries: command.entries,
     ...(command.base ? { base: command.base } : {}),
+    resumeStagingOrdinal: candidate.adopted?.resumeStagingOrdinal ?? 0,
     validateEntry: createProjectionEntryValidator(context.ontology, command.resolved),
   }
-  const staged = await stageCandidateEntries(context, input, command.signal)
-  return {
-    ...candidate,
-    staged,
-    // Commit time starts only after the source candidate is sealed ready.
-    identity: timestampCommitIdentity(command.identity, context.clock()),
-  }
+  return stageCandidateEntries(context, input, command.signal)
 }
 
 async function stageCandidateEntries(
@@ -462,32 +496,6 @@ function projectionActivation(
   }
 }
 
-async function abandonFailedCandidate(
-  context: MaterializerContext,
-  command: PreparedProjectionReplacement,
-  candidate: ProjectionCandidate,
-  error: unknown
-): Promise<void> {
-  if (!shouldAbandonCandidate(error)) return
-  await context.storage.ontology.sources.abandon({
-    kind: "candidate",
-    projectId: context.projectId,
-    source: command.source,
-    materializationId: candidate.materializationId,
-    execution: command.execution,
-    abandonedAt: context.clock().toISOString(),
-  })
-}
-
-function shouldAbandonCandidate(error: unknown): boolean {
-  if (error instanceof MaterializationValidationError) return true
-  if (error instanceof MaterializationCancellationError) return true
-  return (
-    error instanceof MaterializationConflictError &&
-    (error.kind === "run-correlation" || error.kind === "idempotency")
-  )
-}
-
 function validateProjectionWatermark(
   active: Awaited<ReturnType<OntologyStorage["sources"]["getActive"]>>,
   next: PinnedDatasetVersion
@@ -496,6 +504,16 @@ function validateProjectionWatermark(
   assertPinnedDatasetWatermark(active.datasetVersion, next, "Projection replacement")
 }
 
+function sameSourceBase(
+  left: ProjectionSourceBase | undefined,
+  right: ProjectionSourceBase | undefined
+): boolean {
+  return (
+    left?.materializationId === right?.materializationId &&
+    left?.lastCommitId === right?.lastCommitId
+  )
+}
+
 function assertDeltaBase(
   active: OntologySourceRecord | null,
   command: PreparedProjectionReplacement

package/src/materializer/projections/source-materialization.ts

@@ -23,8 +23,10 @@ export interface StagedProjectionMaterialization {
 /**
  * Persist and seal one explicit source-materialization candidate.
  *
- * A candidate is deliberately left staging/ready when execution is interrupted. The next claim
- * reclaims it with a fresh execution token; age is never used to infer abandonment.
+ * A candidate is deliberately left staging/ready when execution is interrupted. The next delivery
+ * of the same run adopts it and resumes at `resumeStagingOrdinal`: the entries are read again from
+ * the start, since they are deterministic for the pinned version, but only the roots from that
+ * ordinal on are written. Age is never used to infer abandonment; the run's end is.
  */
 export async function stageProjectionMaterialization(
   context: Pick<
@@ -42,6 +44,8 @@ export async function stageProjectionMaterialization(
     readonly createdAt: string
     readonly entries: AsyncIterable<ProjectionSourceEntry | ProjectionSourceDeletion>
     readonly base?: ProjectionSourceBase
+    /** First root ordinal to write; earlier ones are already staged. */
+    readonly resumeStagingOrdinal: number
     readonly validateEntry: ProjectionEntryValidator
     readonly signal?: AbortSignal
   }
@@ -77,16 +81,17 @@ export async function stageProjectionMaterialization(
           context.projectionRegistry.resolveSource(input.source.projectionId),
           root
         )
-        yield { root, stagingOrdinal: rootCount++ }
+        const stagingOrdinal = rootCount++
+        if (stagingOrdinal >= input.resumeStagingOrdinal) yield { root, stagingOrdinal }
         continue
       }
       const entry = input.validateEntry(normalizeProjectionSourceEntry(rawEntry))
-      const stagingOrdinal = rootCount
+      const stagingOrdinal = rootCount++
+      assertionCount += entry.assertions.length
+      if (stagingOrdinal < input.resumeStagingOrdinal) continue
       for (const assertion of entry.assertions) {
-        assertionCount += 1
         yield { root: entry.root, assertion, stagingOrdinal }
       }
-      rootCount += 1
     }
     throwIfAborted(input.signal)
   }

package/src/projections/types.ts

@@ -6,6 +6,7 @@ export const PROJECTION_RUN_FAILURE_CODES = [
   "runtime.cancelled",
   "queue.enqueue_failed",
   "projection.execution_failed",
+  "storage.unavailable",
 ] as const satisfies readonly [SixbErrorCode, ...SixbErrorCode[]]
 
 export type ProjectionRunFailureCode = (typeof PROJECTION_RUN_FAILURE_CODES)[number]

package/src/storage/index.ts

@@ -449,8 +449,11 @@ export {
   snapshotObjectReadExecutionLimits,
 } from "./objects"
 export type {
+  AbandonRunSourceMaterializationInput,
   AbandonSourceMaterializationCandidateInput,
   AbandonSourceMaterializationInput,
+  AdoptedSourceMaterialization,
+  AdoptSourceMaterializationInput,
   ApplyMaterializationChunkInput,
   ApplyMaterializationResult,
   AssertSourceMaterializationExecution,
@@ -530,9 +533,9 @@ export type {
   OntologyStorage,
   OntologyVectorStorage,
   ProjectionOntologyCommitIntent,
+  PurgeAbandonedSourceMaterializationsInput,
   PurgePublishedOntologyOutboxInput,
   ReadMaterializationObjectExistenceInput,
-  ReclaimSourceMaterializationInput,
   RescheduleOntologyOutboxLeaseInput,
   SourceActivationWrite,
   SourceReplacementLinkState,
@@ -595,6 +598,7 @@ export {
 export type {
   AdvanceProjectionTelemetryCheckpointInput,
   FailProjectionRunEnqueueInput,
+  FindSupersedingProjectionRunInput,
   FinishProjectionRunInput,
   LinkProjectionRunRecord,
   LinkProjectionTarget,
@@ -616,6 +620,7 @@ export type {
   ProjectionTarget,
   ProjectionTelemetryCheckpoint,
   QueueProjectionRunInput,
+  RecordProjectionAttemptFailureInput,
   RecordProjectionMissingTargetInput,
   StartOrReclaimProjectionRunInput,
   TelemetryProjectionRunRecord,