@sixb/core 0.1.21 → 0.1.22

package/src/materialization/model.ts

@@ -4,6 +4,7 @@ import type { PropertyChange, PropertyChangeMap } from "../events/property-chang
 import type { JsonValue } from "../json"
 import type { ProjectionProtocolIdentity, ProjectionRunFailureCode } from "../projections/types"
 import type { RecordActionRunInput } from "../storage/action-runs/types"
+import type { UniqueValueRef } from "../storage/ontology/materializations"
 import type { ObjectVectorWrite } from "./vectors"
 
 export type { ProjectionProtocolIdentity } from "../projections/types"
@@ -44,6 +45,7 @@ export interface PinnedDatasetVersion {
 interface ProjectionMaterializationIdentityBase {
   readonly projectionId: string
   readonly datasetVersion: PinnedDatasetVersion
+  /** The part of the ontology the projection reads, not the whole ontology. */
   readonly ontologyRevision: string
   readonly projectionRevision: string
   readonly ownershipHash: string
@@ -127,6 +129,11 @@ interface AtomicOntologyEditCommit extends BaseOntologyEditCommit {
   readonly expectedObjects: readonly ExpectedObjectRevision[]
   readonly expectedLinks: readonly ExpectedLinkRevision[]
   readonly expectedLinkScopes: readonly ExpectedLinkScopeRevision[]
+  /**
+   * Unique values no object held when the commit's author read them: a commit that lands after
+   * and gives one to an object makes this commit fail with an `expectation` conflict.
+   */
+  readonly expectedUnclaimedValues?: readonly UniqueValueRef[]
 }
 
 export type OntologyEditCommit =
@@ -160,6 +167,11 @@ export type OntologyEditCommit =
        * it as failed. Ungrouped operations keep per-operation isolation.
        */
       readonly operationGroups?: readonly (readonly string[])[]
+      /**
+       * Objects as the commit's author read them. One that changed since fails the whole commit
+       * with an `expectation` conflict, as in an atomic commit, rather than one operation.
+       */
+      readonly expectedObjects?: readonly ExpectedObjectRevision[]
     })
 
 export type ProjectionEntityRef =
@@ -282,10 +294,18 @@ export interface TelemetryAppend {
   readonly points: readonly TelemetryPointWrite[]
 }
 
-export interface MaterializationItemError {
-  readonly code: "validation"
-  readonly message: string
-}
+export type MaterializationItemError =
+  | {
+      readonly code: "validation"
+      readonly message: string
+    }
+  | {
+      /** Another object of the type holds a value the item claims. */
+      readonly code: "unique_violation"
+      readonly message: string
+      readonly objectTypeId: string
+      readonly propertyId: string
+    }
 
 export interface EffectiveObjectSnapshot {
   readonly ref: OntologyObjectRef

package/src/materializer/README.md

@@ -148,7 +148,7 @@ failed in between replaces nothing, and releases the candidate like any other en
 ordered edit operations
   -> transaction-local EditWorkingState
   -> update object, link-edge, and link-slot overrides
-  -> validate effective objects, endpoints, and cardinality
+  -> validate effective objects, endpoints, cardinality, and unique values
   -> diff final working state against commit-start state
   -> plan and finalize one commit
 ```
@@ -158,6 +158,12 @@ its mutable overrides let operation N+1 observe operation N. In `continue` mode,
 operation rolls back only its working override; provider and infrastructure failures abort the
 whole transaction.
 
+Unique property values are kept in a table of their own, maintained by every object change: its
+plan releases the values the object gives up in the first apply phase and claims those it takes in
+the last, so a commit can move values between objects. Storage refuses a claim on a value another
+object holds. A `continue` batch checks the same rule first, on the holders it loads with its
+state (`edits/unique-values.ts`), so only the colliding item fails.
+
 Action-backed edits carry the run's terminal record. The commit validates the trusted Action
 execution and the record's `executionId`, then inserts the record in its own transaction before new
 work, so edits never land without the run that made them and a run id commits once: a second insert

package/src/materializer/edits/commit.ts

@@ -1,9 +1,11 @@
 import {
   MaterializationConflictError,
   MaterializationValidationError,
+  UniqueValueViolationError,
 } from "../../materialization/errors"
 import type {
   EditCommitResult,
+  MaterializationItemError,
   OntologyEditCommit,
   OntologyEditOperation,
   OntologyMaterializationOrigin,
@@ -34,6 +36,7 @@ import { compileEditExecutionUnits, type EditExecutionUnit } from "./execution-u
 import { loadEditWorkingState } from "./load-state"
 import { applyEditOperation, type EditUndoJournal, undoEditJournal } from "./operations"
 import { stageEditPlan } from "./plan"
+import { assertValuesUnclaimed } from "./unique-values"
 import { loadReferencedUsers, type ReferencedUsers } from "./user-refs"
 import { authorizeVectorIndexingCommit } from "./vector-authority"
 import { commitVectorWrites } from "./vectors"
@@ -185,11 +188,20 @@ async function executeEditTransaction(
   await recordActionRun(storage, command.input)
 
   const session = await beginEditMaterialization(context, storage, command)
+  if (command.input.mode === "atomic") {
+    await assertValuesUnclaimed(
+      context,
+      storage.ontology.materializations,
+      session,
+      command.input.expectedUnclaimedValues ?? []
+    )
+  }
   const workingState = await loadEditWorkingState(
     context,
     storage.ontology.materializations,
     session,
-    command.input.operations
+    command.input.operations,
+    { checkUniqueValuesPerItem: command.input.mode === "continue" }
   )
   const outcomes = applyEditOperations(context, workingState, command)
   const changes = await stageEditPlan(
@@ -262,7 +274,7 @@ function editExpectations(input: NormalizedEditCommit) {
       linkScopes: input.expectedLinkScopes,
     }
   }
-  return { objects: [], links: [], linkScopes: [] }
+  return { objects: input.expectedObjects ?? [], links: [], linkScopes: [] }
 }
 
 function applyEditOperations(
@@ -275,6 +287,7 @@ function applyEditOperations(
   for (const unit of units) {
     outcomes.push(...applyEditExecutionUnit(context, state, command, unit))
   }
+
   return outcomes
 }
 
@@ -296,11 +309,22 @@ function applyOneEditOperation(
   operation: OntologyEditOperation,
   command: PreparedEditCommit
 ): OntologyOperationOutcome {
+  const journal: EditUndoJournal = []
   try {
-    return applyEditOperation(context, state, operation, command.identity, command.users)
+    const outcome = applyEditOperation(
+      context,
+      state,
+      operation,
+      command.identity,
+      command.users,
+      journal
+    )
+    if (command.input.mode === "continue") state.uniqueValues.assertDistinct()
+    return outcome
   } catch (error) {
     if (!isRecoverableEditValidation(command.input, error)) throw error
-    return { id: operation.id, ok: false, error: { code: "validation", message: error.message } }
+    undoEditJournal(journal)
+    return { id: operation.id, ok: false, error: itemError(error) }
   }
 }
 
@@ -334,27 +358,47 @@ function applyEditOperationGroup(
     } catch (error) {
       if (!isRecoverableEditValidation(command.input, error)) throw error
       undoEditJournal(journal)
-      return failedEditGroup(group, error.message)
+      return failedEditGroup(group, itemError(error))
     }
     if (!outcome.ok) {
       undoEditJournal(journal)
-      return failedEditGroup(group, outcome.error.message)
+      return failedEditGroup(group, outcome.error)
     }
     applied.push(outcome)
   }
+
+  // A group applies as a whole: values may move between its objects.
+  if (command.input.mode === "continue") {
+    try {
+      state.uniqueValues.assertDistinct()
+    } catch (error) {
+      if (!isRecoverableEditValidation(command.input, error)) throw error
+      undoEditJournal(journal)
+      return failedEditGroup(group, itemError(error))
+    }
+  }
   return applied
 }
 
 /** Reports every id in a rolled-back group as failed, so no position looks applied. */
 function failedEditGroup(
   group: readonly OntologyEditOperation[],
-  message: string
+  error: MaterializationItemError
 ): OntologyOperationOutcome[] {
-  return group.map((operation) => ({
-    id: operation.id,
-    ok: false as const,
-    error: { code: "validation" as const, message },
-  }))
+  return group.map((operation) => ({ id: operation.id, ok: false as const, error }))
+}
+
+/** Keeps a unique collision recognizable, as the same refusal is from a single write. */
+function itemError(error: MaterializationValidationError): MaterializationItemError {
+  if (error instanceof UniqueValueViolationError) {
+    return {
+      code: "unique_violation",
+      message: error.message,
+      objectTypeId: error.objectTypeId,
+      propertyId: error.propertyId,
+    }
+  }
+  return { code: "validation", message: error.message }
 }
 
 function isRecoverableEditValidation(

package/src/materializer/edits/load-state.ts

@@ -10,9 +10,11 @@ import type { MaterializerContext } from "../context"
 import { loadState, oneStateRequest } from "../effective/load-state"
 import { resolveEffectiveLinkSlotMember, usableLinkSlotOverride } from "../effective/resolve"
 import { buildEditReadSet, isKnownLinkRef, linkCardinality } from "./read-set"
+import { WorkingUniqueValues } from "./unique-values"
 import {
   distinctCardinalityOneScopes,
   type EditWorkingState,
+  type LoadingEditWorkingState,
   mergeWorkingState,
   resolveLinkEdge,
   resolveObject,
@@ -23,9 +25,11 @@ export async function loadEditWorkingState(
   context: MaterializerContext,
   storage: OntologyMaterializationStorage,
   session: MaterializationSession,
-  operations: Parameters<typeof buildEditReadSet>[1]
+  operations: Parameters<typeof buildEditReadSet>[1],
+  /** Continue mode checks unique values per item; storage checks an atomic commit as a whole. */
+  options: { readonly checkUniqueValuesPerItem: boolean }
 ): Promise<EditWorkingState> {
-  const state: EditWorkingState = {
+  const state: LoadingEditWorkingState = {
     objects: new Map(),
     links: {
       edges: new Map(),
@@ -35,7 +39,7 @@ export async function loadEditWorkingState(
   const readSet = buildEditReadSet(context.ontology, operations)
   const explicitLinkKeys = new Set(readSet.links.map(linkRefKey))
   const incidentLinks = new Map<string, MaterializationLinkState>()
-  const loadedState: EditWorkingState = {
+  const loadedState: LoadingEditWorkingState = {
     objects: state.objects,
     links: {
       edges: new Map(),
@@ -75,7 +79,11 @@ export async function loadEditWorkingState(
     [...incidentLinks.values()],
     revivingObjects
   )
-  return state
+
+  const uniqueValues = options.checkUniqueValuesPerItem
+    ? await WorkingUniqueValues.load(context, storage, session, operations, state.objects)
+    : WorkingUniqueValues.untracked(context.ontology)
+  return { ...state, uniqueValues }
 }
 
 function isRevivingObjectOperation(
@@ -95,7 +103,7 @@ async function mergeIncidentState(
   context: MaterializerContext,
   storage: OntologyMaterializationStorage,
   session: MaterializationSession,
-  state: EditWorkingState,
+  state: LoadingEditWorkingState,
   incidentStates: readonly MaterializationLinkState[],
   revivingObjects: ReadonlySet<string>
 ): Promise<void> {
@@ -175,7 +183,7 @@ async function mergeMissingLinkSlotEndpoints(
   context: MaterializerContext,
   storage: OntologyMaterializationStorage,
   session: MaterializationSession,
-  state: EditWorkingState
+  state: LoadingEditWorkingState
 ): Promise<void> {
   const missing = new Map<string, OntologyObjectRef>()
   for (const working of state.links.slots.values()) {
@@ -203,7 +211,7 @@ async function mergeLoadedState(
   context: MaterializerContext,
   storage: OntologyMaterializationStorage,
   session: MaterializationSession,
-  state: EditWorkingState,
+  state: LoadingEditWorkingState,
   request: MaterializationStateRequestChunk
 ): Promise<void> {
   const loaded = await loadState(context, storage, session, request)

package/src/materializer/edits/operations.ts

@@ -119,6 +119,7 @@ function applyObjectOperation(
     context,
     state,
     working,
+    currentEffective,
     transition.next,
     transition.editedAt,
     journal,
@@ -171,6 +172,7 @@ function applyObjectTransition(
   context: MaterializerContext,
   state: EditWorkingState,
   working: WorkingObject,
+  before: ResolvedObjectValue | null,
   next: WorkingObject["override"],
   nextEditedAt: Readonly<Record<string, string>>,
   journal?: EditUndoJournal,
@@ -180,6 +182,8 @@ function applyObjectTransition(
   const previousEditedAt = working.editedAt
   working.override = next
   working.editedAt = { ...nextEditedAt }
+
+  let undoUniqueValues: () => void
   try {
     const resolved = resolveObject(context.ontology, working)
     if (resolved) {
@@ -187,15 +191,23 @@ function applyObjectTransition(
       validateResolved?.(resolved)
     }
     validateWorkingCardinality(state.links.slots)
+    // Last, so an operation refused above leaves the index as it was.
+    undoUniqueValues = state.uniqueValues.transition(
+      working.ref,
+      before?.properties,
+      resolved?.properties
+    )
   } catch (error) {
     working.override = previous
     working.editedAt = previousEditedAt
     throw error
   }
+
   journal?.push({
     restore: () => {
       working.override = previous
       working.editedAt = previousEditedAt
+      undoUniqueValues()
     },
   })
 }

package/src/materializer/edits/plan.ts

@@ -29,6 +29,7 @@ import {
   appendLinkEdgeOverrideWork,
   appendLinkSlotOverrideWork,
   appendObjectOverrideWork,
+  appendUniqueValueWork,
   classificationWork,
   eventWork,
   planWork,
@@ -158,6 +159,7 @@ async function stageObject(
     ...items.map((item) => planWork(item, sortKey)),
   ]
   if (change) {
+    appendUniqueValueWork(work, change, context.ontology)
     work.push(eventWork(buildObjectMaterializationEventDraft(change)))
   }
   await stageWorkBounded(context, storage, session, work)

package/src/materializer/edits/unique-values.ts

@@ -0,0 +1,255 @@
+/**
+ * Unique values while an edit commit applies its operations.
+ *
+ * Storage refuses a duplicate when the commit applies, which fails the whole commit: exactly what
+ * an atomic commit needs, so only continue mode checks earlier, for a batch to fail only the item
+ * that collides:
+ *
+ * - before the operations run, one state read loads the holders of every value they could claim;
+ * - each object transition then moves its own values in a working index, O(changed values);
+ * - `assertDistinct` checks the values claimed since its last call, after each item or group.
+ *
+ * Storage stays the final guard: a value claimed by a commit that lands after this read is still
+ * refused when this commit applies.
+ */
+import type { JsonValue } from "../../json"
+import {
+  MaterializationConflictError,
+  UniqueValueViolationError,
+} from "../../materialization/errors"
+import type { OntologyEditOperation, OntologyObjectRef } from "../../materialization/model"
+import { objectRefKey } from "../../materialization/refs"
+import type { OntologyRegistry } from "../../ontology"
+import { uniqueValueKey } from "../../ontology/unique-values"
+import { normalizeSchemaValue } from "../../ontology/validation"
+import type {
+  ExactUniqueValueWrite,
+  MaterializationSession,
+  OntologyMaterializationStorage,
+  UniqueValueRef,
+} from "../../storage/ontology"
+import type { MaterializerContext } from "../context"
+import { oneStateRequest } from "../effective/load-state"
+import type { WorkingObject } from "./working-state"
+
+type Properties = Readonly<Record<string, JsonValue>> | undefined
+
+export class WorkingUniqueValues {
+  /** Values claimed since the last check, by `uniqueKey`. */
+  private readonly claims = new Map<string, ExactUniqueValueWrite>()
+
+  private constructor(
+    private readonly ontology: OntologyRegistry,
+    /** Holder primary ids by `uniqueKey`: more than one only between two checks. */
+    private readonly holders: Map<string, Set<string>>,
+    private readonly tracked: boolean
+  ) {}
+
+  /** For an atomic commit, which storage checks as a whole: transitions track nothing. */
+  static untracked(ontology: OntologyRegistry): WorkingUniqueValues {
+    return new WorkingUniqueValues(ontology, new Map(), false)
+  }
+
+  /** Reads the holders of every unique value the operations could give to their objects. */
+  static async load(
+    context: Pick<MaterializerContext, "ontology" | "batching">,
+    storage: OntologyMaterializationStorage,
+    session: MaterializationSession,
+    operations: readonly OntologyEditOperation[],
+    objects: ReadonlyMap<string, WorkingObject>
+  ): Promise<WorkingUniqueValues> {
+    const holders = new Map<string, Set<string>>()
+    const requested = candidateValues(context.ontology, operations, objects)
+    if (requested.length === 0) return new WorkingUniqueValues(context.ontology, holders, true)
+
+    const request = {
+      objects: [],
+      links: [],
+      linkScopes: [],
+      incidentObjects: [],
+      points: [],
+      uniqueValues: requested,
+    }
+    for await (const page of storage.streamState({
+      session,
+      requests: oneStateRequest(request),
+      pageRows: context.batching.statePageRows,
+    })) {
+      for (const holder of page.uniqueHolders ?? []) {
+        holders.set(uniqueKey(holder), new Set([holder.primaryId]))
+      }
+    }
+
+    return new WorkingUniqueValues(context.ontology, holders, true)
+  }
+
+  /** Moves the unique values of `ref` from `before` to `after`, and returns how to undo it. */
+  transition(ref: OntologyObjectRef, before: Properties, after: Properties): () => void {
+    if (!this.tracked) return () => undefined
+    const moves: { readonly released: string | null; readonly claimed: string | null }[] = []
+
+    for (const { propertyId, kind } of this.ontology.listUniqueProperties(ref.objectTypeId)) {
+      const released = uniqueValueKey(kind, before?.[propertyId])
+      const claimed = uniqueValueKey(kind, after?.[propertyId])
+      if (released === claimed) continue
+
+      const value = { objectTypeId: ref.objectTypeId, propertyId }
+      const releasedKey = released === null ? null : uniqueKey({ ...value, valueKey: released })
+      if (releasedKey !== null) this.holders.get(releasedKey)?.delete(ref.primaryId)
+
+      let claimedKey: string | null = null
+      if (claimed !== null) {
+        const claim = { ...value, valueKey: claimed, primaryId: ref.primaryId }
+        claimedKey = uniqueKey(claim)
+        this.holdersOf(claimedKey).add(ref.primaryId)
+        this.claims.set(claimedKey, claim)
+      }
+
+      moves.push({ released: releasedKey, claimed: claimedKey })
+    }
+
+    return () => {
+      for (const { released, claimed } of moves) {
+        if (claimed !== null) this.holders.get(claimed)?.delete(ref.primaryId)
+        if (released !== null) this.holdersOf(released).add(ref.primaryId)
+      }
+    }
+  }
+
+  /** Throws `UniqueValueViolationError` when a value claimed since the last check has two holders. */
+  assertDistinct(): void {
+    const claims = [...this.claims.entries()]
+    this.claims.clear()
+
+    for (const [key, claim] of claims) {
+      const holders = this.holders.get(key)
+      if (!holders || holders.size <= 1) continue
+      const holder = [...holders].find((primaryId) => primaryId !== claim.primaryId)
+      if (holder === undefined) continue
+      throw new UniqueValueViolationError(
+        claim.objectTypeId,
+        claim.propertyId,
+        holder,
+        claim.primaryId
+      )
+    }
+  }
+
+  private holdersOf(key: string): Set<string> {
+    let holders = this.holders.get(key)
+    if (!holders) {
+      holders = new Set()
+      this.holders.set(key, holders)
+    }
+    return holders
+  }
+}
+
+/**
+ * Rejects a commit whose author read a unique value as unclaimed when an object now holds it: an
+ * Action that found nothing and creates the holder is then replayed against the current state,
+ * instead of failing on the duplicate.
+ */
+export async function assertValuesUnclaimed(
+  context: Pick<MaterializerContext, "batching">,
+  storage: OntologyMaterializationStorage,
+  session: MaterializationSession,
+  values: readonly UniqueValueRef[]
+): Promise<void> {
+  if (values.length === 0) return
+
+  const request = {
+    objects: [],
+    links: [],
+    linkScopes: [],
+    incidentObjects: [],
+    points: [],
+    uniqueValues: values,
+  }
+  for await (const page of storage.streamState({
+    session,
+    requests: oneStateRequest(request),
+    pageRows: context.batching.statePageRows,
+  })) {
+    const [held] = page.uniqueHolders ?? []
+    if (!held) continue
+    throw new MaterializationConflictError(
+      "expectation",
+      `Expected no '${held.objectTypeId}' object to hold this '${held.propertyId}'.`
+    )
+  }
+}
+
+/**
+ * Every unique value an operated object could hold after the commit: those its operation writes,
+ * and those its source and its edit already assert, which a reset or a restore can bring back.
+ */
+function candidateValues(
+  ontology: OntologyRegistry,
+  operations: readonly OntologyEditOperation[],
+  objects: ReadonlyMap<string, WorkingObject>
+): UniqueValueRef[] {
+  const values = new Map<string, UniqueValueRef>()
+  const add = (ref: OntologyObjectRef, properties: Properties) => {
+    for (const value of uniqueValuesOf(ontology, ref, properties)) {
+      values.set(uniqueKey(value), value)
+    }
+  }
+
+  for (const operation of operations) {
+    if (!("objectTypeId" in operation.ref)) continue
+    if (ontology.listUniqueProperties(operation.ref.objectTypeId).length === 0) continue
+
+    if (operation.kind === "object.create" || operation.kind === "object.upsert") {
+      add(operation.ref, operation.properties)
+    }
+    if (operation.kind === "object.patch") add(operation.ref, operation.set)
+
+    const working = objects.get(objectRefKey(operation.ref))
+    add(operation.ref, working?.source?.assertion.properties)
+    const override = working?.override
+    if (override?.kind === "create") add(operation.ref, override.properties)
+    if (override?.kind === "patch") add(operation.ref, override.set)
+  }
+
+  return [...values.values()]
+}
+
+/** The keys of `properties`' unique values, normalized as the write will store them. */
+function uniqueValuesOf(
+  ontology: OntologyRegistry,
+  ref: OntologyObjectRef,
+  properties: Properties
+): UniqueValueRef[] {
+  if (!properties) return []
+  const objectType = ontology.getObjectTypeById(ref.objectTypeId)
+  if (!objectType) return []
+
+  const values: UniqueValueRef[] = []
+  for (const { propertyId, kind } of ontology.listUniqueProperties(ref.objectTypeId)) {
+    const raw = properties[propertyId]
+    const property = objectType.properties.find((candidate) => candidate.id === propertyId)
+    if (raw === undefined || raw === null || !property) continue
+
+    const valueKey = uniqueValueKey(kind, normalized(ontology, property.schema, raw))
+    if (valueKey !== null) values.push({ objectTypeId: ref.objectTypeId, propertyId, valueKey })
+  }
+  return values
+}
+
+/** An invalid value claims nothing: the operation that writes it is refused anyway. */
+function normalized(
+  ontology: OntologyRegistry,
+  schema: Parameters<typeof normalizeSchemaValue>[0],
+  value: JsonValue
+): JsonValue | undefined {
+  try {
+    return normalizeSchemaValue(schema, value, "", ontology.getValueTypesById())
+  } catch {
+    return undefined
+  }
+}
+
+function uniqueKey(value: UniqueValueRef): string {
+  return JSON.stringify([value.objectTypeId, value.propertyId, value.valueKey])
+}

package/src/materializer/edits/working-state.ts

@@ -29,6 +29,8 @@ import {
   usableLinkSlotOverride,
 } from "../effective/resolve"
 import { linkCardinality } from "./read-set"
+import type { WorkingUniqueValues } from "./unique-values"
+
 export interface WorkingObject {
   ref: OntologyObjectRef
   source: MaterializationObjectState["source"]
@@ -75,11 +77,16 @@ export interface EditWorkingState {
     readonly edges: Map<string, WorkingLinkEdge>
     readonly slots: Map<string, WorkingLinkSlot>
   }
+  /** Who holds the unique values the operations could take, kept current as they apply. */
+  readonly uniqueValues: WorkingUniqueValues
 }
 
+/** The working state while it loads, before its unique value holders are read. */
+export type LoadingEditWorkingState = Omit<EditWorkingState, "uniqueValues">
+
 export function mergeWorkingState(
   ontology: OntologyRegistry,
-  working: EditWorkingState,
+  working: LoadingEditWorkingState,
   state: MaterializationStatePage
 ): void {
   for (const object of state.objects) {