@sixb/core 0.0.1

package/src/materializer/effective/validate.ts

@@ -0,0 +1,180 @@
+import type { JsonValue } from "../../json"
+import { MaterializationValidationError } from "../../materialization/errors"
+import type {
+  OntologyLinkRef,
+  OntologyObjectRef,
+  TelemetryPointWrite,
+} from "../../materialization/model"
+import type { OntologyRegistry } from "../../ontology"
+import {
+  assertKnownProperties,
+  normalizeLinkProperties,
+  normalizeObjectProperties,
+  normalizeSchemaValue,
+  validateLinkProperties,
+  validateObjectProperties,
+  validatePropertyValue,
+  validateTelemetryUnit,
+} from "../../ontology/validation"
+
+export function validateObjectRef(ontology: OntologyRegistry, ref: OntologyObjectRef): void {
+  materializationValidation(() => {
+    ontology.resolveObjectType(ref.objectTypeId)
+  })
+}
+
+export function validateLinkRef(ontology: OntologyRegistry, ref: OntologyLinkRef): void {
+  materializationValidation(() => {
+    const sourceType = ontology.resolveObjectType(ref.source.objectTypeId)
+    ontology.resolveObjectType(ref.target.objectTypeId)
+    const link = sourceType.links.find((candidate) => candidate.id === ref.linkId)
+    if (!link) throw new Error(`[Sixb] Unknown link '${ref.source.objectTypeId}.${ref.linkId}'.`)
+    if (!ontology.isValidLinkTarget(link.targetObjectTypeId, ref.target.objectTypeId)) {
+      throw new Error(
+        `[Sixb] Link '${ref.source.objectTypeId}.${ref.linkId}' cannot target '${ref.target.objectTypeId}'.`
+      )
+    }
+  })
+}
+
+export function validateObjectAuthorityProperties(
+  ontology: OntologyRegistry,
+  ref: OntologyObjectRef,
+  properties: Readonly<Record<string, JsonValue>>
+): Readonly<Record<string, JsonValue>> {
+  return materializationValidation(() => {
+    const objectType = ontology.resolveObjectType(ref.objectTypeId)
+    assertKnownProperties(objectType, properties)
+    const primaryId = ontology.getPrimaryPropertyId(ref.objectTypeId)
+    const result = { ...properties }
+    if (primaryId in result) {
+      if (result[primaryId] !== ref.primaryId) {
+        throw new Error(
+          `[Sixb] Primary property '${primaryId}' must match object identity '${ref.primaryId}'.`
+        )
+      }
+      delete result[primaryId]
+    }
+    for (const propertyId of Object.keys(result)) {
+      const property = objectType.properties.find((candidate) => candidate.id === propertyId)!
+      if (property.mode === "telemetry") {
+        throw new Error(
+          `[Sixb] Static authority cannot write telemetry property '${ref.objectTypeId}.${propertyId}'.`
+        )
+      }
+    }
+    const valueTypesById = ontology.getValueTypesById()
+    validateObjectProperties(objectType, result, valueTypesById)
+    return normalizeObjectProperties(objectType.properties, result, valueTypesById, objectType.id)
+  })
+}
+
+export function validateObjectPatchPropertyIds(
+  ontology: OntologyRegistry,
+  ref: OntologyObjectRef,
+  propertyIds: readonly string[],
+  label: "unset" | "reset"
+): void {
+  materializationValidation(() => {
+    const objectType = ontology.resolveObjectType(ref.objectTypeId)
+    const primaryId = ontology.getPrimaryPropertyId(ref.objectTypeId)
+    for (const propertyId of propertyIds) {
+      const property = objectType.properties.find((candidate) => candidate.id === propertyId)
+      if (!property) {
+        throw new Error(
+          `[Sixb] Object patch ${label} references unknown property '${ref.objectTypeId}.${propertyId}'.`
+        )
+      }
+      if (propertyId === primaryId) {
+        throw new Error(
+          `[Sixb] Object patch ${label} cannot target primary property '${ref.objectTypeId}.${propertyId}'.`
+        )
+      }
+      if (property.mode === "telemetry") {
+        throw new Error(
+          `[Sixb] Object patch ${label} cannot target telemetry property '${ref.objectTypeId}.${propertyId}'.`
+        )
+      }
+    }
+  })
+}
+
+export function validateEffectiveObject(
+  ontology: OntologyRegistry,
+  ref: OntologyObjectRef,
+  properties: Readonly<Record<string, JsonValue>>
+): void {
+  materializationValidation(() => {
+    const objectType = ontology.resolveObjectType(ref.objectTypeId)
+    assertKnownProperties(objectType, properties)
+    for (const property of objectType.properties) {
+      if (
+        property.mode !== "telemetry" &&
+        property.required &&
+        properties[property.id] === undefined
+      ) {
+        throw new Error(
+          `[Sixb] Missing required property '${property.id}' for object type '${objectType.id}'`
+        )
+      }
+    }
+    validateObjectProperties(objectType, properties, ontology.getValueTypesById())
+  })
+}
+
+export function validateLinkAuthorityProperties(
+  ontology: OntologyRegistry,
+  ref: OntologyLinkRef,
+  properties: Readonly<Record<string, JsonValue>> | undefined
+): Readonly<Record<string, JsonValue>> | undefined {
+  return materializationValidation(() => {
+    const objectType = ontology.resolveObjectType(ref.source.objectTypeId)
+    const link = objectType.links.find((candidate) => candidate.id === ref.linkId)
+    if (!link) throw new Error(`[Sixb] Unknown link '${ref.source.objectTypeId}.${ref.linkId}'.`)
+    if (!ontology.isValidLinkTarget(link.targetObjectTypeId, ref.target.objectTypeId)) {
+      throw new Error(
+        `[Sixb] Link '${ref.source.objectTypeId}.${ref.linkId}' cannot target '${ref.target.objectTypeId}'.`
+      )
+    }
+    const valueTypesById = ontology.getValueTypesById()
+    validateLinkProperties(objectType, link, properties, undefined, valueTypesById)
+    return normalizeLinkProperties(objectType, link, properties, valueTypesById)
+  })
+}
+
+export function validateTelemetryPoint(
+  ontology: OntologyRegistry,
+  point: TelemetryPointWrite
+): TelemetryPointWrite {
+  return materializationValidation(() => {
+    const objectType = ontology.resolveObjectType(point.series.object.objectTypeId)
+    const property = objectType.properties.find(
+      (candidate) => candidate.id === point.series.propertyId
+    )
+    if (!property || property.mode !== "telemetry") {
+      throw new Error(
+        `[Sixb] Property '${point.series.object.objectTypeId}.${point.series.propertyId}' is not telemetry-enabled.`
+      )
+    }
+    const path = `${objectType.id}.${property.id}`
+    const valueTypesById = ontology.getValueTypesById()
+    validatePropertyValue(property, point.value, path, valueTypesById)
+    validateTelemetryUnit(property, path, point.unit, valueTypesById)
+    return {
+      ...point,
+      value:
+        point.value === null
+          ? null
+          : normalizeSchemaValue(property.schema, point.value, path, valueTypesById),
+    }
+  })
+}
+
+function materializationValidation<T>(run: () => T): T {
+  try {
+    return run()
+  } catch (error) {
+    if (error instanceof MaterializationValidationError) throw error
+    throw new MaterializationValidationError(error instanceof Error ? error.message : String(error))
+  }
+}

package/src/materializer/execution/commit-lifecycle.ts

@@ -0,0 +1,57 @@
+import { MaterializationConflictError } from "../../materialization/errors"
+import type {
+  EditCommitResult,
+  ProjectionCommitResult,
+  TelemetryCommitResult,
+} from "../../materialization/model"
+import { isStorageSerializationFailure } from "../../storage"
+import type { OntologyCommitRecord } from "../../storage/ontology"
+import type { MaterializerContext, MaterializerStorage } from "../context"
+import type { CommitIdentity } from "../shared/identity"
+
+export async function replayCommitRecord(
+  context: Pick<MaterializerContext, "projectId" | "storage">,
+  identity: CommitIdentity,
+  storage: MaterializerStorage = context.storage
+): Promise<OntologyCommitRecord | null> {
+  const existing = await storage.ontology.commits.getByIdempotencyKey({
+    projectId: context.projectId,
+    idempotencyKey: identity.idempotencyKey,
+  })
+  if (!existing) return null
+  if (existing.requestHash !== identity.requestHash) {
+    throw new MaterializationConflictError(
+      "idempotency",
+      `Idempotency key '${identity.idempotencyKey}' was reused with different intent.`
+    )
+  }
+  return existing
+}
+
+export async function replayCommit<
+  TResult extends EditCommitResult | ProjectionCommitResult | TelemetryCommitResult,
+>(
+  context: Pick<MaterializerContext, "projectId" | "storage">,
+  identity: CommitIdentity,
+  storage: MaterializerStorage = context.storage
+): Promise<TResult | null> {
+  const existing = await replayCommitRecord(context, identity, storage)
+  if (!existing) return null
+  return { ...structuredClone(existing.result), created: false } as TResult
+}
+
+export async function withSerializationRetry<T>(
+  context: Pick<MaterializerContext, "maxSerializationRetries" | "onSerializationRetry">,
+  run: () => Promise<T>
+): Promise<T> {
+  for (let attempt = 0; ; attempt += 1) {
+    try {
+      return await run()
+    } catch (error) {
+      if (!isStorageSerializationFailure(error) || attempt >= context.maxSerializationRetries) {
+        throw error
+      }
+      context.onSerializationRetry?.(attempt + 1, error)
+    }
+  }
+}

package/src/materializer/execution/plan-stream.ts

@@ -0,0 +1,97 @@
+import { utf8JsonByteLength } from "../../materialization/refs"
+import type {
+  ExactEffectiveLinkDelete,
+  ExactEffectiveLinkWrite,
+  ExactEffectiveObjectDelete,
+  ExactEffectiveObjectWrite,
+  ExactLinkOverrideDelete,
+  ExactLinkOverrideWrite,
+  ExactObjectOverrideDelete,
+  ExactObjectOverrideWrite,
+  ExactTimeseriesPointWrite,
+  MaterializationPlanChunk,
+  MaterializationPlanWorkItem,
+  OntologyOutboxWrite,
+} from "../../storage/ontology"
+import type { MaterializationBatching } from "../shared/batching"
+import { chunkBySize } from "../shared/chunking"
+
+export type MaterializationPlanItem =
+  | MaterializationPlanWorkItem
+  | { readonly kind: "outbox"; readonly value: OntologyOutboxWrite }
+
+export async function* planStream(
+  items: Iterable<MaterializationPlanItem> | AsyncIterable<MaterializationPlanItem>,
+  batching: MaterializationBatching
+): AsyncIterable<MaterializationPlanChunk> {
+  for await (const itemsChunk of chunkBySize(items, {
+    maxRows: batching.planChunkRows,
+    maxBytes: batching.planChunkBytes,
+    byteLength: utf8JsonByteLength,
+  })) {
+    const chunk = mutableChunk()
+    for (const item of itemsChunk) appendItem(chunk, item)
+    yield chunk
+  }
+}
+
+type MutableChunk = {
+  overrides: {
+    objectUpserts: ExactObjectOverrideWrite[]
+    objectDeletes: ExactObjectOverrideDelete[]
+    linkUpserts: ExactLinkOverrideWrite[]
+    linkDeletes: ExactLinkOverrideDelete[]
+  }
+  effective: {
+    objectUpserts: ExactEffectiveObjectWrite[]
+    objectDeletes: ExactEffectiveObjectDelete[]
+    linkUpserts: ExactEffectiveLinkWrite[]
+    linkDeletes: ExactEffectiveLinkDelete[]
+  }
+  timeseries: { pointUpserts: ExactTimeseriesPointWrite[] }
+  outbox: OntologyOutboxWrite[]
+}
+
+function mutableChunk(): MutableChunk {
+  return {
+    overrides: { objectUpserts: [], objectDeletes: [], linkUpserts: [], linkDeletes: [] },
+    effective: { objectUpserts: [], objectDeletes: [], linkUpserts: [], linkDeletes: [] },
+    timeseries: { pointUpserts: [] },
+    outbox: [],
+  }
+}
+
+function appendItem(chunk: MutableChunk, item: MaterializationPlanItem): void {
+  switch (item.kind) {
+    case "object-override-upsert":
+      chunk.overrides.objectUpserts.push(item.value)
+      break
+    case "object-override-delete":
+      chunk.overrides.objectDeletes.push(item.value)
+      break
+    case "link-override-upsert":
+      chunk.overrides.linkUpserts.push(item.value)
+      break
+    case "link-override-delete":
+      chunk.overrides.linkDeletes.push(item.value)
+      break
+    case "object-upsert":
+      chunk.effective.objectUpserts.push(item.value)
+      break
+    case "object-delete":
+      chunk.effective.objectDeletes.push(item.value)
+      break
+    case "link-upsert":
+      chunk.effective.linkUpserts.push(item.value)
+      break
+    case "link-delete":
+      chunk.effective.linkDeletes.push(item.value)
+      break
+    case "point-upsert":
+      chunk.timeseries.pointUpserts.push(item.value)
+      break
+    case "outbox":
+      chunk.outbox.push(item.value)
+      break
+  }
+}

package/src/materializer/execution/run-correlation.ts

@@ -0,0 +1,73 @@
+import {
+  MaterializationConflictError,
+  MaterializationValidationError,
+} from "../../materialization/errors"
+import type { ProjectionMaterializationIdentity } from "../../materialization/model"
+import type { ProjectionDefinition, ResolvedProjection } from "../../projections/types"
+import type { ProjectionRunRecord, ProjectionRunStorage, Storage } from "../../storage"
+
+export interface LockProjectionExecutionInput {
+  readonly projectId: string
+  readonly projectionRunId: string
+  readonly executionToken: string
+  readonly identity: ProjectionMaterializationIdentity
+  readonly resolved: ResolvedProjection<ProjectionDefinition>
+  readonly capabilityErrorMessage?: string
+}
+
+export interface LockedProjectionExecution {
+  readonly projectionRuns: ProjectionRunStorage
+  readonly run: ProjectionRunRecord
+}
+
+/** Locks and validates the current execution inside the materialization transaction. */
+export async function lockProjectionRunForMaterialization(
+  storage: Pick<Storage, "projectionRuns">,
+  input: LockProjectionExecutionInput
+): Promise<LockedProjectionExecution> {
+  const projectionRuns = storage.projectionRuns
+  if (!projectionRuns) {
+    throw new MaterializationValidationError(
+      input.capabilityErrorMessage ??
+        "Storage transaction does not provide projection run capabilities."
+    )
+  }
+
+  const run = await projectionRuns.lockForMaterialization({
+    id: input.projectionRunId,
+    projectId: input.projectId,
+    executionToken: input.executionToken,
+    identity: input.identity,
+  })
+  assertProjectionRunTargets(run, input.resolved)
+  return { projectionRuns, run }
+}
+
+export function assertProjectionRunTargets(
+  run: ProjectionRunRecord,
+  resolved: ResolvedProjection<ProjectionDefinition>
+): void {
+  if (projectionRunTargetsMatch(run, resolved.definition)) return
+
+  throw new MaterializationConflictError(
+    "run-correlation",
+    `Projection run '${run.id}' target object types do not match projection '${resolved.projectionId}'.`
+  )
+}
+
+function projectionRunTargetsMatch(
+  run: ProjectionRunRecord,
+  definition: ProjectionDefinition
+): boolean {
+  switch (definition._tag) {
+    case "ObjectProjectionDefinition":
+    case "TelemetryProjectionDefinition":
+      return "objectTypeId" in run.target && run.target.objectTypeId === definition.objectTypeId
+    case "LinkProjectionDefinition":
+      return (
+        "sourceObjectTypeId" in run.target &&
+        run.target.sourceObjectTypeId === definition.sourceObjectTypeId &&
+        run.target.targetObjectTypeId === definition.targetObjectTypeId
+      )
+  }
+}

package/src/materializer/execution/work-executor.ts

@@ -0,0 +1,143 @@
+import { MaterializationValidationError } from "../../materialization/errors"
+import { utf8JsonByteLength } from "../../materialization/refs"
+import type {
+  MaterializationSession,
+  MaterializationWorkRecord,
+  OntologyMaterializationStorage,
+} from "../../storage/ontology"
+import type { MaterializerContext } from "../context"
+import { sequenceMaterializationEvent } from "../effective/build-events"
+import { throwIfAborted } from "../shared/abort"
+import { chunkBySize } from "../shared/chunking"
+import type { TimedCommitIdentity } from "../shared/identity"
+import { type MaterializationPlanItem, planStream } from "./plan-stream"
+import { outboxItem } from "./work-records"
+
+type BatchingContext = Pick<MaterializerContext, "batching">
+
+type EventContext = Pick<MaterializerContext, "batching" | "projectId">
+
+export async function applyItems(
+  context: BatchingContext,
+  storage: OntologyMaterializationStorage,
+  session: MaterializationSession,
+  items: Iterable<MaterializationPlanItem>,
+  signal?: AbortSignal
+): Promise<void> {
+  for await (const chunk of planStream(items, context.batching)) {
+    throwIfAborted(signal)
+    await storage.applyChunk({ session, chunk })
+  }
+}
+
+export async function stageWorkBounded(
+  context: BatchingContext,
+  storage: OntologyMaterializationStorage,
+  session: MaterializationSession,
+  records: readonly MaterializationWorkRecord[]
+): Promise<void> {
+  for await (const chunk of chunkBySize(records, {
+    maxRows: context.batching.planChunkRows,
+    maxBytes: context.batching.planChunkBytes,
+    byteLength: utf8JsonByteLength,
+  })) {
+    await storage.stageWork({ session, records: chunk })
+  }
+}
+
+export async function drainStagedWork(
+  context: BatchingContext,
+  storage: OntologyMaterializationStorage,
+  session: MaterializationSession,
+  signal?: AbortSignal
+): Promise<void> {
+  let phase: number | null = null
+  let pending: MaterializationPlanItem[] = []
+  const flush = async () => {
+    if (pending.length === 0) return
+    await applyItems(context, storage, session, pending, signal)
+    pending = []
+  }
+  for await (const page of storage.streamWork({
+    session,
+    order: "apply",
+    pageRows: context.batching.planChunkRows,
+  })) {
+    for (const record of page.records) {
+      if (record.kind !== "plan") {
+        throw new MaterializationValidationError("Provider returned non-plan apply work.")
+      }
+      if (phase !== null && record.applyPhase !== phase) await flush()
+      phase = record.applyPhase
+      pending.push(record.item)
+      if (pending.length >= context.batching.planChunkRows) await flush()
+    }
+  }
+  await flush()
+}
+
+export async function validateStagedCardinality(
+  context: BatchingContext,
+  storage: OntologyMaterializationStorage,
+  session: MaterializationSession,
+  signal?: AbortSignal
+): Promise<void> {
+  let currentScope: string | null = null
+  let occupant: string | null = null
+  for await (const page of storage.streamWork({
+    session,
+    order: "cardinality",
+    pageRows: context.batching.statePageRows,
+  })) {
+    throwIfAborted(signal)
+    for (const record of page.records) {
+      if (record.kind !== "cardinality") {
+        throw new MaterializationValidationError(
+          "Provider returned non-cardinality cardinality work."
+        )
+      }
+      if (record.scopeSortKey !== currentScope) {
+        currentScope = record.scopeSortKey
+        occupant = null
+      }
+      if (!record.occupied) continue
+      if (occupant && occupant !== record.linkSortKey) {
+        throw new MaterializationValidationError(
+          `Link scope '${record.ref.source.objectTypeId}.${record.ref.linkId}' has cardinality one.`
+        )
+      }
+      occupant = record.linkSortKey
+    }
+  }
+}
+
+export async function drainStagedEvents(
+  context: EventContext,
+  storage: OntologyMaterializationStorage,
+  session: MaterializationSession,
+  identity: TimedCommitIdentity,
+  signal?: AbortSignal
+): Promise<number> {
+  let ordinal = 0
+  for await (const page of storage.streamWork({
+    session,
+    order: "event",
+    pageRows: context.batching.planChunkRows,
+  })) {
+    const items: MaterializationPlanItem[] = []
+    for (const record of page.records) {
+      if (record.kind !== "event") {
+        throw new MaterializationValidationError("Provider returned non-event event work.")
+      }
+      const event = sequenceMaterializationEvent(
+        context.projectId,
+        identity.commitId,
+        ordinal++,
+        record.draft
+      )
+      items.push(outboxItem(event, identity.committedAt))
+    }
+    await applyItems(context, storage, session, items, signal)
+  }
+  return ordinal
+}