@sixb/core 0.1.15 → 0.1.17

package/src/storage/ontology/provider.ts

@@ -18,20 +18,16 @@ import { linkRefKey, objectRefKey, projectionEntityKey } from "../../materializa
 import type { OntologyCommitOriginSelector, OntologyCommitWrite } from "./commits"
 import type {
   FinalizeMaterializationInput,
-  MaterializationEventWorkRecord,
   MaterializationLinkScopeRevision,
-  MaterializationPlanChunk,
   MaterializationPlanHeader,
-  MaterializationPlanWorkItem,
   MaterializationSession,
   MaterializationWorkRecord,
   SourceActivationWrite,
   StageMaterializationWorkInput,
   StoredLinkSlotOverride,
-  StreamMaterializationWorkInput,
 } from "./materializations"
 import { assertNonblank, assertTimestamp } from "./provider-validation"
-import { assertWorkRecord, workUniquenessKey } from "./provider-work"
+import { assertWorkRecord, materializationPlanKindRank, workUniquenessKey } from "./provider-work"
 import type {
   AssertSourceMaterializationExecutionInput,
   BeginSourceMaterializationInput,
@@ -46,6 +42,7 @@ import type {
  * from `@sixb/core/storage` instead.
  */
 
+export { materializationApplyPhase } from "./materializations"
 export { assertMaterializationHeader } from "./provider-header-validation"
 export {
   assertNonblank,
@@ -56,13 +53,28 @@ export {
 } from "./provider-validation"
 export {
   assertPageRows,
-  assertPlanChunkCorrelations,
+  assertReplacementPlanCommit,
   assertWorkRecord,
+  type CardinalityValidator,
+  type ClaimedOutboxDraft,
+  cardinalityViolation,
+  claimedOutboxRows,
   compareCardinalityWork,
   compareEventWork,
   comparePlanWork,
+  createCardinalityValidator,
+  type ExactEffectiveWrites,
+  type ExactOverrideWrites,
+  type ExactTimeseriesWrites,
+  type MaterializationPlanChunk,
   materializationChunkRows,
-  materializationPlanItems,
+  materializationOutboxWrite,
+  materializationPlanChunk,
+  materializationPlanKindRank,
+  type OntologyOutboxWrite,
+  type OutboxEventContext,
+  type PreparedReplacementIdentity,
+  prepareReplacementIdentities,
   workUniquenessKey,
 } from "./provider-work"
 
@@ -168,40 +180,15 @@ function isJsonValue(value: unknown): value is import("../../json").JsonValue {
   return isRecord(value) && Object.values(value).every(isJsonValue)
 }
 
-export interface ProviderReplacementSessionState {
-  readonly incremental?: boolean
-  readonly sourceId: string
-  readonly candidateMaterializationId: string
-  readonly previousMaterializationId: string | null
-  readonly projectionKind: "object" | "link"
-  objectStreamStarted: boolean
-  objectStreamCompleted: boolean
-  linkStreamStarted: boolean
-  linkStreamCompleted: boolean
-}
-
-export interface ProviderMaterializationLaneState {
-  started: boolean
-  completed: boolean
-  emittedCount: number
-}
-
 export class ProviderMaterializationSessionState {
   readonly id = randomUUID()
   readonly providerToken = {}
-  readonly workStreams: Record<
-    StreamMaterializationWorkInput["order"],
-    ProviderMaterializationLaneState
-  > = {
-    apply: { started: false, completed: false, emittedCount: 0 },
-    cardinality: { started: false, completed: false, emittedCount: 0 },
-    event: { started: false, completed: false, emittedCount: 0 },
-  }
   active = true
+  /** Staging closes once vector changes stream or the plan applies. */
   workSealed = false
-  appliedPlanCount = 0
-  appliedOutboxCount = 0
-  replacement: ProviderReplacementSessionState | null = null
+  vectorChangesStreamed = false
+  applied = false
+  appliedEventCount = 0
 
   constructor(
     readonly header: MaterializationPlanHeader,
@@ -213,6 +200,37 @@ export class ProviderMaterializationSessionState {
   }
 }
 
+/** Seals staging for the vector-change stream: once per session, before the plan applies. */
+export function beginMaterializationVectorChanges(
+  state: Pick<
+    ProviderMaterializationSessionState,
+    "workSealed" | "vectorChangesStreamed" | "applied"
+  >
+): void {
+  if (state.vectorChangesStreamed || state.applied) {
+    throw new MaterializationConflictError(
+      "effective-state",
+      "Materialization vector changes stream once per session, before the plan applies."
+    )
+  }
+  state.vectorChangesStreamed = true
+  state.workSealed = true
+}
+
+/** Seals staging for the apply step: once per session. */
+export function beginMaterializationApply(
+  state: Pick<ProviderMaterializationSessionState, "workSealed" | "applied">
+): void {
+  if (state.applied) {
+    throw new MaterializationConflictError(
+      "effective-state",
+      "A materialization plan applies once per session."
+    )
+  }
+  state.applied = true
+  state.workSealed = true
+}
+
 /**
  * Tracks the materialization sessions owned by one storage transaction.
  *
@@ -258,13 +276,19 @@ export interface PreparedMaterializationWork {
 }
 
 export function prepareMaterializationWork(
-  state: Pick<ProviderMaterializationSessionState, "header" | "workSealed" | "replacement">,
+  state: Pick<ProviderMaterializationSessionState, "header" | "workSealed">,
   input: StageMaterializationWorkInput
 ): readonly PreparedMaterializationWork[] {
+  if (state.header.plan) {
+    throw new MaterializationConflictError(
+      "effective-state",
+      "A plan-bound materialization session applies its plan and stages no work."
+    )
+  }
   if (state.workSealed) {
     throw new MaterializationConflictError(
       "effective-state",
-      "Materialization work cannot be staged after draining begins."
+      "Materialization work cannot be staged once vector changes stream or the plan applies."
     )
   }
   const keys = new Set<string>()
@@ -275,13 +299,10 @@ export function prepareMaterializationWork(
     if (keys.has(record.recordKey) || uniqueKeys.has(uniqueKey)) {
       throw duplicateMaterializationWork(record.recordKey)
     }
-    if (
-      record.kind === "incident-object" &&
-      (!state.replacement || state.replacement.linkStreamStarted)
-    ) {
+    if (record.kind === "incident-object" || record.kind === "object-existence") {
       throw new MaterializationConflictError(
         "effective-state",
-        "Incident replacement work must be staged before link state is streamed."
+        "Replacement work is staged on a replacement plan, not on a session."
       )
     }
     keys.add(record.recordKey)
@@ -290,75 +311,15 @@ export function prepareMaterializationWork(
   })
 }
 
-export interface MaterializationChunkSequenceState {
-  readonly workStreams: ProviderMaterializationSessionState["workStreams"]
-  readonly appliedPlanCount: number
-  readonly appliedOutboxCount: number
-}
-
-export function correlateMaterializationChunk(
-  state: MaterializationChunkSequenceState,
-  chunk: MaterializationPlanChunk,
-  actualItems: readonly MaterializationPlanWorkItem[],
-  expectedItems: readonly MaterializationPlanWorkItem[],
-  expectedEvents: readonly MaterializationEventWorkRecord[]
-): { readonly appliedPlanCount: number; readonly appliedOutboxCount: number } {
-  if (
-    actualItems.length > 0 &&
-    (!state.workStreams.apply.started ||
-      state.appliedPlanCount + actualItems.length > state.workStreams.apply.emittedCount)
-  ) {
-    invalidMaterializationCorrelation(
-      "Materialization plan items cannot be applied before they are streamed."
-    )
-  }
-  for (let index = 0; index < actualItems.length; index += 1) {
-    if (stableJsonStringify(actualItems[index]) !== stableJsonStringify(expectedItems[index])) {
-      invalidMaterializationCorrelation(
-        "Materialization plan items must be applied in exact streamed order."
-      )
-    }
-  }
-
-  if (
-    chunk.outbox.length > 0 &&
-    (!state.workStreams.event.started ||
-      state.appliedOutboxCount + chunk.outbox.length > state.workStreams.event.emittedCount)
-  ) {
-    invalidMaterializationCorrelation(
-      "Materialization events cannot be applied before they are streamed."
-    )
-  }
-  for (let index = 0; index < chunk.outbox.length; index += 1) {
-    const expected = expectedEvents[index]
-    const actual = chunk.outbox[index]?.envelope
-    if (!expected || !actual) {
-      invalidMaterializationCorrelation(
-        "Materialization outbox events must follow exact streamed order."
-      )
-    }
-    const { id: _id, commitOrdinal, ...draft } = actual
-    if (
-      commitOrdinal !== state.appliedOutboxCount + index ||
-      stableJsonStringify(draft) !== stableJsonStringify(expected.draft)
-    ) {
-      invalidMaterializationCorrelation(
-        "Materialization outbox events must follow exact streamed order."
-      )
-    }
-  }
-  return {
-    appliedPlanCount: state.appliedPlanCount + actualItems.length,
-    appliedOutboxCount: state.appliedOutboxCount + chunk.outbox.length,
-  }
-}
-
 export function assertMaterializationFinalizationCorrelation(
-  state: Pick<ProviderMaterializationSessionState, "header" | "appliedOutboxCount">,
+  state: Pick<ProviderMaterializationSessionState, "header" | "applied" | "appliedEventCount">,
   input: FinalizeMaterializationInput
 ): void {
   const { commit } = state.header
   const { result, sourceActivations } = input.finalization
+  if (!state.applied) {
+    invalidMaterializationCorrelation("A materialization plan must apply before it finalizes.")
+  }
   if (
     result.commitId !== commit.id ||
     result.kind !== commit.intent.kind ||
@@ -366,7 +327,7 @@ export function assertMaterializationFinalizationCorrelation(
     result.created !== true ||
     !Number.isSafeInteger(result.eventCount) ||
     result.eventCount < 0 ||
-    result.eventCount !== state.appliedOutboxCount
+    result.eventCount !== state.appliedEventCount
   ) {
     invalidMaterializationCorrelation(
       "Materialization result does not correlate with its commit intent."
@@ -392,34 +353,8 @@ export function assertMaterializationFinalizationCorrelation(
   }
 }
 
-export function assertMaterializationLaneCompletion(
-  state: Pick<
-    ProviderMaterializationSessionState,
-    "workStreams" | "appliedPlanCount" | "appliedOutboxCount"
-  >,
-  counts: { readonly apply: number; readonly cardinality: number; readonly event: number }
-): void {
-  if (counts.apply > 0 && !state.workStreams.apply.completed) {
-    invalidMaterializationCorrelation("Materialization plan work was not fully streamed.")
-  }
-  if (state.appliedPlanCount !== counts.apply) {
-    invalidMaterializationCorrelation("Materialization plan work was not applied exactly once.")
-  }
-  if (counts.cardinality > 0 && !state.workStreams.cardinality.completed) {
-    invalidMaterializationCorrelation("Materialization cardinality work was not fully validated.")
-  }
-  if (counts.event > 0 && !state.workStreams.event.completed) {
-    invalidMaterializationCorrelation("Materialization event work was not fully drained.")
-  }
-  if (counts.event !== state.appliedOutboxCount) {
-    invalidMaterializationCorrelation(
-      "Materialization event work was not fully written to the outbox."
-    )
-  }
-}
-
 export function assertSourceActivationCorrelation(
-  state: Pick<ProviderMaterializationSessionState, "header" | "replacement">,
+  state: Pick<ProviderMaterializationSessionState, "header">,
   activation: SourceActivationWrite
 ): void {
   const { commit } = state.header
@@ -445,18 +380,13 @@ export function assertSourceActivationCorrelation(
       "Source activation does not correlate with its projection commit."
     )
   }
-  const replacement = state.replacement
   if (
-    !replacement ||
-    replacement.sourceId !== activation.source.projectionId ||
-    replacement.candidateMaterializationId !== activation.materializationId ||
-    replacement.projectionKind !== activation.projectionKind ||
-    (activation.projectionKind === "object" &&
-      (!replacement.objectStreamCompleted || !replacement.linkStreamCompleted)) ||
-    (activation.projectionKind === "link" && !replacement.linkStreamCompleted)
+    !state.header.plan ||
+    state.header.plan.source.projectionId !== activation.source.projectionId ||
+    state.header.plan.materializationId !== activation.materializationId
   ) {
     invalidMaterializationCorrelation(
-      "Source activation does not match fully streamed replacement state."
+      "Source activation does not match the replacement plan the session applies."
     )
   }
 }
@@ -582,33 +512,6 @@ export function materializationWorkColumns(record: MaterializationWorkRecord): {
   return { lane: "none", majorOrder: 0, minorOrder: 0, sortOne: "", sortTwo: "" }
 }
 
-function materializationPlanKindRank(kind: MaterializationPlanWorkItem["kind"]): number {
-  switch (kind) {
-    case "object-override-upsert":
-      return 0
-    case "object-override-delete":
-      return 1
-    case "link-override-upsert":
-      return 2
-    case "link-override-delete":
-      return 3
-    case "link-slot-override-upsert":
-      return 4
-    case "link-slot-override-delete":
-      return 5
-    case "point-upsert":
-      return 6
-    case "link-delete":
-      return 7
-    case "object-delete":
-      return 8
-    case "object-upsert":
-      return 9
-    case "link-upsert":
-      return 10
-  }
-}
-
 export function duplicateMaterializationWork(key?: string): MaterializationConflictError {
   return new MaterializationConflictError(
     "effective-state",

package/src/storage/ontology/replacement-plans.ts

@@ -0,0 +1,117 @@
+import type {
+  EffectiveChangeCounts,
+  ProjectionEntityRef,
+  ProjectionExecution,
+  ProjectionSourceRef,
+} from "../../materialization/model"
+import type { OntologyCommitWrite } from "./commits"
+import type {
+  MaterializationObjectExistence,
+  MaterializationWorkRecord,
+  SourceReplacementLinkState,
+  SourceReplacementObjectState,
+} from "./materializations"
+
+/** A ready replacement candidate whose plan the calling execution builds or commits. */
+export interface ReplacementPlanRef {
+  readonly projectId: string
+  readonly source: ProjectionSourceRef
+  readonly materializationId: string
+  readonly execution: ProjectionExecution
+}
+
+export interface OpenReplacementPlanInput extends ReplacementPlanRef {
+  /** The commit the plan's work will carry, timed for a plan opened now. */
+  readonly commit: OntologyCommitWrite
+}
+
+export interface OpenedReplacementPlan {
+  /**
+   * The time of the commit the plan's work carries: a resumed plan keeps the one it was opened
+   * with. Its work is staged, and its session begins, with the commit at exactly this time.
+   */
+  readonly committedAt: string
+}
+
+export interface StreamReplacementPlanStateInput extends ReplacementPlanRef {
+  /** Objects are planned before links: link state needs the planned existence of its endpoints. */
+  readonly entityKind: "object" | "link"
+  readonly pageRows: number
+}
+
+export interface ReplacementPlanStatePage {
+  readonly objects: readonly SourceReplacementObjectState[]
+  readonly links: readonly SourceReplacementLinkState[]
+  /** Existence of the endpoints of `links`: as this plan leaves them, else as effective. */
+  readonly endpoints: readonly MaterializationObjectExistence[]
+}
+
+export interface PlannedReplacementIdentity {
+  readonly identity: ProjectionEntityRef
+  /** Every work record this identity contributes, possibly none. */
+  readonly records: readonly MaterializationWorkRecord[]
+}
+
+export interface StageReplacementPlanInput extends ReplacementPlanRef {
+  /** The commit the plan carries, timed as `open` returned; every record is checked against it. */
+  readonly commit: OntologyCommitWrite
+  /** Identities of the page streamed last, each planned with the revision it was read at. */
+  readonly planned: readonly PlannedReplacementIdentity[]
+}
+
+export interface PurgeReplacementPlansInput {
+  readonly projectId: string
+  /** Maximum rows one call deletes. */
+  readonly limit: number
+}
+
+export type ReplacementPlanStatus =
+  | { readonly fresh: true; readonly counts: EffectiveChangeCounts }
+  /** Identities that changed since they were planned, and those planned from them, are planned again. */
+  | { readonly fresh: false; readonly unplanned: number }
+
+/**
+ * The durable plan of a ready replacement candidate.
+ *
+ * It is built outside the commit transaction, one page at a time, and belongs to the candidate: a
+ * redelivery resumes it, and the candidate's end removes it. Every identity keeps the revision of
+ * the inputs its plan was read from. `refresh`, inside the commit transaction, plans again what
+ * changed since, so a commit only ever applies a plan that is still exact.
+ *
+ * Every commit records which entities' plan inputs it changed: the objects whose row, override or
+ * telemetry it wrote, the links whose row or edge override it wrote or whose live source assertion
+ * it activated or retired, and the link scopes whose slot override it wrote. `refresh` reads only
+ * what the commits since the plan was last fresh recorded, so its cost follows those commits, not
+ * the size of the plan.
+ */
+export interface OntologyReplacementPlanStorage {
+  /**
+   * Opens the candidate's plan, or resumes the one an earlier execution of its run left. A plan
+   * decides the entities of the active source it replaces: once that moved, it starts over.
+   */
+  open(input: OpenReplacementPlanInput): Promise<OpenedReplacementPlan>
+  /**
+   * State of the identities still to plan, in canonical order; each page reads one snapshot. The
+   * link stream first adds the links that identities planned since the previous link stream reach:
+   * those incident to an object whose existence flips, and the members of every scope a planned
+   * link changes.
+   */
+  streamState(input: StreamReplacementPlanStateInput): AsyncIterable<ReplacementPlanStatePage>
+  /** Plans identities streamed since they were last unplanned; each is planned once. */
+  stage(input: StageReplacementPlanInput): Promise<void>
+  /**
+   * Within the commit transaction, before the plan-bound session begins: unplans every planned
+   * identity whose inputs a commit changed since the plan was last fresh, with every identity
+   * planned from it, and adds the links such a commit brought within the plan's reach. A plan
+   * older than the changes storage still records is checked whole. A fresh plan returns the
+   * change counts its commit reports.
+   */
+  refresh(input: ReplacementPlanRef): Promise<ReplacementPlanStatus>
+  /**
+   * Deletes what is left of plans whose candidate is no longer ready, and the changes commits
+   * recorded that no ready plan still needs, a bounded number of rows at a time, and returns how
+   * many it deleted. A commit or an abandon may leave its plan behind: the rows are a cache
+   * nothing reads once the candidate moved on.
+   */
+  purge(input: PurgeReplacementPlansInput): Promise<number>
+}

package/src/storage/ontology/sources.ts

@@ -185,15 +185,6 @@ export interface PurgeAbandonedSourceMaterializationsInput {
   readonly limit: number
 }
 
-export interface SummarizeTerminalSourceMaterializationsInput {
-  readonly projectId: string
-}
-
-export interface TerminalSourceMaterializationSummary {
-  readonly count: number
-  readonly oldestTerminalAt: string | null
-}
-
 export interface AssertSourceMaterializationExecutionInput {
   readonly projectId: string
   readonly source: ProjectionSourceRef
@@ -234,7 +225,4 @@ export interface OntologySourceStorage {
   purgeAbandoned(
     input: PurgeAbandonedSourceMaterializationsInput
   ): Promise<CleanupTerminalSourceMaterializationsResult>
-  summarizeTerminal(
-    input: SummarizeTerminalSourceMaterializationsInput
-  ): Promise<TerminalSourceMaterializationSummary>
 }

package/src/storage/operation-scope.ts

@@ -181,13 +181,44 @@ function scopeMethod(
   if (kind === "async-generator") {
     return (...args: unknown[]) => {
       scope.assertAvailable()
-      return Reflect.apply(implementation, target, args)
+      return scopeSteps(
+        Reflect.apply(implementation, target, args) as AsyncIterable<unknown>,
+        scope
+      )
     }
   }
 
   return implementation.bind(target)
 }
 
+/**
+ * Each step of a stream runs in its own operation scope, released between steps: a page holds the
+ * provider's lock while it reads, never while its consumer works on it.
+ */
+function scopeSteps<T>(
+  stream: AsyncIterable<T>,
+  scope: StorageOperationScope
+): AsyncIterableIterator<T> {
+  let iterator: AsyncIterator<T> | undefined
+  const steps = () => {
+    iterator ??= stream[Symbol.asyncIterator]()
+    return iterator
+  }
+  const scoped: AsyncIterableIterator<T> = {
+    next: (...args) => scope.run(() => steps().next(...args)),
+    return: (value) =>
+      scope.run(async () => (await steps().return?.(value)) ?? { done: true, value }),
+    throw: (error) =>
+      scope.run(async () => {
+        const current = steps()
+        if (!current.throw) throw error
+        return current.throw(error)
+      }),
+    [Symbol.asyncIterator]: () => scoped,
+  }
+  return scoped
+}
+
 export function createAuthOperationScope<T extends AuthStorage>(
   target: T,
   scope: StorageOperationScope
@@ -230,6 +261,7 @@ export function createOntologyOperationScope<T extends OntologyStorage>(
     commits: createOperationScopedFacade(target.commits, scope),
     sources: createOperationScopedFacade(target.sources, scope),
     materializations: createOperationScopedFacade(target.materializations, scope),
+    replacementPlans: createOperationScopedFacade(target.replacementPlans, scope),
     outbox: createOperationScopedFacade(target.outbox, scope),
     ...(target.vectors ? { vectors: createOperationScopedFacade(target.vectors, scope) } : {}),
   }) as T

package/src/storage/projection-runs/in-memory.ts

@@ -285,13 +285,16 @@ export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
       assertProjectionRunNonEmpty(input.projectId, "projectId")
       assertProjectionRunListWindow(input.limit, "limit")
       assertProjectionRunListWindow(input.offset, "offset")
-      if (input.statuses?.length === 0) return { runs: [], hasMore: false, total: 0 }
+      if (input.statuses?.length === 0 || input.errorCodes?.length === 0) {
+        return { runs: [], hasMore: false, total: 0 }
+      }
 
       const order = input.order ?? "desc"
       const offset = input.offset ?? 0
       const limit = input.limit ?? this.rows.size
       const statuses = input.statuses ? new Set(input.statuses) : null
       const objectTypeIds = input.objectTypeIds ? new Set(input.objectTypeIds) : null
+      const errorCodes = input.errorCodes ? new Set(input.errorCodes) : null
       const filtered = [...this.rows.values()]
         .filter((record) => record.projectId === input.projectId)
         .filter((record) =>
@@ -314,6 +317,9 @@ export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
             : true
         )
         .filter((record) => (statuses ? statuses.has(record.status) : true))
+        .filter((record) =>
+          errorCodes ? record.error !== undefined && errorCodes.has(record.error.code) : true
+        )
         .filter((record) =>
           matchesRunListDateFilters(record, {
             statuses,

package/src/storage/projection-runs/types.ts

@@ -241,6 +241,8 @@ export interface ListProjectionRunsInput {
    */
   readonly objectTypeIds?: readonly string[]
   readonly statuses?: readonly ProjectionRunStatus[]
+  /** Runs whose recorded `error` has one of these codes; an empty set matches no runs. */
+  readonly errorCodes?: readonly ProjectionRunFailureCode[]
   readonly startedAfter?: Date
   readonly startedBefore?: Date
   readonly limit?: number

package/src/storage/types.ts

@@ -6,6 +6,7 @@ import type { AiUsageStorage } from "./ai-usage"
 import type { AuthStorage } from "./auth"
 import type { ConnectorConnectionStorage } from "./connector-connections"
 import type { ExecutionStorage } from "./executions"
+import type { FileDownloadGrantStorage } from "./file-download-grants"
 import type { FileUploadSessionStore } from "./file-upload-sessions"
 import type { ObjectStorage } from "./objects/types"
 import type { OntologyStorage } from "./ontology"
@@ -370,6 +371,7 @@ export interface Storage {
   shareGrants?: ShareGrantStorage
   shareSessions?: ShareSessionStorage
   fileUploadSessions?: FileUploadSessionStore
+  fileDownloadGrants?: FileDownloadGrantStorage
   connectorConnections?: ConnectorConnectionStorage
 
   /** Lightweight reachability probe. It must not open a write transaction or run migrations. */

package/src/testing/ai-cost-storage-contract.ts

@@ -250,6 +250,7 @@ export function runAiCostStorageContractSuite<TStorage extends AiCostStorage>(
               cacheWriteInputTokens: 0,
               textOutputTokens: 0,
               reasoningOutputTokens: 0,
+              audioDurationMs: 0,
             },
           },
           costs: {