@sixb/core 0.1.14 → 0.1.15

package/src/storage/ontology/in-memory/sources.ts

@@ -16,8 +16,11 @@ import {
   sourceStageRoots,
 } from "../provider"
 import type {
+  AbandonRunSourceMaterializationInput,
   AbandonSourceMaterializationCandidateInput,
   AbandonSourceMaterializationInput,
+  AdoptedSourceMaterialization,
+  AdoptSourceMaterializationInput,
   AssertSourceMaterializationExecution,
   BeginSourceMaterializationInput,
   CleanupTerminalSourceMaterializationsInput,
@@ -26,7 +29,7 @@ import type {
   MarkSourceMaterializationReadyInput,
   OntologySourceRecord,
   OntologySourceStorage,
-  ReclaimSourceMaterializationInput,
+  PurgeAbandonedSourceMaterializationsInput,
   StageSourceAssertion,
   StageSourceRowsInput,
   StageSourceRowsResult,
@@ -38,6 +41,7 @@ import {
   assertNonblank,
   type InMemoryOntologyState,
   type InMemorySourceMaterialization,
+  type InMemorySourceRoot,
   insertBounded,
   sourceMaterializationKey,
   sourceMaterializationRecord,
@@ -78,7 +82,7 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
       )
       if (nonterminal.length > 0) {
         throw sourceConflict(
-          `Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; reclaim it before beginning another.`
+          `Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`
         )
       }
 
@@ -191,8 +195,42 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
     })
   }
 
+  async adopt(
+    input: AdoptSourceMaterializationInput
+  ): Promise<AdoptedSourceMaterialization | null> {
+    return this.runRootOperation(async () => {
+      assertProjectAndSource(input)
+      assertSourceExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken)
+      assertCanonicalTimestamp(input.adoptedAt, "Source adoptedAt")
+      await this.assertCurrentExecution(input)
+      const [candidate, ...others] = this.findRunCandidates(
+        input.projectId,
+        input.source.projectionId,
+        input.execution.projectionRunId
+      )
+      if (!candidate) return null
+      if (others.length > 0) throw multipleRunCandidates(input.execution.projectionRunId)
+      const [key, materialization] = candidate
+      const adopted: InMemorySourceMaterialization = {
+        ...materialization,
+        executionToken: input.execution.executionToken,
+        // Workers' clocks differ; adoption never moves the record back in time.
+        updatedAt:
+          input.adoptedAt > materialization.updatedAt ? input.adoptedAt : materialization.updatedAt,
+      }
+      this.state.sourceMaterializations.set(key, adopted)
+      return {
+        record: sourceMaterializationRecord(adopted),
+        resumeStagingOrdinal:
+          adopted.status === "ready"
+            ? (adopted.rootCount ?? 0)
+            : Math.max(0, ...adopted.ordinalRoots.keys()),
+      }
+    })
+  }
+
   async abandon(input: AbandonSourceMaterializationCandidateInput): Promise<OntologySourceRecord>
-  async abandon(input: ReclaimSourceMaterializationInput): Promise<OntologySourceRecord | null>
+  async abandon(input: AbandonRunSourceMaterializationInput): Promise<OntologySourceRecord | null>
   async abandon(input: AbandonSourceMaterializationInput): Promise<OntologySourceRecord | null> {
     return this.runRootOperation(async () => {
       assertProjectAndSource(input)
@@ -201,7 +239,7 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
       await this.assertCurrentExecution(input)
       return input.kind === "candidate"
         ? this.abandonCandidate(input)
-        : this.abandonForReclaim(input)
+        : this.abandonRunCandidate(input)
     })
   }
 
@@ -211,18 +249,14 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
     return this.runRootOperation(() => {
       assertNonblank(input.projectId, "Terminal source cleanup project id")
       assertCanonicalTimestamp(input.terminalBefore, "Terminal source cleanup cutoff")
-      if (!Number.isSafeInteger(input.limit) || input.limit <= 0) {
-        throw new MaterializationValidationError(
-          "Terminal source cleanup limit must be a positive safe integer."
-        )
-      }
+      assertCleanupLimit(input.limit)
 
       const candidates: [string, InMemorySourceMaterialization][] = []
       for (const entry of this.state.sourceMaterializations.entries()) {
         const materialization = entry[1]
         if (
           materialization.projectId !== input.projectId ||
-          (materialization.status !== "superseded" && materialization.status !== "abandoned") ||
+          materialization.status !== "superseded" ||
           materialization.terminalAt === null ||
           materialization.terminalAt >= input.terminalBefore
         ) {
@@ -239,42 +273,69 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
         insertBounded(candidates, entry, input.limit, compareTerminalMaterializations)
       }
 
-      let remaining = input.limit
-      let rowsDeleted = 0
-      let materializationsDeleted = 0
-      for (const [key, candidate] of candidates) {
+      return this.deleteRetiredRoots(
+        candidates,
+        input.limit,
+        (root) => !root.active && root.retiredAt !== null && root.retiredAt < input.terminalBefore
+      )
+    })
+  }
+
+  async purgeAbandoned(
+    input: PurgeAbandonedSourceMaterializationsInput
+  ): Promise<CleanupTerminalSourceMaterializationsResult> {
+    return this.runRootOperation(() => {
+      assertNonblank(input.projectId, "Abandoned source purge project id")
+      assertCleanupLimit(input.limit)
+      const abandoned = [...this.state.sourceMaterializations.entries()]
+        .filter(
+          ([, materialization]) =>
+            materialization.projectId === input.projectId && materialization.status === "abandoned"
+        )
+        .sort(compareTerminalMaterializations)
+      return this.deleteRetiredRoots(abandoned, input.limit, () => true)
+    })
+  }
+
+  private deleteRetiredRoots(
+    candidates: readonly [string, InMemorySourceMaterialization][],
+    limit: number,
+    retired: (root: InMemorySourceRoot) => boolean
+  ): CleanupTerminalSourceMaterializationsResult {
+    let remaining = limit
+    let rowsDeleted = 0
+    let materializationsDeleted = 0
+    for (const [key, candidate] of candidates) {
+      if (remaining === 0) break
+      const current = this.state.sourceMaterializations.get(key)
+      if (current !== candidate) continue
+      current.rootOrdinals.clear()
+      current.ordinalRoots.clear()
+      for (const [rootKey, root] of current.roots) {
         if (remaining === 0) break
-        const current = this.state.sourceMaterializations.get(key)
-        if (current !== candidate) continue
-        current.rootOrdinals.clear()
-        current.ordinalRoots.clear()
-        for (const [rootKey, root] of current.roots) {
+        if (!retired(root)) continue
+        for (const entityKey of root.entityKeys) {
           if (remaining === 0) break
-          if (root.active || root.retiredAt === null || root.retiredAt >= input.terminalBefore)
-            continue
-          for (const entityKey of root.entityKeys) {
-            if (remaining === 0) break
-            current.rowsByEntity.delete(entityKey)
-            root.entityKeys.delete(entityKey)
-            rowsDeleted += 1
-            remaining -= 1
-          }
-          if (root.entityKeys.size === 0 && remaining > 0) {
-            current.roots.delete(rootKey)
-            current.rootOrdinals.delete(rootKey)
-            current.ordinalRoots.delete(root.stagingOrdinal)
-            rowsDeleted += 1
-            remaining -= 1
-          }
+          current.rowsByEntity.delete(entityKey)
+          root.entityKeys.delete(entityKey)
+          rowsDeleted += 1
+          remaining -= 1
         }
-        if (current.roots.size === 0 && current.rowsByEntity.size === 0 && remaining > 0) {
-          this.state.sourceMaterializations.delete(key)
-          materializationsDeleted += 1
+        if (root.entityKeys.size === 0 && remaining > 0) {
+          current.roots.delete(rootKey)
+          current.rootOrdinals.delete(rootKey)
+          current.ordinalRoots.delete(root.stagingOrdinal)
+          rowsDeleted += 1
           remaining -= 1
         }
       }
-      return { rowsDeleted, materializationsDeleted }
-    })
+      if (current.roots.size === 0 && current.rowsByEntity.size === 0 && remaining > 0) {
+        this.state.sourceMaterializations.delete(key)
+        materializationsDeleted += 1
+        remaining -= 1
+      }
+    }
+    return { rowsDeleted, materializationsDeleted }
   }
 
   async summarizeTerminal(
@@ -356,26 +417,15 @@ export class InMemoryOntologySourceStorage implements OntologySourceStorage {
     return this.transitionToAbandoned(materialization, input.abandonedAt)
   }
 
-  private abandonForReclaim(input: ReclaimSourceMaterializationInput): OntologySourceRecord | null {
+  private abandonRunCandidate(
+    input: AbandonRunSourceMaterializationInput
+  ): OntologySourceRecord | null {
     const candidates = this.findRunCandidates(
       input.projectId,
       input.source.projectionId,
       input.execution.projectionRunId
     )
-    if (
-      candidates.some(
-        ([, materialization]) => materialization.executionToken === input.execution.executionToken
-      )
-    ) {
-      throw sourceConflict(
-        `Projection run '${input.execution.projectionRunId}' already has a source materialization owned by the current execution.`
-      )
-    }
-    if (candidates.length > 1) {
-      throw sourceConflict(
-        `Projection run '${input.execution.projectionRunId}' has multiple nonterminal source materializations.`
-      )
-    }
+    if (candidates.length > 1) throw multipleRunCandidates(input.execution.projectionRunId)
     const candidate = candidates[0]?.[1]
     return candidate ? this.transitionToAbandoned(candidate, input.abandonedAt) : null
   }
@@ -609,6 +659,20 @@ function assertNotBefore(
   }
 }
 
+function assertCleanupLimit(limit: number): void {
+  if (!Number.isSafeInteger(limit) || limit <= 0) {
+    throw new MaterializationValidationError(
+      "Terminal source cleanup limit must be a positive safe integer."
+    )
+  }
+}
+
+function multipleRunCandidates(projectionRunId: string) {
+  return sourceConflict(
+    `Projection run '${projectionRunId}' has multiple nonterminal source materializations.`
+  )
+}
+
 function compareTerminalMaterializations(
   [, left]: readonly [string, InMemorySourceMaterialization],
   [, right]: readonly [string, InMemorySourceMaterialization]

package/src/storage/ontology/index.ts

@@ -97,8 +97,11 @@ export type {
 } from "./outbox"
 export { ONTOLOGY_OUTBOX_FAILURE_CODES } from "./outbox"
 export type {
+  AbandonRunSourceMaterializationInput,
   AbandonSourceMaterializationCandidateInput,
   AbandonSourceMaterializationInput,
+  AdoptedSourceMaterialization,
+  AdoptSourceMaterializationInput,
   AssertSourceMaterializationExecution,
   AssertSourceMaterializationExecutionInput,
   BeginSourceMaterializationInput,
@@ -109,7 +112,7 @@ export type {
   OntologySourceMaterializationStatus,
   OntologySourceRecord,
   OntologySourceStorage,
-  ReclaimSourceMaterializationInput,
+  PurgeAbandonedSourceMaterializationsInput,
   StageSourceAssertion,
   StageSourceRoot,
   StageSourceRowsInput,

package/src/storage/ontology/provider.ts

@@ -11,6 +11,8 @@ import type {
   ExpectedObjectRevision,
   OntologyLinkRef,
   OntologyObjectRef,
+  ProjectionEntityRef,
+  ProjectionSourceAssertion,
 } from "../../materialization/model"
 import { linkRefKey, objectRefKey, projectionEntityKey } from "../../materialization/refs"
 import type { OntologyCommitOriginSelector, OntologyCommitWrite } from "./commits"
@@ -716,7 +718,6 @@ export function sourceStageRow(row: StageSourceAssertion): SourceStageRow {
 export interface SourceStageRoot {
   readonly root: StageSourceAssertion["root"]
   readonly rootKey: string
-  readonly sortKey: string
   readonly stagingOrdinal: number
   readonly deleted: boolean
 }
@@ -763,7 +764,6 @@ export function sourceStageRoots(
       roots.set(key, {
         root: row.root,
         rootKey: key,
-        sortKey: utf8SortKey(key),
         stagingOrdinal: row.stagingOrdinal,
         deleted,
       })
@@ -965,12 +965,69 @@ export function sourceEntityColumns(entity: StageSourceAssertion["root"]): {
   }
 }
 
-export function utf8SortKey(canonicalKey: string): string {
-  let key = ""
-  for (const byte of new TextEncoder().encode(canonicalKey)) {
-    key += byte.toString(16).padStart(2, "0")
+/** The inverse of `projectionEntityKey`, for providers that store only the canonical key. */
+export function sourceEntityFromKey(key: string): ProjectionEntityRef {
+  const parts: unknown = JSON.parse(key)
+  if (Array.isArray(parts) && parts.every((part) => typeof part === "string")) {
+    if (parts[0] === "object" && parts.length === 3) {
+      return { kind: "object", ref: { objectTypeId: parts[1]!, primaryId: parts[2]! } }
+    }
+    if (parts[0] === "link" && parts.length === 6) {
+      return {
+        kind: "link",
+        ref: {
+          source: { objectTypeId: parts[1]!, primaryId: parts[2]! },
+          linkId: parts[3]!,
+          target: { objectTypeId: parts[4]!, primaryId: parts[5]! },
+        },
+      }
+    }
   }
-  return key
+  throw new MaterializationConflictError(
+    "source-materialization",
+    `Stored source key ${key} is not a canonical entity key.`
+  )
+}
+
+/** What an assertion carries beyond its identity, or null when that is nothing. */
+export function sourceAssertionPayload(
+  assertion: ProjectionSourceAssertion
+): Readonly<Record<string, unknown>> | null {
+  const { kind: _kind, ref: _ref, ...payload } = assertion
+  return Object.keys(payload).length === 0 ? null : payload
+}
+
+/** Rebuilds an assertion from its typed identity columns and its payload. */
+export function sourceAssertionFromColumns(
+  columns: {
+    readonly entity_kind: "object" | "link"
+    readonly object_type_id: string | null
+    readonly primary_id: string | null
+    readonly source_type_id: string | null
+    readonly source_primary_id: string | null
+    readonly link_id: string | null
+    readonly target_type_id: string | null
+    readonly target_primary_id: string | null
+  },
+  payload: unknown
+): ProjectionSourceAssertion {
+  const rest = (payload ?? {}) as Record<string, unknown>
+  if (columns.entity_kind === "object") {
+    return {
+      kind: "object",
+      ref: { objectTypeId: columns.object_type_id!, primaryId: columns.primary_id! },
+      ...rest,
+    } as ProjectionSourceAssertion
+  }
+  return {
+    kind: "link",
+    ref: {
+      source: { objectTypeId: columns.source_type_id!, primaryId: columns.source_primary_id! },
+      linkId: columns.link_id!,
+      target: { objectTypeId: columns.target_type_id!, primaryId: columns.target_primary_id! },
+    },
+    ...rest,
+  } as ProjectionSourceAssertion
 }
 
 export function isExactStagingManifest(

package/src/storage/ontology/sources.ts

@@ -128,7 +128,7 @@ interface BaseAbandonSourceMaterializationInput {
   readonly abandonedAt: string
 }
 
-/** Abandon the current execution's exact candidate after ingress or semantic failure. */
+/** Abandon the current execution's exact candidate, e.g. an adopted one whose delta base moved. */
 export interface AbandonSourceMaterializationCandidateInput
   extends BaseAbandonSourceMaterializationInput {
   readonly kind: "candidate"
@@ -136,20 +136,38 @@ export interface AbandonSourceMaterializationCandidateInput
 }
 
 /**
- * Reclaim a logical run after redelivery. The current execution may abandon the prior execution's
- * staging/ready candidate, but never one already owned by its own token.
+ * Release whatever candidate a run still holds, whichever execution staged it. Called in the same
+ * transaction as the run's terminal transition, so no candidate outlives its run.
  */
-export interface ReclaimSourceMaterializationInput extends BaseAbandonSourceMaterializationInput {
-  readonly kind: "reclaim"
+export interface AbandonRunSourceMaterializationInput
+  extends BaseAbandonSourceMaterializationInput {
+  readonly kind: "run"
 }
 
 export type AbandonSourceMaterializationInput =
   | AbandonSourceMaterializationCandidateInput
-  | ReclaimSourceMaterializationInput
+  | AbandonRunSourceMaterializationInput
+
+export interface AdoptSourceMaterializationInput {
+  readonly projectId: string
+  readonly source: ProjectionSourceRef
+  readonly execution: ProjectionExecution
+  readonly adoptedAt: string
+}
+
+export interface AdoptedSourceMaterialization {
+  readonly record: OntologySourceRecord
+  /**
+   * Where staging resumes. Every root before this ordinal is complete; the root at it may be
+   * partial and is staged again, which staging accepts for identical rows. Equals the root count
+   * once the candidate is ready.
+   */
+  readonly resumeStagingOrdinal: number
+}
 
 export interface CleanupTerminalSourceMaterializationsInput {
   readonly projectId: string
-  /** Exclusive cutoff for retired roots in terminal manifests, and for empty terminal manifests. */
+  /** Exclusive cutoff for retired roots in superseded manifests, and for empty ones. */
   readonly terminalBefore: string
   /** Maximum total assertion, root-reference, and manifest deletions performed by one call. */
   readonly limit: number
@@ -161,6 +179,12 @@ export interface CleanupTerminalSourceMaterializationsResult {
   readonly materializationsDeleted: number
 }
 
+export interface PurgeAbandonedSourceMaterializationsInput {
+  readonly projectId: string
+  /** Maximum total assertion, root-reference, and manifest deletions performed by one call. */
+  readonly limit: number
+}
+
 export interface SummarizeTerminalSourceMaterializationsInput {
   readonly projectId: string
 }
@@ -195,11 +219,21 @@ export interface OntologySourceStorage {
   stageRows(input: StageSourceRowsInput): Promise<StageSourceRowsResult>
   markReady(input: MarkSourceMaterializationReadyInput): Promise<OntologySourceRecord>
   getActive(input: GetActiveOntologySourceInput): Promise<OntologySourceRecord | null>
+  /** Hands a redelivered run's candidate to its current execution; null when it has none. */
+  adopt(input: AdoptSourceMaterializationInput): Promise<AdoptedSourceMaterialization | null>
   abandon(input: AbandonSourceMaterializationCandidateInput): Promise<OntologySourceRecord>
-  abandon(input: ReclaimSourceMaterializationInput): Promise<OntologySourceRecord | null>
+  abandon(input: AbandonRunSourceMaterializationInput): Promise<OntologySourceRecord | null>
+  /** Retired roots of superseded manifests, after their retention. */
   cleanupTerminal(
     input: CleanupTerminalSourceMaterializationsInput
   ): Promise<CleanupTerminalSourceMaterializationsResult>
+  /**
+   * Abandoned candidates, whole and oldest first. Nothing reads an abandoned candidate, so it has
+   * no retention; deleting one manifest at a time keeps each delete on the source-version index.
+   */
+  purgeAbandoned(
+    input: PurgeAbandonedSourceMaterializationsInput
+  ): Promise<CleanupTerminalSourceMaterializationsResult>
   summarizeTerminal(
     input: SummarizeTerminalSourceMaterializationsInput
   ): Promise<TerminalSourceMaterializationSummary>

package/src/storage/projection-runs/in-memory.ts

@@ -26,8 +26,10 @@ import {
   finishProjectionRunRecord,
   immutableDatasetVersionConflict,
   mergeProjectionRunProgress,
+  parseProjectionAttemptFailure,
   planProjectionRunReclaim,
   projectionRunNotFound,
+  projectionRunSupersedes,
   publicProjectionRunRecord,
   requireTelemetryProjectionRun,
   type StoredProjectionRunRecord,
@@ -36,6 +38,7 @@ import {
 import {
   type AdvanceProjectionTelemetryCheckpointInput,
   type FailProjectionRunEnqueueInput,
+  type FindSupersedingProjectionRunInput,
   type FinishProjectionRunInput,
   type ListLatestProjectionRunsInput,
   type ListLatestProjectionRunsResult,
@@ -47,6 +50,7 @@ import {
   type ProjectionRunStorage,
   projectionRunObjectTypesVisible,
   type QueueProjectionRunInput,
+  type RecordProjectionAttemptFailureInput,
   type RecordProjectionMissingTargetInput,
   type StartOrReclaimProjectionRunInput,
   type TelemetryProjectionRunRecord,
@@ -60,6 +64,10 @@ function projectionRunKey(projectId: string, id: string): string {
   return JSON.stringify([projectId, id])
 }
 
+function compareText(left: string, right: string): number {
+  return left < right ? -1 : left > right ? 1 : 0
+}
+
 function compareRuns(a: ProjectionRunRecord, b: ProjectionRunRecord, order: "asc" | "desc") {
   return compareStartedAt(a, b, order)
 }
@@ -152,7 +160,6 @@ export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
         executionToken,
         startedAt: existing.startedAt ?? new Date(input.startedAt ?? new Date()),
         finishedAt: undefined,
-        error: undefined,
       }
 
       this.rows.set(key, structuredClone(record))
@@ -246,6 +253,17 @@ export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
     })
   }
 
+  async recordAttemptFailure(
+    input: RecordProjectionAttemptFailureInput
+  ): Promise<ProjectionRunRecord> {
+    return this.runRootOperation(() => {
+      const existing = this.requireExecution(input)
+      const next = { ...existing, error: parseProjectionAttemptFailure(existing, input.error) }
+      this.rows.set(projectionRunKey(input.projectId, input.id), structuredClone(next))
+      return publicProjectionRunRecord(next)
+    })
+  }
+
   async finish(input: FinishProjectionRunInput): Promise<ProjectionRunRecord> {
     return this.runRootOperation(() => {
       const existing = this.requireExecution(input)
@@ -311,12 +329,33 @@ export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
     })
   }
 
+  async findSupersedingRun(
+    input: FindSupersedingProjectionRunInput
+  ): Promise<ProjectionRunRecord | null> {
+    return this.runRootOperation(() => {
+      const run = this.rows.get(projectionRunKey(input.projectId, input.id))
+      if (!run) throw projectionRunNotFound(input.projectId, input.id)
+      const newest = [...this.rows.values()]
+        .filter((candidate) => projectionRunSupersedes(candidate, run))
+        .sort(
+          (left, right) =>
+            compareText(
+              right.identity.datasetVersion.createdAt,
+              left.identity.datasetVersion.createdAt
+            ) || compareText(right.id, left.id)
+        )[0]
+      return newest ? publicProjectionRunRecord(newest) : null
+    })
+  }
+
   async listLatestByProjectionIds(
     input: ListLatestProjectionRunsInput
   ): Promise<ListLatestProjectionRunsResult> {
     return this.runRootOperation(() => {
       const runs = latestStartedAtByOwnerId(
-        [...this.rows.values()].filter((record) => record.projectId === input.projectId),
+        [...this.rows.values()].filter(
+          (record) => record.projectId === input.projectId && record.status !== "superseded"
+        ),
         input.projectionIds,
         (record) => record.identity.projectionId
       )

package/src/storage/projection-runs/index.ts

@@ -3,6 +3,7 @@ export { InMemoryProjectionRunStorage } from "./in-memory"
 export type {
   AdvanceProjectionTelemetryCheckpointInput,
   FailProjectionRunEnqueueInput,
+  FindSupersedingProjectionRunInput,
   FinishProjectionRunInput,
   LinkProjectionRunRecord,
   LinkProjectionTarget,
@@ -27,6 +28,7 @@ export type {
   ProjectionTarget,
   ProjectionTelemetryCheckpoint,
   QueueProjectionRunInput,
+  RecordProjectionAttemptFailureInput,
   RecordProjectionMissingTargetInput,
   StartOrReclaimProjectionRunInput,
   TelemetryProjectionRunRecord,