@sixb/core 0.1.1 → 0.1.3

package/src/storage/ontology/provider.ts

@@ -9,7 +9,7 @@ import type {
   OntologyLinkRef,
   OntologyObjectRef,
 } from "../../materialization/model"
-import { linkRefSortKey, objectRefSortKey, projectionEntityKey } from "../../materialization/refs"
+import { projectionEntityKey } from "../../materialization/refs"
 import type { OntologyCommitOriginSelector, OntologyCommitWrite } from "./commits"
 import type {
   FinalizeMaterializationInput,
@@ -22,6 +22,7 @@ import type {
   MaterializationWorkRecord,
   SourceActivationWrite,
   StageMaterializationWorkInput,
+  StoredLinkSlotOverride,
   StreamMaterializationWorkInput,
 } from "./materializations"
 import { assertNonblank, assertTimestamp } from "./provider-validation"
@@ -61,7 +62,6 @@ export {
 } from "./provider-work"
 
 export interface LinkScopeAccumulator {
-  readonly scopeSortKey: string
   readonly source: OntologyObjectRef
   readonly linkId: string
   readonly hash: ReturnType<typeof createHash>
@@ -70,13 +70,11 @@ export interface LinkScopeAccumulator {
 
 export function startScopeAccumulator(
   source: OntologyObjectRef,
-  linkId: string,
-  scopeSortKey: string
+  linkId: string
 ): LinkScopeAccumulator {
   const hash = createHash("sha256")
   hash.update("[")
   return {
-    scopeSortKey,
     source: structuredClone(source),
     linkId,
     hash,
@@ -101,7 +99,7 @@ export function appendScopeSnapshot(
 
 export function finishScopeAccumulator(
   accumulator: LinkScopeAccumulator
-): MaterializationLinkScopeState {
+): Pick<MaterializationLinkScopeState, "source" | "linkId" | "effectiveCount" | "fingerprint"> {
   accumulator.hash.update("]")
   return {
     source: accumulator.source,
@@ -111,6 +109,60 @@ export function finishScopeAccumulator(
   }
 }
 
+export function linkSlotOverrideValue(
+  value: unknown,
+  scopeLabel: string
+): StoredLinkSlotOverride["value"] {
+  if (!isRecord(value) || typeof value.kind !== "string") {
+    throw invalidLinkSlotOverride(scopeLabel)
+  }
+  if (value.kind === "legacy-conflict") {
+    return { kind: "legacy-conflict" }
+  }
+  if (
+    (value.kind !== "set" && value.kind !== "clear") ||
+    !isRecord(value.target) ||
+    typeof value.target.objectTypeId !== "string" ||
+    value.target.objectTypeId.trim().length === 0 ||
+    typeof value.target.primaryId !== "string" ||
+    value.target.primaryId.trim().length === 0
+  ) {
+    throw invalidLinkSlotOverride(scopeLabel)
+  }
+  const target = {
+    objectTypeId: value.target.objectTypeId,
+    primaryId: value.target.primaryId,
+  }
+  if (value.kind === "clear") return { kind: "clear", target }
+  if (value.properties === undefined) return { kind: "set", target }
+  if (!isJsonObject(value.properties)) throw invalidLinkSlotOverride(scopeLabel)
+  return { kind: "set", target, properties: structuredClone(value.properties) }
+}
+
+function invalidLinkSlotOverride(scopeLabel: string): MaterializationConflictError {
+  return new MaterializationConflictError(
+    "effective-state",
+    `Cardinality-one link slot ${scopeLabel} contains an invalid stored override.`
+  )
+}
+
+function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+}
+
+function isJsonObject(
+  value: unknown
+): value is Readonly<Record<string, import("../../json").JsonValue>> {
+  return isRecord(value) && Object.values(value).every(isJsonValue)
+}
+
+function isJsonValue(value: unknown): value is import("../../json").JsonValue {
+  if (value === null || typeof value === "string" || typeof value === "boolean") return true
+  if (typeof value === "number") return Number.isFinite(value)
+  if (Array.isArray(value)) return value.every(isJsonValue)
+  return isRecord(value) && Object.values(value).every(isJsonValue)
+}
+
 export interface ProviderReplacementSessionState {
   readonly sourceId: string
   readonly candidateMaterializationId: string
@@ -417,44 +469,6 @@ export function uniqueSorted<T>(
   )
 }
 
-export type OverrideEntity =
-  | { readonly kind: "object"; readonly ref: OntologyObjectRef }
-  | { readonly kind: "link"; readonly ref: OntologyLinkRef }
-
-export function overrideEntityColumns(entity: OverrideEntity): {
-  readonly sortKey: string
-  readonly objectTypeId: string | null
-  readonly primaryId: string | null
-  readonly sourceTypeId: string | null
-  readonly sourcePrimaryId: string | null
-  readonly linkId: string | null
-  readonly targetTypeId: string | null
-  readonly targetPrimaryId: string | null
-} {
-  if (entity.kind === "object") {
-    return {
-      sortKey: objectRefSortKey(entity.ref),
-      objectTypeId: entity.ref.objectTypeId,
-      primaryId: entity.ref.primaryId,
-      sourceTypeId: null,
-      sourcePrimaryId: null,
-      linkId: null,
-      targetTypeId: null,
-      targetPrimaryId: null,
-    }
-  }
-  return {
-    sortKey: linkRefSortKey(entity.ref),
-    objectTypeId: null,
-    primaryId: null,
-    sourceTypeId: entity.ref.source.objectTypeId,
-    sourcePrimaryId: entity.ref.source.primaryId,
-    linkId: entity.ref.linkId,
-    targetTypeId: entity.ref.target.objectTypeId,
-    targetPrimaryId: entity.ref.target.primaryId,
-  }
-}
-
 export function sameNonnegativeCounts(actual: object, expected: object): boolean {
   const actualCounts = actual as Record<string, unknown>
   return Object.entries(expected).every(
@@ -489,7 +503,7 @@ export function materializationWorkColumns(record: MaterializationWorkRecord): {
   if (record.kind === "cardinality") {
     return {
       lane: "cardinality",
-      majorOrder: 0,
+      majorOrder: record.view === "effective" ? 0 : 1,
       minorOrder: 0,
       sortOne: record.scopeSortKey,
       sortTwo: record.linkSortKey,
@@ -517,23 +531,29 @@ function materializationPlanKindRank(kind: MaterializationPlanWorkItem["kind"]):
       return 2
     case "link-override-delete":
       return 3
-    case "point-upsert":
+    case "link-slot-override-upsert":
       return 4
-    case "link-delete":
+    case "link-slot-override-delete":
       return 5
-    case "object-delete":
+    case "point-upsert":
       return 6
-    case "object-upsert":
+    case "link-delete":
       return 7
-    case "link-upsert":
+    case "object-delete":
       return 8
+    case "object-upsert":
+      return 9
+    case "link-upsert":
+      return 10
   }
 }
 
-export function duplicateMaterializationWork(key: string): MaterializationConflictError {
+export function duplicateMaterializationWork(key?: string): MaterializationConflictError {
   return new MaterializationConflictError(
     "effective-state",
-    `Duplicate materialization work key '${key}'.`
+    key === undefined
+      ? "Duplicate materialization work."
+      : `Duplicate materialization work key '${key}'.`
   )
 }
 

package/src/storage/operation-scope.ts

@@ -215,6 +215,7 @@ export function createAgentOperationScope<T extends AgentStorage>(
     threads: createOperationScopedFacade(target.threads, scope),
     runs: createOperationScopedFacade(target.runs, scope),
     messages: createOperationScopedFacade(target.messages, scope),
+    checkpoints: createOperationScopedFacade(target.checkpoints, scope),
   }) as T
 }
 
@@ -236,6 +237,7 @@ export function createWorkflowRunOperationScope<T extends WorkflowRunStorage>(
 ): T {
   return createOperationScopedFacade<WorkflowRunStorage>(target, scope, {
     nodes: createOperationScopedFacade(target.nodes, scope),
+    agentNodes: createOperationScopedFacade(target.agentNodes, scope),
   }) as T
 }
 

package/src/storage/pipeline-runs/idempotency.ts

@@ -0,0 +1,14 @@
+import type { PipelineRunRecord, QueuePipelineRunInput } from "./types"
+
+export function canRequeuePipelineRunAfterEnqueueFailure(
+  existing: PipelineRunRecord,
+  input: QueuePipelineRunInput
+): boolean {
+  return (
+    existing.status === "failed" &&
+    existing.error?.code === "queue.enqueue_failed" &&
+    existing.error.retryable &&
+    existing.executionId === input.executionId &&
+    existing.pipelineId === input.pipelineId
+  )
+}

package/src/storage/pipeline-runs/in-memory.ts

@@ -1,3 +1,6 @@
+import { parseSixbFailure } from "../../errors/internal"
+import type { SixbFailure } from "../../errors/types"
+import type { ExecutionStorage } from "../executions"
 import {
   cloneRecord,
   compareStartedAt,
@@ -9,6 +12,8 @@ import {
   toStatusSet,
 } from "../run-listing"
 import { PipelineRunError } from "./errors"
+import { canRequeuePipelineRunAfterEnqueueFailure } from "./idempotency"
+import { assertPipelineRunExecution } from "./provider"
 import type {
   FinishPipelineRunInput,
   FinishPipelineStepRunInput,
@@ -18,16 +23,20 @@ import type {
   ListPipelineRunsResult,
   ListPipelineStepRunsInput,
   ListPipelineStepRunsResult,
-  PipelineRunFailure,
+  PipelineRunFailureCode,
   PipelineRunRecord,
   PipelineRunStorage,
   PipelineStepRunRecord,
+  QueuePipelineRunInput,
   StartPipelineRunInput,
   StartPipelineStepRunInput,
 } from "./types"
+import { PIPELINE_RUN_FAILURE_CODES } from "./types"
 
-function normalizeError(error: PipelineRunFailure | undefined): PipelineRunFailure | undefined {
-  return error ? cloneRecord(error) : undefined
+function normalizeError(
+  error: SixbFailure<PipelineRunFailureCode> | undefined
+): SixbFailure<PipelineRunFailureCode> | undefined {
+  return error ? parseSixbFailure(error, PIPELINE_RUN_FAILURE_CODES) : undefined
 }
 
 function assertNonNegativeInteger(value: number | undefined, fieldName: string): void {
@@ -40,6 +49,8 @@ export class InMemoryPipelineRunStorage implements PipelineRunStorage {
   private readonly runs = new Map<string, PipelineRunRecord>()
   private readonly steps = new Map<string, PipelineStepRunRecord>()
 
+  constructor(private readonly executions: ExecutionStorage) {}
+
   snapshot(): InMemoryPipelineRunStorageSnapshot {
     return {
       runs: structuredClone(this.runs),
@@ -59,28 +70,97 @@ export class InMemoryPipelineRunStorage implements PipelineRunStorage {
     }
   }
 
-  async start(input: StartPipelineRunInput): Promise<PipelineRunRecord> {
+  async queue(input: QueuePipelineRunInput): Promise<PipelineRunRecord> {
     const key = storageKey(input.projectId, input.id)
-    if (this.runs.has(key)) {
+    const existing = this.runs.get(key)
+    if (existing && !canRequeuePipelineRunAfterEnqueueFailure(existing, input)) {
       throw new PipelineRunError(
         `[Sixb] Pipeline run '${input.id}' already exists for project '${input.projectId}'.`
       )
     }
+    if (
+      [...this.runs.values()].some(
+        (run) =>
+          run.projectId === input.projectId &&
+          run.id !== input.id &&
+          run.executionId === input.executionId
+      )
+    ) {
+      throw new PipelineRunError(
+        `[Sixb] Execution '${input.executionId}' already belongs to another Pipeline run.`
+      )
+    }
+    await assertPipelineRunExecution({
+      executions: this.executions,
+      projectId: input.projectId,
+      executionId: input.executionId,
+      runId: input.id,
+      pipelineId: input.pipelineId,
+    })
+
+    const queuedAt = new Date(input.queuedAt ?? new Date())
+    if (existing) {
+      const next: PipelineRunRecord = {
+        ...existing,
+        status: "queued",
+        queuedAt,
+        startedAt: undefined,
+        finishedAt: undefined,
+        output: undefined,
+        error: undefined,
+      }
+      this.runs.set(key, cloneRecord(next))
+      return cloneRecord(next)
+    }
 
     const record: PipelineRunRecord = {
       id: input.id,
       projectId: input.projectId,
+      executionId: input.executionId,
       pipelineId: input.pipelineId,
-      status: "running",
-      startedAt: new Date(input.startedAt ?? new Date()),
+      status: "queued",
+      queuedAt,
     }
 
     this.runs.set(key, cloneRecord(record))
     return cloneRecord(record)
   }
 
+  async start(input: StartPipelineRunInput): Promise<PipelineRunRecord> {
+    const key = storageKey(input.projectId, input.id)
+    const existing = this.runs.get(key)
+    if (!existing) {
+      throw new PipelineRunError(
+        `[Sixb] Pipeline run '${input.id}' not found for project '${input.projectId}'.`
+      )
+    }
+    if (existing.status !== "queued") {
+      throw new PipelineRunError(
+        `[Sixb] Pipeline run '${input.id}' cannot start from status '${existing.status}'.`
+      )
+    }
+
+    const next: PipelineRunRecord = {
+      ...existing,
+      status: "running",
+      startedAt: new Date(input.startedAt ?? new Date()),
+      error: undefined,
+    }
+    this.runs.set(key, cloneRecord(next))
+    return cloneRecord(next)
+  }
+
   async finish(input: FinishPipelineRunInput): Promise<PipelineRunRecord> {
-    const existing = this.requireRunningPipelineRun(input.projectId, input.id)
+    const existing = this.requireExistingPipelineRun(input.projectId, input.id)
+    const enqueueFailure = input.status === "failed" && input.error.code === "queue.enqueue_failed"
+    if (
+      (enqueueFailure && existing.status !== "queued") ||
+      (!enqueueFailure && existing.status !== "running")
+    ) {
+      throw new PipelineRunError(
+        `[Sixb] Pipeline run '${input.id}' cannot finish from status '${existing.status}'.`
+      )
+    }
     const base: PipelineRunRecord = {
       ...existing,
       status: input.status,

package/src/storage/pipeline-runs/index.ts

@@ -1,4 +1,5 @@
 export { PipelineRunError } from "./errors"
+export { canRequeuePipelineRunAfterEnqueueFailure } from "./idempotency"
 export { InMemoryPipelineRunStorage } from "./in-memory"
 export type {
   FinishPipelineRunInput,
@@ -9,11 +10,14 @@ export type {
   ListPipelineRunsResult,
   ListPipelineStepRunsInput,
   ListPipelineStepRunsResult,
-  PipelineRunFailure,
+  PipelineRunFailureCode,
   PipelineRunRecord,
   PipelineRunStatus,
   PipelineRunStorage,
   PipelineStepRunRecord,
+  PipelineStepRunStatus,
+  QueuePipelineRunInput,
   StartPipelineRunInput,
   StartPipelineStepRunInput,
 } from "./types"
+export { PIPELINE_RUN_FAILURE_CODES } from "./types"

package/src/storage/pipeline-runs/provider.ts

@@ -0,0 +1,25 @@
+import type { ExecutionStorage } from "../executions"
+import { findPrimitiveRunExecution } from "../executions/run-link"
+import { PipelineRunError } from "./errors"
+
+/** Validate the semantic link between a durable Pipeline run and its immutable execution. */
+export async function assertPipelineRunExecution(input: {
+  readonly executions: ExecutionStorage
+  readonly projectId: string
+  readonly executionId: string
+  readonly runId: string
+  readonly pipelineId: string
+}): Promise<void> {
+  const execution = await findPrimitiveRunExecution({
+    executions: input.executions,
+    projectId: input.projectId,
+    executionId: input.executionId,
+    primitive: { kind: "pipeline", id: input.pipelineId, runId: input.runId },
+    sourceTypes: ["execution", "schedule", "event"],
+  })
+  if (!execution) {
+    throw new PipelineRunError(
+      `[Sixb] Execution '${input.executionId}' does not authorize Pipeline run '${input.runId}'.`
+    )
+  }
+}

package/src/storage/pipeline-runs/types.ts

@@ -1,22 +1,30 @@
+import type { SixbErrorCode, SixbFailure } from "../../errors/types"
 import type { DatasetVersionRef, DatasetWriteMode } from "../../lake-storage"
 
-export type PipelineRunStatus = "running" | "succeeded" | "failed" | "cancelled"
+export type PipelineRunStatus = "queued" | "running" | "succeeded" | "failed" | "cancelled"
+export type PipelineStepRunStatus = Exclude<PipelineRunStatus, "queued">
+/** Error codes a pipeline or pipeline-step run can persist and expose. */
+export const PIPELINE_RUN_FAILURE_CODES = [
+  "internal.unexpected",
+  "runtime.cancelled",
+  "queue.enqueue_failed",
+  "pipeline.step_failed",
+] as const satisfies readonly [SixbErrorCode, ...SixbErrorCode[]]
 
-export interface PipelineRunFailure {
-  readonly name?: string
-  readonly message: string
-}
+export type PipelineRunFailureCode = (typeof PIPELINE_RUN_FAILURE_CODES)[number]
 
 export interface PipelineRunRecord {
   readonly id: string
   readonly projectId: string
+  readonly executionId: string
   readonly pipelineId: string
   readonly status: PipelineRunStatus
-  readonly startedAt: Date
+  readonly queuedAt: Date
+  readonly startedAt?: Date
   readonly finishedAt?: Date
   /** Final step output in sequential V1. */
   readonly output?: DatasetVersionRef
-  readonly error?: PipelineRunFailure
+  readonly error?: SixbFailure<PipelineRunFailureCode>
 }
 
 export interface PipelineStepRunRecord {
@@ -27,19 +35,26 @@ export interface PipelineStepRunRecord {
   readonly stepId: string
   readonly datasetId: string
   readonly mode: DatasetWriteMode
-  readonly status: PipelineRunStatus
+  readonly status: PipelineStepRunStatus
   readonly startedAt: Date
   readonly finishedAt?: Date
   readonly inputs: readonly DatasetVersionRef[]
   readonly output?: DatasetVersionRef
   readonly rowsWritten?: number
-  readonly error?: PipelineRunFailure
+  readonly error?: SixbFailure<PipelineRunFailureCode>
 }
 
-export interface StartPipelineRunInput {
+export interface QueuePipelineRunInput {
   readonly id: string
   readonly projectId: string
+  readonly executionId: string
   readonly pipelineId: string
+  readonly queuedAt?: Date
+}
+
+export interface StartPipelineRunInput {
+  readonly id: string
+  readonly projectId: string
   readonly startedAt?: Date
 }
 
@@ -54,9 +69,16 @@ export type FinishPipelineRunInput =
   | {
       readonly id: string
       readonly projectId: string
-      readonly status: "failed" | "cancelled"
+      readonly status: "failed"
+      readonly finishedAt?: Date
+      readonly error: SixbFailure<PipelineRunFailureCode>
+    }
+  | {
+      readonly id: string
+      readonly projectId: string
+      readonly status: "cancelled"
       readonly finishedAt?: Date
-      readonly error?: PipelineRunFailure
+      readonly error?: SixbFailure<PipelineRunFailureCode>
     }
 
 export interface StartPipelineStepRunInput {
@@ -86,7 +108,7 @@ export type FinishPipelineStepRunInput =
       readonly status: "failed" | "cancelled"
       readonly finishedAt?: Date
       readonly rowsWritten?: number
-      readonly error?: PipelineRunFailure
+      readonly error?: SixbFailure<PipelineRunFailureCode>
     }
 
 export interface ListPipelineRunsInput {
@@ -122,7 +144,7 @@ export interface ListPipelineStepRunsInput {
   readonly pipelineId?: string
   readonly stepId?: string
   readonly datasetId?: string
-  readonly statuses?: readonly PipelineRunStatus[]
+  readonly statuses?: readonly PipelineStepRunStatus[]
   readonly startedAfter?: Date
   readonly startedBefore?: Date
   readonly limit?: number
@@ -137,6 +159,7 @@ export interface ListPipelineStepRunsResult {
 }
 
 export interface PipelineRunStorage {
+  queue(input: QueuePipelineRunInput): Promise<PipelineRunRecord>
   start(input: StartPipelineRunInput): Promise<PipelineRunRecord>
   finish(input: FinishPipelineRunInput): Promise<PipelineRunRecord>
   startStep(input: StartPipelineStepRunInput): Promise<PipelineStepRunRecord>