@sixb/core 0.1.1 → 0.1.2

package/src/storage/pipeline-runs/in-memory.ts

@@ -1,3 +1,6 @@
+import { parseSixbFailure } from "../../errors/internal"
+import type { SixbFailure } from "../../errors/types"
+import type { ExecutionStorage } from "../executions"
 import {
   cloneRecord,
   compareStartedAt,
@@ -9,6 +12,8 @@ import {
   toStatusSet,
 } from "../run-listing"
 import { PipelineRunError } from "./errors"
+import { canRequeuePipelineRunAfterEnqueueFailure } from "./idempotency"
+import { assertPipelineRunExecution } from "./provider"
 import type {
   FinishPipelineRunInput,
   FinishPipelineStepRunInput,
@@ -18,16 +23,20 @@ import type {
   ListPipelineRunsResult,
   ListPipelineStepRunsInput,
   ListPipelineStepRunsResult,
-  PipelineRunFailure,
+  PipelineRunFailureCode,
   PipelineRunRecord,
   PipelineRunStorage,
   PipelineStepRunRecord,
+  QueuePipelineRunInput,
   StartPipelineRunInput,
   StartPipelineStepRunInput,
 } from "./types"
+import { PIPELINE_RUN_FAILURE_CODES } from "./types"
 
-function normalizeError(error: PipelineRunFailure | undefined): PipelineRunFailure | undefined {
-  return error ? cloneRecord(error) : undefined
+function normalizeError(
+  error: SixbFailure<PipelineRunFailureCode> | undefined
+): SixbFailure<PipelineRunFailureCode> | undefined {
+  return error ? parseSixbFailure(error, PIPELINE_RUN_FAILURE_CODES) : undefined
 }
 
 function assertNonNegativeInteger(value: number | undefined, fieldName: string): void {
@@ -40,6 +49,8 @@ export class InMemoryPipelineRunStorage implements PipelineRunStorage {
   private readonly runs = new Map<string, PipelineRunRecord>()
   private readonly steps = new Map<string, PipelineStepRunRecord>()
 
+  constructor(private readonly executions: ExecutionStorage) {}
+
   snapshot(): InMemoryPipelineRunStorageSnapshot {
     return {
       runs: structuredClone(this.runs),
@@ -59,28 +70,97 @@ export class InMemoryPipelineRunStorage implements PipelineRunStorage {
     }
   }
 
-  async start(input: StartPipelineRunInput): Promise<PipelineRunRecord> {
+  async queue(input: QueuePipelineRunInput): Promise<PipelineRunRecord> {
     const key = storageKey(input.projectId, input.id)
-    if (this.runs.has(key)) {
+    const existing = this.runs.get(key)
+    if (existing && !canRequeuePipelineRunAfterEnqueueFailure(existing, input)) {
       throw new PipelineRunError(
         `[Sixb] Pipeline run '${input.id}' already exists for project '${input.projectId}'.`
       )
     }
+    if (
+      [...this.runs.values()].some(
+        (run) =>
+          run.projectId === input.projectId &&
+          run.id !== input.id &&
+          run.executionId === input.executionId
+      )
+    ) {
+      throw new PipelineRunError(
+        `[Sixb] Execution '${input.executionId}' already belongs to another Pipeline run.`
+      )
+    }
+    await assertPipelineRunExecution({
+      executions: this.executions,
+      projectId: input.projectId,
+      executionId: input.executionId,
+      runId: input.id,
+      pipelineId: input.pipelineId,
+    })
+
+    const queuedAt = new Date(input.queuedAt ?? new Date())
+    if (existing) {
+      const next: PipelineRunRecord = {
+        ...existing,
+        status: "queued",
+        queuedAt,
+        startedAt: undefined,
+        finishedAt: undefined,
+        output: undefined,
+        error: undefined,
+      }
+      this.runs.set(key, cloneRecord(next))
+      return cloneRecord(next)
+    }
 
     const record: PipelineRunRecord = {
       id: input.id,
       projectId: input.projectId,
+      executionId: input.executionId,
       pipelineId: input.pipelineId,
-      status: "running",
-      startedAt: new Date(input.startedAt ?? new Date()),
+      status: "queued",
+      queuedAt,
     }
 
     this.runs.set(key, cloneRecord(record))
     return cloneRecord(record)
   }
 
+  async start(input: StartPipelineRunInput): Promise<PipelineRunRecord> {
+    const key = storageKey(input.projectId, input.id)
+    const existing = this.runs.get(key)
+    if (!existing) {
+      throw new PipelineRunError(
+        `[Sixb] Pipeline run '${input.id}' not found for project '${input.projectId}'.`
+      )
+    }
+    if (existing.status !== "queued") {
+      throw new PipelineRunError(
+        `[Sixb] Pipeline run '${input.id}' cannot start from status '${existing.status}'.`
+      )
+    }
+
+    const next: PipelineRunRecord = {
+      ...existing,
+      status: "running",
+      startedAt: new Date(input.startedAt ?? new Date()),
+      error: undefined,
+    }
+    this.runs.set(key, cloneRecord(next))
+    return cloneRecord(next)
+  }
+
   async finish(input: FinishPipelineRunInput): Promise<PipelineRunRecord> {
-    const existing = this.requireRunningPipelineRun(input.projectId, input.id)
+    const existing = this.requireExistingPipelineRun(input.projectId, input.id)
+    const enqueueFailure = input.status === "failed" && input.error.code === "queue.enqueue_failed"
+    if (
+      (enqueueFailure && existing.status !== "queued") ||
+      (!enqueueFailure && existing.status !== "running")
+    ) {
+      throw new PipelineRunError(
+        `[Sixb] Pipeline run '${input.id}' cannot finish from status '${existing.status}'.`
+      )
+    }
     const base: PipelineRunRecord = {
       ...existing,
       status: input.status,

package/src/storage/pipeline-runs/index.ts

@@ -1,4 +1,5 @@
 export { PipelineRunError } from "./errors"
+export { canRequeuePipelineRunAfterEnqueueFailure } from "./idempotency"
 export { InMemoryPipelineRunStorage } from "./in-memory"
 export type {
   FinishPipelineRunInput,
@@ -9,11 +10,14 @@ export type {
   ListPipelineRunsResult,
   ListPipelineStepRunsInput,
   ListPipelineStepRunsResult,
-  PipelineRunFailure,
+  PipelineRunFailureCode,
   PipelineRunRecord,
   PipelineRunStatus,
   PipelineRunStorage,
   PipelineStepRunRecord,
+  PipelineStepRunStatus,
+  QueuePipelineRunInput,
   StartPipelineRunInput,
   StartPipelineStepRunInput,
 } from "./types"
+export { PIPELINE_RUN_FAILURE_CODES } from "./types"

package/src/storage/pipeline-runs/provider.ts

@@ -0,0 +1,25 @@
+import type { ExecutionStorage } from "../executions"
+import { findPrimitiveRunExecution } from "../executions/run-link"
+import { PipelineRunError } from "./errors"
+
+/** Validate the semantic link between a durable Pipeline run and its immutable execution. */
+export async function assertPipelineRunExecution(input: {
+  readonly executions: ExecutionStorage
+  readonly projectId: string
+  readonly executionId: string
+  readonly runId: string
+  readonly pipelineId: string
+}): Promise<void> {
+  const execution = await findPrimitiveRunExecution({
+    executions: input.executions,
+    projectId: input.projectId,
+    executionId: input.executionId,
+    primitive: { kind: "pipeline", id: input.pipelineId, runId: input.runId },
+    sourceTypes: ["execution", "schedule", "event"],
+  })
+  if (!execution) {
+    throw new PipelineRunError(
+      `[Sixb] Execution '${input.executionId}' does not authorize Pipeline run '${input.runId}'.`
+    )
+  }
+}

package/src/storage/pipeline-runs/types.ts

@@ -1,22 +1,30 @@
+import type { SixbErrorCode, SixbFailure } from "../../errors/types"
 import type { DatasetVersionRef, DatasetWriteMode } from "../../lake-storage"
 
-export type PipelineRunStatus = "running" | "succeeded" | "failed" | "cancelled"
+export type PipelineRunStatus = "queued" | "running" | "succeeded" | "failed" | "cancelled"
+export type PipelineStepRunStatus = Exclude<PipelineRunStatus, "queued">
+/** Error codes a pipeline or pipeline-step run can persist and expose. */
+export const PIPELINE_RUN_FAILURE_CODES = [
+  "internal.unexpected",
+  "runtime.cancelled",
+  "queue.enqueue_failed",
+  "pipeline.step_failed",
+] as const satisfies readonly [SixbErrorCode, ...SixbErrorCode[]]
 
-export interface PipelineRunFailure {
-  readonly name?: string
-  readonly message: string
-}
+export type PipelineRunFailureCode = (typeof PIPELINE_RUN_FAILURE_CODES)[number]
 
 export interface PipelineRunRecord {
   readonly id: string
   readonly projectId: string
+  readonly executionId: string
   readonly pipelineId: string
   readonly status: PipelineRunStatus
-  readonly startedAt: Date
+  readonly queuedAt: Date
+  readonly startedAt?: Date
   readonly finishedAt?: Date
   /** Final step output in sequential V1. */
   readonly output?: DatasetVersionRef
-  readonly error?: PipelineRunFailure
+  readonly error?: SixbFailure<PipelineRunFailureCode>
 }
 
 export interface PipelineStepRunRecord {
@@ -27,19 +35,26 @@ export interface PipelineStepRunRecord {
   readonly stepId: string
   readonly datasetId: string
   readonly mode: DatasetWriteMode
-  readonly status: PipelineRunStatus
+  readonly status: PipelineStepRunStatus
   readonly startedAt: Date
   readonly finishedAt?: Date
   readonly inputs: readonly DatasetVersionRef[]
   readonly output?: DatasetVersionRef
   readonly rowsWritten?: number
-  readonly error?: PipelineRunFailure
+  readonly error?: SixbFailure<PipelineRunFailureCode>
 }
 
-export interface StartPipelineRunInput {
+export interface QueuePipelineRunInput {
   readonly id: string
   readonly projectId: string
+  readonly executionId: string
   readonly pipelineId: string
+  readonly queuedAt?: Date
+}
+
+export interface StartPipelineRunInput {
+  readonly id: string
+  readonly projectId: string
   readonly startedAt?: Date
 }
 
@@ -54,9 +69,16 @@ export type FinishPipelineRunInput =
   | {
       readonly id: string
       readonly projectId: string
-      readonly status: "failed" | "cancelled"
+      readonly status: "failed"
+      readonly finishedAt?: Date
+      readonly error: SixbFailure<PipelineRunFailureCode>
+    }
+  | {
+      readonly id: string
+      readonly projectId: string
+      readonly status: "cancelled"
       readonly finishedAt?: Date
-      readonly error?: PipelineRunFailure
+      readonly error?: SixbFailure<PipelineRunFailureCode>
     }
 
 export interface StartPipelineStepRunInput {
@@ -86,7 +108,7 @@ export type FinishPipelineStepRunInput =
       readonly status: "failed" | "cancelled"
       readonly finishedAt?: Date
       readonly rowsWritten?: number
-      readonly error?: PipelineRunFailure
+      readonly error?: SixbFailure<PipelineRunFailureCode>
     }
 
 export interface ListPipelineRunsInput {
@@ -122,7 +144,7 @@ export interface ListPipelineStepRunsInput {
   readonly pipelineId?: string
   readonly stepId?: string
   readonly datasetId?: string
-  readonly statuses?: readonly PipelineRunStatus[]
+  readonly statuses?: readonly PipelineStepRunStatus[]
   readonly startedAfter?: Date
   readonly startedBefore?: Date
   readonly limit?: number
@@ -137,6 +159,7 @@ export interface ListPipelineStepRunsResult {
 }
 
 export interface PipelineRunStorage {
+  queue(input: QueuePipelineRunInput): Promise<PipelineRunRecord>
   start(input: StartPipelineRunInput): Promise<PipelineRunRecord>
   finish(input: FinishPipelineRunInput): Promise<PipelineRunRecord>
   startStep(input: StartPipelineStepRunInput): Promise<PipelineStepRunRecord>

package/src/storage/projection-runs/in-memory.ts

@@ -1,19 +1,28 @@
 import { randomUUID } from "node:crypto"
+import type { ExecutionStorage } from "../executions"
 import type { AssertSourceMaterializationExecutionInput } from "../ontology/sources"
-import { latestStartedAtByOwnerId } from "../run-listing"
+import {
+  compareStartedAt,
+  latestStartedAtByOwnerId,
+  matchesRunListDateFilters,
+} from "../run-listing"
 import { ProjectionRunError } from "./errors"
 import {
   advanceProjectionTelemetry,
   assertGenericProgressDoesNotAdvanceTelemetry,
   assertProjectionMissingTarget,
-  assertProjectionRunExecution,
+  assertProjectionRunAttempt,
+  assertProjectionRunDurableExecution,
   assertProjectionRunIdentityMatches,
   assertProjectionRunListWindow,
   assertProjectionRunNonEmpty,
+  assertProjectionRunQueueInput,
   assertProjectionRunRunning,
   assertProjectionRunStartInput,
+  canRequeueProjectionRunAfterEnqueueFailure,
   createProjectionRunClaim,
   createProjectionRunRecord,
+  failProjectionRunEnqueue,
   finishProjectionRunRecord,
   immutableDatasetVersionConflict,
   mergeProjectionRunProgress,
@@ -23,9 +32,10 @@ import {
   requireTelemetryProjectionRun,
   type StoredProjectionRunRecord,
   staleProjectionRunExecution,
-} from "./lifecycle"
+} from "./provider"
 import {
   type AdvanceProjectionTelemetryCheckpointInput,
+  type FailProjectionRunEnqueueInput,
   type FinishProjectionRunInput,
   type ListLatestProjectionRunsInput,
   type ListLatestProjectionRunsResult,
@@ -36,6 +46,7 @@ import {
   type ProjectionRunRecord,
   type ProjectionRunStorage,
   projectionRunObjectTypesVisible,
+  type QueueProjectionRunInput,
   type RecordProjectionMissingTargetInput,
   type StartOrReclaimProjectionRunInput,
   type TelemetryProjectionRunRecord,
@@ -50,10 +61,7 @@ function projectionRunKey(projectId: string, id: string): string {
 }
 
 function compareRuns(a: ProjectionRunRecord, b: ProjectionRunRecord, order: "asc" | "desc") {
-  const delta = a.startedAt.getTime() - b.startedAt.getTime()
-  if (delta !== 0) return order === "asc" ? delta : -delta
-  if (a.id === b.id) return 0
-  return order === "asc" ? a.id.localeCompare(b.id) : b.id.localeCompare(a.id)
+  return compareStartedAt(a, b, order)
 }
 
 export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
@@ -63,16 +71,18 @@ export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
   private readonly runRootOperation: RunRootOperation
   private readonly createExecutionToken: () => string
 
-  constructor(
-    input: {
-      readonly runRootOperation?: RunRootOperation
-      readonly executionToken?: () => string
-    } = {}
-  ) {
+  constructor(input: {
+    readonly executions: ExecutionStorage
+    readonly runRootOperation?: RunRootOperation
+    readonly executionToken?: () => string
+  }) {
+    this.executions = input.executions
     this.runRootOperation = input.runRootOperation ?? runDirectly
     this.createExecutionToken = input.executionToken ?? randomUUID
   }
 
+  private readonly executions: ExecutionStorage
+
   snapshot(): InMemoryProjectionRunStorageSnapshot {
     return structuredClone({
       rows: this.rows,
@@ -89,24 +99,78 @@ export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
     }
   }
 
+  async queue(input: QueueProjectionRunInput): Promise<ProjectionRunRecord> {
+    return this.runRootOperation(async () => {
+      assertProjectionRunQueueInput(input)
+      this.assertDatasetVersionIsImmutable(input)
+      const key = projectionRunKey(input.projectId, input.id)
+      const existing = this.rows.get(key)
+      if (existing && !canRequeueProjectionRunAfterEnqueueFailure(existing, input)) {
+        throw new ProjectionRunError(
+          `[Sixb] Projection run '${input.id}' already exists for project '${input.projectId}'.`
+        )
+      }
+      if (
+        [...this.rows.values()].some(
+          (run) =>
+            run.projectId === input.projectId &&
+            run.id !== input.id &&
+            run.executionId === input.executionId
+        )
+      ) {
+        throw new ProjectionRunError(
+          `[Sixb] Execution '${input.executionId}' already belongs to another Projection run.`
+        )
+      }
+      await assertProjectionRunDurableExecution({
+        executions: this.executions,
+        projectId: input.projectId,
+        executionId: input.executionId,
+        runId: input.id,
+        projectionId: input.identity.projectionId,
+        datasetId: input.identity.datasetVersion.datasetId,
+        datasetVersionId: input.identity.datasetVersion.versionId,
+      })
+      const record = createProjectionRunRecord(input)
+      this.rows.set(key, structuredClone(record))
+      return publicProjectionRunRecord(record)
+    })
+  }
+
   async startOrReclaim(input: StartOrReclaimProjectionRunInput): Promise<ProjectionRunClaim> {
     return this.runRootOperation(() => {
       assertProjectionRunStartInput(input)
-      this.assertDatasetVersionIsImmutable(input)
-
       const key = projectionRunKey(input.projectId, input.id)
       const existing = this.rows.get(key)
-      const attempt = existing ? planProjectionRunReclaim(existing, input).attempt : 1
-      const executionToken = this.issueExecutionToken(key, input.id, existing?.executionToken)
-      const record: StoredProjectionRunRecord = existing
-        ? { ...existing, attempt, executionToken }
-        : { ...createProjectionRunRecord(input), executionToken }
+      if (!existing) throw projectionRunNotFound(input.projectId, input.id)
+      const attempt = planProjectionRunReclaim(existing, input).attempt
+      const executionToken = this.issueExecutionToken(key, input.id, existing.executionToken)
+      const record: StoredProjectionRunRecord = {
+        ...existing,
+        status: "running",
+        attempt,
+        executionToken,
+        startedAt: existing.startedAt ?? new Date(input.startedAt ?? new Date()),
+        finishedAt: undefined,
+        error: undefined,
+      }
 
       this.rows.set(key, structuredClone(record))
       return createProjectionRunClaim(record)
     })
   }
 
+  async failEnqueue(input: FailProjectionRunEnqueueInput): Promise<ProjectionRunRecord> {
+    return this.runRootOperation(() => {
+      const key = projectionRunKey(input.projectId, input.id)
+      const existing = this.rows.get(key)
+      if (!existing) throw projectionRunNotFound(input.projectId, input.id)
+      const next = failProjectionRunEnqueue(existing, input)
+      this.rows.set(key, structuredClone(next))
+      return publicProjectionRunRecord(next)
+    })
+  }
+
   async lockForMaterialization(
     input: LockProjectionRunForMaterializationInput
   ): Promise<ProjectionRunRecord> {
@@ -232,8 +296,13 @@ export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
             : true
         )
         .filter((record) => (statuses ? statuses.has(record.status) : true))
-        .filter((record) => (input.startedAfter ? record.startedAt >= input.startedAfter : true))
-        .filter((record) => (input.startedBefore ? record.startedAt <= input.startedBefore : true))
+        .filter((record) =>
+          matchesRunListDateFilters(record, {
+            statuses,
+            startedAfter: input.startedAfter,
+            startedBefore: input.startedBefore,
+          })
+        )
         .sort((left, right) => compareRuns(left, right, order))
 
       const total = filtered.length
@@ -255,7 +324,7 @@ export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
     })
   }
 
-  private assertDatasetVersionIsImmutable(input: StartOrReclaimProjectionRunInput): void {
+  private assertDatasetVersionIsImmutable(input: QueueProjectionRunInput): void {
     for (const candidate of this.rows.values()) {
       if (
         candidate.projectId === input.projectId &&
@@ -292,7 +361,7 @@ export class InMemoryProjectionRunStorage implements ProjectionRunStorage {
     input: LockProjectionRunForMaterializationInput
   ): StoredProjectionRunRecord & { readonly executionToken: string } {
     const record = this.requireRunning(input.projectId, input.id)
-    assertProjectionRunExecution(record, input)
+    assertProjectionRunAttempt(record, input)
     return record
   }
 

package/src/storage/projection-runs/index.ts

@@ -2,6 +2,7 @@ export { ProjectionRunError } from "./errors"
 export { InMemoryProjectionRunStorage } from "./in-memory"
 export type {
   AdvanceProjectionTelemetryCheckpointInput,
+  FailProjectionRunEnqueueInput,
   FinishProjectionRunInput,
   LinkProjectionRunRecord,
   LinkProjectionTarget,
@@ -18,18 +19,21 @@ export type {
   ProjectionMaterializationIdentity,
   ProjectionMissingTarget,
   ProjectionRunClaim,
+  ProjectionRunFailureCode,
   ProjectionRunProgress,
   ProjectionRunRecord,
   ProjectionRunStatus,
   ProjectionRunStorage,
   ProjectionTarget,
   ProjectionTelemetryCheckpoint,
+  QueueProjectionRunInput,
   RecordProjectionMissingTargetInput,
   StartOrReclaimProjectionRunInput,
   TelemetryProjectionRunRecord,
   UpdateProjectionRunInput,
 } from "./types"
 export {
+  PROJECTION_RUN_FAILURE_CODES,
   PROJECTION_RUN_PROGRESS_KEYS,
   projectionRunObjectTypesVisible,
   zeroProjectionRunProgress,