@sixb/pipeline-worker 0.1.1 → 0.1.3

package/src/run-step.ts

@@ -1,12 +1,13 @@
 import type { DatasetDefinition, PipelineDefinition, PipelineStepDefinition } from "@sixb/core"
+import { captureSixbFailure } from "@sixb/core/internal/errors"
 import type { DatasetVersion } from "@sixb/core/lake-storage"
-import type { PipelineStepRunRecord } from "@sixb/core/storage"
+import { PIPELINE_RUN_FAILURE_CODES, type PipelineStepRunRecord } from "@sixb/core/storage"
 import {
+  createPipelineStepFailure,
   createStepBookkeepingError,
   requireRegisteredDataset,
   statusForFailure,
   throwIfAborted,
-  toPipelineRunFailure,
 } from "./errors"
 import { executeRunStep } from "./execute-run-step"
 import { executeSqlStep } from "./execute-sql-step"
@@ -44,17 +45,37 @@ export async function runStep(input: {
   }
 
   throwIfAborted(signal)
-  const resolvedInputs = await resolveStepInputs({ runtime, pipeline, step })
-  const inputRefs = resolvedInputs.map((resolved) => resolved.ref)
-  const outputDataset = requireRegisteredDataset({
-    dataset: runtime.getDatasetById(step.output.id),
-    pipelineId: pipeline.id,
-    stepId: step.id,
-    role: "output",
-    datasetId: step.output.id,
-  })
+  let resolvedInputs: readonly ResolvedStepInput[]
+  let outputDataset: DatasetDefinition
+  try {
+    resolvedInputs = await resolveStepInputs({
+      runtime,
+      pipeline,
+      step,
+      pipelineRunId: job.id,
+    })
+    outputDataset = requireRegisteredDataset({
+      dataset: runtime.datasets.getById(step.output.id),
+      pipelineId: pipeline.id,
+      pipelineRunId: job.id,
+      stepId: step.id,
+      role: "output",
+      datasetId: step.output.id,
+    })
 
-  await runtime.lakeStorage.createDataset(outputDataset)
+    await runtime.lakeStorage.createDataset(outputDataset)
+  } catch (error) {
+    if (statusForFailure(signal, error) === "failed") {
+      throw createPipelineStepFailure({
+        pipelineId: pipeline.id,
+        pipelineRunId: job.id,
+        stepId: step.id,
+        cause: error,
+      })
+    }
+    throw error
+  }
+  const inputRefs = resolvedInputs.map((resolved) => resolved.ref)
 
   const stepRunId = `${job.id}:step:${stepIndex + 1}:${step.id}`
   const startedStepRun = await runtime.pipelineRunsStorage.startStep({
@@ -101,8 +122,9 @@ export async function runStep(input: {
     } catch (error) {
       throw createStepBookkeepingError({
         pipelineId: pipeline.id,
+        pipelineRunId: job.id,
         stepId: step.id,
-        runId: stepRunId,
+        stepRunId,
         version: committedVersion,
         cause: error,
       })
@@ -116,19 +138,41 @@ export async function runStep(input: {
     }
   } catch (error) {
     if (!committedVersion) {
+      const status = statusForFailure(signal, error)
+      const failureError =
+        status === "failed"
+          ? createPipelineStepFailure({
+              pipelineId: pipeline.id,
+              pipelineRunId: job.id,
+              stepId: step.id,
+              stepRunId,
+              cause: error,
+            })
+          : error
       const failedStepRun = await runtime.pipelineRunsStorage
         .finishStep({
           projectId: runtime.id,
           id: stepRunId,
-          status: statusForFailure(signal, error),
+          status,
           rowsWritten,
-          error: toPipelineRunFailure(error),
+          error: captureSixbFailure(failureError, {
+            allowedCodes: PIPELINE_RUN_FAILURE_CODES,
+            defaultCode: status === "cancelled" ? "runtime.cancelled" : "internal.unexpected",
+            details: {
+              pipelineId: pipeline.id,
+              pipelineRunId: job.id,
+              stepId: step.id,
+              stepRunId,
+            },
+          }),
         })
         .catch(() => null)
 
       if (failedStepRun) {
         await input.onStepFinished?.(failedStepRun, lifecycleContext)
       }
+
+      throw failureError
     }
 
     throw error

package/src/step-inputs.ts

@@ -5,12 +5,13 @@ import type {
   PipelineStepInput,
   PipelineStepRunContext,
 } from "@sixb/core"
+import { createSixbError } from "@sixb/core/internal/errors"
 import type {
   DatasetVersion,
   DatasetVersionRef,
   ReadDatasetRowsInput,
 } from "@sixb/core/lake-storage"
-import { PipelineWorkerError, requireRegisteredDataset } from "./errors"
+import { requireRegisteredDataset } from "./errors"
 import type { PipelineWorkerContext } from "./types"
 
 export interface ResolvedStepInput {
@@ -24,14 +25,16 @@ export async function resolveStepInputs(input: {
   readonly runtime: PipelineWorkerContext
   readonly pipeline: PipelineDefinition
   readonly step: PipelineStepDefinition
+  readonly pipelineRunId: string
 }): Promise<readonly ResolvedStepInput[]> {
-  const { runtime, pipeline, step } = input
+  const { runtime, pipeline, step, pipelineRunId } = input
   const resolved: ResolvedStepInput[] = []
 
   for (const [name, declaredDataset] of Object.entries(step.inputs)) {
     const dataset = requireRegisteredDataset({
-      dataset: runtime.getDatasetById(declaredDataset.id),
+      dataset: runtime.datasets.getById(declaredDataset.id),
       pipelineId: pipeline.id,
+      pipelineRunId,
       stepId: step.id,
       role: "input",
       name,
@@ -40,8 +43,17 @@ export async function resolveStepInputs(input: {
 
     const version = await runtime.lakeStorage.getLatestVersion(dataset.id)
     if (!version) {
-      throw new PipelineWorkerError(
-        `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' input '${name}' dataset '${dataset.id}' has no committed version.`
+      throw createSixbError(
+        "internal.unexpected",
+        `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' input '${name}' dataset '${dataset.id}' has no committed version.`,
+        {
+          details: {
+            pipelineId: pipeline.id,
+            pipelineRunId,
+            stepId: step.id,
+            datasetId: dataset.id,
+          },
+        }
       )
     }
 

package/src/types.ts

@@ -1,40 +1,40 @@
 import type {
-  DatasetDefinition,
   DomainEventLog,
   LakeStorage,
-  PipelineDefinition,
   Queues,
+  SixbDefinitions,
+  SixbFailure,
   Storage,
 } from "@sixb/core"
-import type { LogsRuntime } from "@sixb/core/internal/logging"
+import type { LoggingService } from "@sixb/core/internal/logging"
+import type { PrimitiveExecutionHost } from "@sixb/core/internal/primitive-execution"
 import type { DatasetVersion } from "@sixb/core/lake-storage"
 import type {
+  PipelineRunFailureCode,
   PipelineRunRecord,
   PipelineRunStorage,
   PipelineStepRunRecord,
 } from "@sixb/core/storage"
 
-export type PipelineLogSession = ReturnType<LogsRuntime["startExecution"]>
+export type PipelineLogSession = ReturnType<LoggingService["startExecution"]>
 
 export interface PipelineWorkerContext {
   readonly id: string
   readonly pipelineRunsStorage: PipelineRunStorage
   readonly lakeStorage: LakeStorage
-  readonly logs?: LogsRuntime
-  getDatasetById(datasetId: string): DatasetDefinition | null
-  getPipelineById(pipelineId: string): PipelineDefinition | null
+  readonly logging?: LoggingService
+  readonly datasets: Pick<SixbDefinitions["datasets"], "getById">
+  readonly pipelines: Pick<SixbDefinitions["pipelines"], "getById">
 }
 
-export interface PipelineWorkerSixb {
+export interface PipelineWorkerHost extends PrimitiveExecutionHost {
   readonly id: string
   readonly events?: DomainEventLog
-  readonly logs?: LogsRuntime
+  readonly logging?: LoggingService
   readonly lakeStorage: LakeStorage
   readonly queues: Queues
   readonly storage: Storage
-  listPipelines(): readonly PipelineDefinition[]
-  getPipelineById(pipelineId: string): PipelineDefinition | null
-  getDatasetById(datasetId: string): DatasetDefinition | null
+  readonly definitions: Pick<SixbDefinitions, "pipelines" | "datasets">
 }
 
 export interface PipelineJob {
@@ -44,9 +44,11 @@ export interface PipelineJob {
 
 export interface RunPipelineJobInput {
   readonly runtime: PipelineWorkerContext
-  readonly job: PipelineJob
+  /** Durable run loaded before the execution scope is restored. */
+  readonly run: PipelineRunRecord
   readonly signal?: AbortSignal
   readonly onRunStarted?: PipelineRunStartedHandler
+  readonly onRunFinished?: PipelineRunFinishedHandler
   readonly onRunFailed?: PipelineRunFailedHandler
   readonly onStepStarted?: PipelineStepStartedHandler
   readonly onStepFinished?: PipelineStepFinishedHandler
@@ -57,7 +59,13 @@ export type PipelineStepCommittedHandler = (result: PipelineStepRunResult) => Pr
 
 export type PipelineRunStartedHandler = (run: PipelineRunRecord) => Promise<void> | void
 
-export type PipelineRunFailedHandler = (error: unknown, run: PipelineRunRecord) => void
+export type PipelineRunFinishedHandler = (run: PipelineRunRecord) => Promise<void> | void
+
+export type PipelineRunFailedHandler = (
+  error: unknown,
+  run: PipelineRunRecord,
+  failure: SixbFailure<PipelineRunFailureCode>
+) => void
 
 export interface PipelineStepLifecycleContext {
   readonly stepIndex: number

package/src/worker.ts

@@ -1,8 +1,9 @@
 import { reportRunFailure } from "@sixb/core/internal/error-reporting"
+import { bindDurablePrimitiveExecution } from "@sixb/core/internal/primitive-execution"
 import type { QueueWorkerFailureDecision } from "@sixb/core/internal/workers"
 import { QueueWorker } from "@sixb/core/internal/workers"
 import type { ClaimedQueueJob, PipelineRunRequestedQueueJob } from "@sixb/core/queues"
-import type { PipelineRunStorage } from "@sixb/core/storage"
+import { PIPELINE_RUN_FAILURE_CODES, type PipelineRunStorage } from "@sixb/core/storage"
 import {
   emitDatasetVersionCommitted,
   emitPipelineRunFinished,
@@ -11,35 +12,40 @@ import {
   emitPipelineRunStepStarted,
 } from "./events"
 import { PipelineRunAlreadyStartedError, runPipelineJob } from "./run-pipeline-job"
-import type {
-  PipelineJob,
-  PipelineRunResult,
-  PipelineWorkerContext,
-  PipelineWorkerSixb,
-} from "./types"
-
-export class PipelineWorker extends QueueWorker<PipelineRunRequestedQueueJob> {
+import type { PipelineJob, PipelineWorkerContext, PipelineWorkerHost } from "./types"
+
+export interface PipelineWorkerOptions {
+  /** Maximum pipeline run jobs this worker claims and executes at once. Defaults to 1. */
+  readonly concurrency?: number
+}
+
+export class PipelineWorker extends QueueWorker<
+  PipelineRunRequestedQueueJob,
+  typeof PIPELINE_RUN_FAILURE_CODES
+> {
   private readonly context: PipelineWorkerContext
-  private readonly sixb: PipelineWorkerSixb
+  private readonly host: PipelineWorkerHost
 
-  constructor(sixb: PipelineWorkerSixb) {
-    if (sixb.listPipelines().length === 0) {
+  constructor(host: PipelineWorkerHost, options: PipelineWorkerOptions = {}) {
+    if (host.definitions.pipelines.list().length === 0) {
       throw new Error("[SixbPipelineWorker] No pipeline definitions are registered.")
     }
 
-    const pipelineRunsStorage = sixb.storage.pipelineRuns
+    const pipelineRunsStorage = host.storage.pipelineRuns
     if (!pipelineRunsStorage) {
       throw new Error("[SixbPipelineWorker] Pipeline workers require storage.pipelineRuns support.")
     }
 
     super({
-      projectId: sixb.id,
-      queue: sixb.queues.pipelines,
-      workerId: `pipeline-worker-${sixb.id}`,
+      projectId: host.id,
+      queue: host.queues.pipelines,
+      failureCodes: PIPELINE_RUN_FAILURE_CODES,
+      workerId: `pipeline-worker-${host.id}`,
+      claimLimit: options.concurrency,
     })
 
-    this.context = buildPipelineContext(sixb, pipelineRunsStorage)
-    this.sixb = sixb
+    this.context = buildPipelineContext(host, pipelineRunsStorage)
+    this.host = host
   }
 
   protected async execute(
@@ -47,106 +53,132 @@ export class PipelineWorker extends QueueWorker<PipelineRunRequestedQueueJob> {
     signal: AbortSignal
   ): Promise<void> {
     const { job } = claimed
+    const pipelineRuns = this.host.storage.pipelineRuns
+    if (!pipelineRuns) {
+      throw new Error("[SixbPipelineWorker] Pipeline workers require storage.pipelineRuns support.")
+    }
+    const run = await pipelineRuns.getById({ projectId: this.host.id, id: job.payload.runId })
+    if (!run) {
+      throw new Error(`[SixbPipelineWorker] Pipeline run '${job.payload.runId}' was not found.`)
+    }
+
+    const durableExecution = await this.host.storage.executions.getById({
+      projectId: this.host.id,
+      id: run.executionId,
+    })
+    if (!durableExecution) {
+      throw new Error(
+        `[SixbPipelineWorker] Pipeline run '${run.id}' references missing execution '${run.executionId}'.`
+      )
+    }
+
     const pipelineJob: PipelineJob = {
-      id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,
-      pipelineId: job.payload.pipelineId,
+      id: run.id,
+      pipelineId: run.pipelineId,
     }
+    // Pipeline steps currently use host-owned lake and run storage directly. Binding still happens
+    // here so the durable execution and its trusted primitive authority are validated before work.
+    bindDurablePrimitiveExecution(this.host, {
+      execution: durableExecution,
+      primitive: { kind: "pipeline", id: run.pipelineId, runId: run.id },
+    })
 
-    let result: PipelineRunResult
     try {
-      result = await runPipelineJob({
+      await runPipelineJob({
         runtime: this.context,
-        job: pipelineJob,
+        run,
         signal,
-        onRunStarted: (run) => emitPipelineRunStarted(this.sixb.events, run),
-        onRunFailed: (error, run) => {
-          reportRunFailure(this.sixb, error, {
-            projectId: this.sixb.id,
-            occurredAt: run.finishedAt,
+        onRunStarted: (run) =>
+          emitPipelineRunStarted(this.host.events, run, durableExecution.correlationId),
+        onRunFinished: (run) =>
+          emitPipelineRunFinished(this.host.events, run, durableExecution.correlationId),
+        onRunFailed: (error, run, failure) => {
+          reportRunFailure(this.host, error, {
+            projectId: this.host.id,
             attempt: job.attempt,
+            runKind: "pipeline",
             run: {
-              kind: "pipeline",
               runId: run.id,
               pipelineId: run.pipelineId,
             },
+            failure,
           })
         },
         onStepStarted: (step, context) =>
-          emitPipelineRunStepStarted(this.sixb.events, step, context),
+          emitPipelineRunStepStarted(
+            this.host.events,
+            step,
+            context,
+            durableExecution.correlationId
+          ),
         onStepFinished: (step, context) =>
-          emitPipelineRunStepFinished(this.sixb.events, step, context),
-        onStepCommitted: (step) => emitDatasetVersionCommitted(this.sixb.events, pipelineJob, step),
+          emitPipelineRunStepFinished(
+            this.host.events,
+            step,
+            context,
+            durableExecution.correlationId
+          ),
+        onStepCommitted: (step) =>
+          emitDatasetVersionCommitted(
+            this.host.events,
+            pipelineJob,
+            step,
+            durableExecution.correlationId
+          ),
       })
     } catch (error) {
       if (error instanceof PipelineRunAlreadyStartedError) return
       throw error
     }
-
-    await emitPipelineRunFinished(this.sixb.events, {
-      id: pipelineJob.id,
-      pipelineId: pipelineJob.pipelineId,
-      status: "succeeded",
-      datasetId: result.version?.datasetId,
-      versionId: result.version?.versionId,
-    })
   }
 
   protected override async onExecutionError(
     claimed: ClaimedQueueJob<PipelineRunRequestedQueueJob>,
     _error: unknown
-  ): Promise<QueueWorkerFailureDecision> {
-    const { job } = claimed
-    await emitPipelineRunFinished(this.sixb.events, {
-      id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,
-      pipelineId: job.payload.pipelineId,
-      status: "failed",
-    })
-
+  ): Promise<QueueWorkerFailureDecision<(typeof PIPELINE_RUN_FAILURE_CODES)[number]>> {
     // Pipeline runs are not all-or-nothing in V1. A previous step may have committed an append
     // output, so failed pipeline jobs must not be retried automatically.
-    return { kind: "fail" }
+    const run = await this.host.storage.pipelineRuns?.getById({
+      projectId: this.host.id,
+      id: claimed.job.payload.runId,
+    })
+    return run?.status === "failed" && run.error
+      ? { kind: "fail", failure: run.error }
+      : { kind: "fail" }
   }
 
   protected override async onAbortError(
     claimed: ClaimedQueueJob<PipelineRunRequestedQueueJob>,
     _error: unknown
-  ): Promise<QueueWorkerFailureDecision> {
-    const { job } = claimed
-    const pipelineJob: PipelineJob = {
-      id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,
-      pipelineId: job.payload.pipelineId,
-    }
+  ): Promise<QueueWorkerFailureDecision<(typeof PIPELINE_RUN_FAILURE_CODES)[number]>> {
+    const run = await this.host.storage.pipelineRuns?.getById({
+      projectId: this.host.id,
+      id: claimed.job.payload.runId,
+    })
+    if (!run) return { kind: "fail" }
 
-    if (!(await hasCommittedStep(this.context, pipelineJob.id))) {
+    if (!(await hasCommittedStep(this.context, run.id))) {
       return { kind: "retry" }
     }
 
     // After a step commit, rerunning the whole pipeline could duplicate append outputs.
-    await emitPipelineRunFinished(this.sixb.events, {
-      id: pipelineJob.id,
-      pipelineId: pipelineJob.pipelineId,
-      status: "cancelled",
-    })
-
-    return { kind: "fail" }
+    return run.status === "cancelled" && run.error
+      ? { kind: "fail", failure: run.error }
+      : { kind: "fail" }
   }
 }
 
 function buildPipelineContext(
-  sixb: PipelineWorkerSixb,
+  host: PipelineWorkerHost,
   pipelineRunsStorage: PipelineRunStorage
 ): PipelineWorkerContext {
   return {
-    id: sixb.id,
+    id: host.id,
     pipelineRunsStorage,
-    lakeStorage: sixb.lakeStorage,
-    logs: sixb.logs,
-    getPipelineById(pipelineId) {
-      return sixb.getPipelineById(pipelineId)
-    },
-    getDatasetById(datasetId) {
-      return sixb.getDatasetById(datasetId)
-    },
+    lakeStorage: host.lakeStorage,
+    logging: host.logging,
+    pipelines: host.definitions.pipelines,
+    datasets: host.definitions.datasets,
   }
 }