@sixb/pipeline-worker 0.1.0 → 0.1.2

package/src/errors.ts

@@ -1,25 +1,9 @@
 import type { DatasetDefinition, PipelineDefinition } from "@sixb/core"
+import { createSixbError, isSixbError, summarizeErrorMessage } from "@sixb/core/internal/errors"
 import type { DatasetVersion } from "@sixb/core/lake-storage"
-import type { PipelineRunFailure, PipelineRunStatus } from "@sixb/core/storage"
+import type { PipelineRunStatus } from "@sixb/core/storage"
 import type { PipelineJob } from "./types"
 
-export class PipelineWorkerError extends Error {
-  readonly name = "PipelineWorkerError"
-}
-
-export function toPipelineRunFailure(error: unknown): PipelineRunFailure {
-  if (error instanceof Error) {
-    return {
-      name: error.name,
-      message: error.message,
-    }
-  }
-
-  return {
-    message: String(error),
-  }
-}
-
 export function throwIfAborted(signal: AbortSignal): void {
   if (signal.aborted) {
     throw createAbortError()
@@ -33,39 +17,94 @@ export function statusForFailure(
   return signal.aborted || isAbortError(error) ? "cancelled" : "failed"
 }
 
-export function requireFinishedAt(runId: string, finishedAt: Date | undefined): Date {
-  if (finishedAt) {
-    return finishedAt
+export function requireFinishedAt(input: {
+  readonly pipelineId: string
+  readonly runId: string
+  readonly finishedAt: Date | undefined
+}): Date {
+  if (input.finishedAt) {
+    return input.finishedAt
   }
 
-  throw new PipelineWorkerError(
-    `[SixbPipelineWorker] Pipeline run '${runId}' finished without a finishedAt timestamp.`
+  throw createSixbError(
+    "internal.unexpected",
+    `[SixbPipelineWorker] Pipeline run '${input.runId}' finished without a finishedAt timestamp.`,
+    { details: { pipelineId: input.pipelineId, runId: input.runId } }
   )
 }
 
 export function createStepBookkeepingError(options: {
   readonly pipelineId: string
+  readonly pipelineRunId: string
   readonly stepId: string
-  readonly runId: string
+  readonly stepRunId: string
   readonly version: DatasetVersion
   readonly cause: unknown
 }): Error {
   // The dataset version is already durable; retrying the whole pipeline could duplicate appends.
-  return new PipelineWorkerError(
-    `[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' committed dataset version '${options.version.versionId}', but failed to finalize step run '${options.runId}'. The dataset commit may already have succeeded and the step run record may need repair.`,
-    { cause: options.cause }
+  return createSixbError(
+    "internal.unexpected",
+    `[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' committed dataset version '${options.version.versionId}', but failed to finalize step run '${options.stepRunId}'. The dataset commit may already have succeeded and the step run record may need repair.`,
+    {
+      cause: options.cause,
+      details: {
+        pipelineId: options.pipelineId,
+        pipelineRunId: options.pipelineRunId,
+        stepId: options.stepId,
+        stepRunId: options.stepRunId,
+        datasetId: options.version.datasetId,
+        versionId: options.version.versionId,
+      },
+    }
   )
 }
 
+export function createPipelineStepFailure(options: {
+  readonly pipelineId: string
+  readonly pipelineRunId: string
+  readonly stepId: string
+  readonly stepRunId?: string
+  readonly cause: unknown
+}) {
+  return createSixbError(
+    "pipeline.step_failed",
+    summarizeErrorMessage(options.cause, "Pipeline step execution failed."),
+    {
+      cause: options.cause,
+      details: {
+        pipelineId: options.pipelineId,
+        pipelineRunId: options.pipelineRunId,
+        stepId: options.stepId,
+        ...(options.stepRunId ? { stepRunId: options.stepRunId } : {}),
+      },
+    }
+  )
+}
+
+export function unwrapPipelineStepFailure(error: unknown): unknown {
+  return isSixbError(error) && error.code === "pipeline.step_failed" && error.cause !== undefined
+    ? error.cause
+    : error
+}
+
 export function createPipelineBookkeepingError(options: {
   readonly pipelineId: string
   readonly runId: string
   readonly version: DatasetVersion
   readonly cause: unknown
 }): Error {
-  return new PipelineWorkerError(
+  return createSixbError(
+    "internal.unexpected",
     `[SixbPipelineWorker] Pipeline '${options.pipelineId}' committed final dataset version '${options.version.versionId}', but failed to finalize pipeline run '${options.runId}'. The dataset commit may already have succeeded and the pipeline run record may need repair.`,
-    { cause: options.cause }
+    {
+      cause: options.cause,
+      details: {
+        pipelineId: options.pipelineId,
+        runId: options.runId,
+        datasetId: options.version.datasetId,
+        versionId: options.version.versionId,
+      },
+    }
   )
 }
 
@@ -74,7 +113,11 @@ export function requirePipeline(
   job: PipelineJob
 ): PipelineDefinition {
   if (!pipeline) {
-    throw new PipelineWorkerError(`[SixbPipelineWorker] Unknown pipeline '${job.pipelineId}'.`)
+    throw createSixbError(
+      "internal.unexpected",
+      `[SixbPipelineWorker] Unknown pipeline '${job.pipelineId}'.`,
+      { details: { pipelineId: job.pipelineId, runId: job.id } }
+    )
   }
 
   return pipeline
@@ -83,6 +126,7 @@ export function requirePipeline(
 export function requireRegisteredDataset(options: {
   readonly dataset: DatasetDefinition | null
   readonly pipelineId: string
+  readonly pipelineRunId: string
   readonly stepId: string
   readonly role: "input" | "output"
   readonly name?: string
@@ -94,8 +138,17 @@ export function requireRegisteredDataset(options: {
 
   const namedRole = options.role === "input" ? `input '${options.name ?? "unknown"}'` : "output"
 
-  throw new PipelineWorkerError(
-    `[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' ${namedRole} references unknown dataset '${options.datasetId}'.`
+  throw createSixbError(
+    "internal.unexpected",
+    `[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' ${namedRole} references unknown dataset '${options.datasetId}'.`,
+    {
+      details: {
+        pipelineId: options.pipelineId,
+        pipelineRunId: options.pipelineRunId,
+        stepId: options.stepId,
+        datasetId: options.datasetId,
+      },
+    }
   )
 }
 

package/src/events.ts

@@ -10,8 +10,12 @@ const SOURCE = "SixbPipelineWorker"
 
 export async function emitPipelineRunStarted(
   events: DomainEventLog | undefined,
-  run: Pick<PipelineRunRecord, "id" | "pipelineId" | "startedAt">
+  run: Pick<PipelineRunRecord, "id" | "pipelineId" | "startedAt">,
+  correlationId: string
 ): Promise<void> {
+  if (!run.startedAt) {
+    throw new Error(`[SixbPipelineWorker] Pipeline run '${run.id}' started without a timestamp.`)
+  }
   await events?.emit(
     {
       events: [
@@ -24,6 +28,7 @@ export async function emitPipelineRunStarted(
           },
         },
       ],
+      correlationId,
     },
     { source: SOURCE }
   )
@@ -35,7 +40,8 @@ export async function emitPipelineRunStepStarted(
     PipelineStepRunRecord,
     "id" | "pipelineRunId" | "pipelineId" | "stepId" | "datasetId" | "startedAt"
   >,
-  context: PipelineStepLifecycleContext
+  context: PipelineStepLifecycleContext,
+  correlationId: string
 ): Promise<void> {
   await events?.emit(
     {
@@ -54,6 +60,7 @@ export async function emitPipelineRunStepStarted(
           },
         },
       ],
+      correlationId,
     },
     { source: SOURCE }
   )
@@ -74,7 +81,8 @@ export async function emitPipelineRunStepFinished(
     | "rowsWritten"
     | "error"
   >,
-  context: PipelineStepLifecycleContext
+  context: PipelineStepLifecycleContext,
+  correlationId: string
 ): Promise<void> {
   await events?.emit(
     {
@@ -93,10 +101,11 @@ export async function emitPipelineRunStepFinished(
             finishedAt: requireFinishedAt(step.id, step.finishedAt).toISOString(),
             ...(step.output ? { versionId: step.output.versionId } : {}),
             ...(step.rowsWritten !== undefined ? { rowsWritten: step.rowsWritten } : {}),
-            ...(step.error ? { error: step.error.message } : {}),
+            ...(step.error ? { error: step.error } : {}),
           },
         },
       ],
+      correlationId,
     },
     { source: SOURCE }
   )
@@ -105,7 +114,8 @@ export async function emitPipelineRunStepFinished(
 export async function emitDatasetVersionCommitted(
   events: DomainEventLog | undefined,
   job: PipelineJob,
-  step: PipelineStepRunResult
+  step: PipelineStepRunResult,
+  correlationId: string
 ): Promise<void> {
   if (!step.versionCreated) return
 
@@ -128,6 +138,7 @@ export async function emitDatasetVersionCommitted(
           },
         },
       ],
+      correlationId,
     },
     { source: SOURCE }
   )
@@ -135,37 +146,32 @@ export async function emitDatasetVersionCommitted(
 
 export async function emitPipelineRunFinished(
   events: DomainEventLog | undefined,
-  job: {
-    readonly id: string
-    readonly pipelineId: string
-    readonly status: "succeeded" | "failed" | "cancelled"
-    readonly datasetId?: string
-    readonly versionId?: string
-  }
+  run: Pick<PipelineRunRecord, "id" | "pipelineId" | "status" | "output" | "error">,
+  correlationId: string
 ): Promise<void> {
   await events?.emit(
-    { events: [{ type: "pipeline.run.finished", payload: buildPipelineRunFinishedPayload(job) }] },
+    {
+      events: [{ type: "pipeline.run.finished", payload: buildPipelineRunFinishedPayload(run) }],
+      correlationId,
+    },
     { source: SOURCE }
   )
 }
 
-function buildPipelineRunFinishedPayload(job: {
-  readonly id: string
-  readonly pipelineId: string
-  readonly status: "succeeded" | "failed" | "cancelled"
-  readonly datasetId?: string
-  readonly versionId?: string
-}) {
+function buildPipelineRunFinishedPayload(
+  run: Pick<PipelineRunRecord, "id" | "pipelineId" | "status" | "output" | "error">
+) {
   return {
-    pipelineId: job.pipelineId,
-    runId: job.id,
-    status: job.status,
-    ...(job.datasetId && job.versionId
+    pipelineId: run.pipelineId,
+    runId: run.id,
+    status: requireTerminalStatus(run.status, `Pipeline run '${run.id}'`),
+    ...(run.output
       ? {
-          datasetId: job.datasetId,
-          versionId: job.versionId,
+          datasetId: run.output.datasetId,
+          versionId: run.output.versionId,
         }
       : {}),
+    ...(run.error ? { error: run.error } : {}),
   }
 }
 
@@ -180,8 +186,8 @@ function requireFinishedAt(stepRunId: string, finishedAt: Date | undefined): Dat
 function requireTerminalStatus(
   status: PipelineRunStatus,
   context: string
-): Exclude<PipelineRunStatus, "running"> {
-  if (status === "running") {
+): Exclude<PipelineRunStatus, "queued" | "running"> {
+  if (status === "queued" || status === "running") {
     throw new Error(`[SixbPipelineWorker] ${context} is still running.`)
   }
 

package/src/execute-run-step.ts

@@ -5,8 +5,9 @@ import type {
   PipelineStepDefinition,
   PipelineStepRunContext,
 } from "@sixb/core"
+import { createSixbError } from "@sixb/core/internal/errors"
 import type { DatasetVersion } from "@sixb/core/lake-storage"
-import { PipelineWorkerError, throwIfAborted } from "./errors"
+import { throwIfAborted } from "./errors"
 import { createStepInputs, type ResolvedStepInput } from "./step-inputs"
 import type { PipelineJob, PipelineLogSession, PipelineWorkerContext } from "./types"
 
@@ -27,8 +28,17 @@ export async function executeRunStep(input: {
   const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input
 
   if (step.executor.kind !== "run") {
-    throw new PipelineWorkerError(
-      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses SQL execution, which is not supported by the run step executor.`
+    throw createSixbError(
+      "internal.unexpected",
+      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses SQL execution, which is not supported by the run step executor.`,
+      {
+        details: {
+          pipelineId: pipeline.id,
+          pipelineRunId: job.id,
+          stepId: step.id,
+          datasetId: outputDataset.id,
+        },
+      }
     )
   }
 

package/src/execute-sql-step.ts

@@ -5,12 +5,13 @@ import type {
   PipelineDefinition,
   PipelineStepDefinition,
 } from "@sixb/core"
+import { createSixbError } from "@sixb/core/internal/errors"
 import type {
   DatasetVersion,
   DatasetWriteMode,
   LakeSqlTransformCapabilities,
 } from "@sixb/core/lake-storage"
-import { PipelineWorkerError, throwIfAborted } from "./errors"
+import { throwIfAborted } from "./errors"
 import type { ResolvedStepInput } from "./step-inputs"
 import type { PipelineJob, PipelineWorkerContext } from "./types"
 
@@ -27,7 +28,9 @@ function assertSqlModeSupported(input: {
   readonly dialect: string
   readonly mode: DatasetWriteMode
   readonly pipelineId: string
+  readonly pipelineRunId: string
   readonly stepId: string
+  readonly datasetId: string
 }): void {
   const supported =
     input.mode === "append"
@@ -35,9 +38,18 @@ function assertSqlModeSupported(input: {
       : input.capabilities.supportsSnapshot
   if (supported) return
 
-  throw new PipelineWorkerError(
+  throw createSixbError(
+    "internal.unexpected",
     `[SixbPipelineWorker] Pipeline '${input.pipelineId}' step '${input.stepId}' writes in ` +
-      `'${input.mode}' mode, which the ${input.dialect} SQL executor does not support.`
+      `'${input.mode}' mode, which the ${input.dialect} SQL executor does not support.`,
+    {
+      details: {
+        pipelineId: input.pipelineId,
+        pipelineRunId: input.pipelineRunId,
+        stepId: input.stepId,
+        datasetId: input.datasetId,
+      },
+    }
   )
 }
 
@@ -57,14 +69,32 @@ export async function executeSqlStep(input: {
   const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input
 
   if (step.executor.kind !== "sql") {
-    throw new PipelineWorkerError(
-      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses run execution, which is not supported by the SQL step executor.`
+    throw createSixbError(
+      "internal.unexpected",
+      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses run execution, which is not supported by the SQL step executor.`,
+      {
+        details: {
+          pipelineId: pipeline.id,
+          pipelineRunId: job.id,
+          stepId: step.id,
+          datasetId: outputDataset.id,
+        },
+      }
     )
   }
 
   if (!hasSqlExecutor(runtime.lakeStorage)) {
-    throw new PipelineWorkerError(
-      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' requires SQL transform support, but lake storage does not provide lakeStorage.sql.execute(...).`
+    throw createSixbError(
+      "internal.unexpected",
+      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' requires SQL transform support, but lake storage does not provide lakeStorage.sql.execute(...).`,
+      {
+        details: {
+          pipelineId: pipeline.id,
+          pipelineRunId: job.id,
+          stepId: step.id,
+          datasetId: outputDataset.id,
+        },
+      }
     )
   }
 
@@ -77,7 +107,9 @@ export async function executeSqlStep(input: {
     dialect: runtime.lakeStorage.sql.dialect,
     mode: step.mode,
     pipelineId: pipeline.id,
+    pipelineRunId: job.id,
     stepId: step.id,
+    datasetId: outputDataset.id,
   })
 
   throwIfAborted(signal)

package/src/index.ts

@@ -1,2 +1,2 @@
-export type { PipelineWorkerSixb } from "./types"
+export type { PipelineWorkerHost } from "./types"
 export { PipelineWorker } from "./worker"

package/src/run-pipeline-job.ts

@@ -1,16 +1,22 @@
-import { resolveLogsRuntime } from "@sixb/core/internal/logging"
+import { captureSixbFailure } from "@sixb/core/internal/errors"
+import { resolveLoggingService } from "@sixb/core/internal/logging"
 import type { DatasetVersion } from "@sixb/core/lake-storage"
-import type { PipelineRunRecord } from "@sixb/core/storage"
+import { PIPELINE_RUN_FAILURE_CODES, type PipelineRunRecord } from "@sixb/core/storage"
 import {
   createPipelineBookkeepingError,
   requireFinishedAt,
   requirePipeline,
   statusForFailure,
   throwIfAborted,
-  toPipelineRunFailure,
+  unwrapPipelineStepFailure,
 } from "./errors"
 import { runStep } from "./run-step"
-import type { PipelineRunResult, PipelineStepRunResult, RunPipelineJobInput } from "./types"
+import type {
+  PipelineRunFinishedHandler,
+  PipelineRunResult,
+  PipelineStepRunResult,
+  RunPipelineJobInput,
+} from "./types"
 
 export class PipelineRunAlreadyStartedError extends Error {
   override readonly name = "PipelineRunAlreadyStartedError"
@@ -21,10 +27,15 @@ export class PipelineRunAlreadyStartedError extends Error {
 }
 
 export async function runPipelineJob(input: RunPipelineJobInput): Promise<PipelineRunResult> {
-  const { runtime, job } = input
+  const { runtime, run: durableRun } = input
+  const job = { id: durableRun.id, pipelineId: durableRun.pipelineId }
   const signal = input.signal ?? new AbortController().signal
-  const pipeline = requirePipeline(runtime.getPipelineById(job.pipelineId), job)
-  const logSession = resolveLogsRuntime(runtime.id, runtime.logs).startExecution({
+  if (durableRun.projectId !== runtime.id) {
+    throw new Error(
+      `[SixbPipelineWorker] Pipeline run '${durableRun.id}' belongs to project '${durableRun.projectId}', not '${runtime.id}'.`
+    )
+  }
+  const logSession = resolveLoggingService(runtime.id, runtime.logging).startExecution({
     kind: "pipeline",
     id: job.id,
   })
@@ -40,20 +51,24 @@ export async function runPipelineJob(input: RunPipelineJobInput): Promise<Pipeli
       startedRun = await runtime.pipelineRunsStorage.start({
         projectId: runtime.id,
         id: job.id,
-        pipelineId: pipeline.id,
       })
     } catch (error) {
       const existing = await runtime.pipelineRunsStorage.getById({
         projectId: runtime.id,
         id: job.id,
       })
-      if (existing?.pipelineId === pipeline.id) {
+      if (
+        existing?.pipelineId === durableRun.pipelineId &&
+        existing.executionId === durableRun.executionId
+      ) {
         throw new PipelineRunAlreadyStartedError(existing)
       }
       throw error
     }
     await input.onRunStarted?.(startedRun)
 
+    const pipeline = requirePipeline(runtime.pipelines.getById(job.pipelineId), job)
+
     for (const [stepIndex, node] of pipeline.graph.nodes.entries()) {
       const stepResult = await runStep({
         runtime,
@@ -97,34 +112,62 @@ export async function runPipelineJob(input: RunPipelineJobInput): Promise<Pipeli
       }
       throw error
     }
-    finished = true
+    const finishedRun = {
+      ...run,
+      finishedAt: requireFinishedAt({
+        pipelineId: pipeline.id,
+        runId: job.id,
+        finishedAt: run.finishedAt,
+      }),
+    }
 
+    finished = true
+    await notifyRunFinished(input.onRunFinished, finishedRun)
     return {
-      run: {
-        ...run,
-        finishedAt: requireFinishedAt(job.id, run.finishedAt),
-      },
+      run: finishedRun,
       steps,
       version: finalVersion,
     }
   } catch (error) {
+    const reportedError = unwrapPipelineStepFailure(error)
     if (startedRun && !finished) {
       const status = statusForFailure(signal, error)
       try {
+        const failure = captureSixbFailure(error, {
+          allowedCodes: PIPELINE_RUN_FAILURE_CODES,
+          defaultCode: status === "cancelled" ? "runtime.cancelled" : "internal.unexpected",
+          details: { pipelineId: durableRun.pipelineId, runId: job.id },
+        })
         const run = await runtime.pipelineRunsStorage.finish({
           projectId: runtime.id,
           id: job.id,
           status,
-          error: toPipelineRunFailure(error),
+          error: failure,
         })
-        if (status === "failed" && run.status === "failed") input.onRunFailed?.(error, run)
+        if (status === "failed" && run.status === "failed") {
+          input.onRunFailed?.(reportedError, run, failure)
+        }
+        await notifyRunFinished(input.onRunFinished, run)
       } catch {
         // The run did not transition to the requested terminal status.
       }
     }
 
-    throw error
+    throw reportedError
   } finally {
     await logSession.flush()
   }
 }
+
+async function notifyRunFinished(
+  handler: PipelineRunFinishedHandler | undefined,
+  run: PipelineRunRecord
+): Promise<void> {
+  try {
+    await handler?.(run)
+  } catch (error) {
+    // The built-in event log reports lost batches itself. Anything reaching here is a broken
+    // invariant in a custom lifecycle handler and must not change an already durable outcome.
+    console.error("[SixbPipelineWorker] Pipeline run lifecycle handler failed:", error)
+  }
+}