@sixb/pipeline-worker 0.0.1

package/src/execute-run-step.ts

@@ -0,0 +1,98 @@
+import type {
+  DatasetDefinition,
+  DatasetRow,
+  PipelineDefinition,
+  PipelineStepDefinition,
+  PipelineStepRunContext,
+} from "@sixb/core"
+import type { DatasetVersion } from "@sixb/core/lake-storage"
+import { PipelineWorkerError, throwIfAborted } from "./errors"
+import { createStepInputs, type ResolvedStepInput } from "./step-inputs"
+import type { PipelineJob, PipelineLogSession, PipelineWorkerContext } from "./types"
+
+export async function executeRunStep(input: {
+  readonly runtime: PipelineWorkerContext
+  readonly pipeline: PipelineDefinition
+  readonly step: PipelineStepDefinition
+  readonly job: PipelineJob
+  readonly signal: AbortSignal
+  readonly logSession: PipelineLogSession
+  readonly outputDataset: DatasetDefinition
+  readonly resolvedInputs: readonly ResolvedStepInput[]
+}): Promise<{
+  readonly version: DatasetVersion
+  readonly versionCreated: boolean
+  readonly rowsWritten: number
+}> {
+  const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input
+
+  if (step.executor.kind !== "run") {
+    throw new PipelineWorkerError(
+      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses SQL execution, which is not supported by the run step executor.`
+    )
+  }
+
+  const logger = input.logSession.withContext({ stepId: step.id })
+  let rowsWritten = 0
+  const write = await runtime.lakeStorage.beginWrite({
+    dataset: outputDataset,
+    mode: step.mode,
+    inputs: resolvedInputs.map((resolved) => resolved.ref),
+    producer: {
+      kind: "pipeline",
+      id: pipeline.id,
+      runId: job.id,
+      stepId: step.id,
+    },
+  })
+
+  try {
+    const context: PipelineStepRunContext = {
+      projectId: runtime.id,
+      pipelineId: pipeline.id,
+      stepId: step.id,
+      runId: job.id,
+      signal,
+      logger,
+      inputs: createStepInputs(runtime.lakeStorage, resolvedInputs),
+      output: {
+        async writeRows(rows) {
+          await write.writeRows(
+            countRowsWritten(rows, signal, () => {
+              rowsWritten += 1
+            })
+          )
+        },
+      },
+    }
+
+    await step.executor.handler(context)
+    throwIfAborted(signal)
+
+    const commit = await write.commit({
+      commitMessage: `pipeline ${pipeline.id} step ${step.id} run ${job.id}`,
+    })
+    const { outcome, ...version } = commit
+
+    return {
+      version,
+      versionCreated: outcome === "created",
+      rowsWritten,
+    }
+  } catch (error) {
+    await write.abort().catch(() => {})
+    throw error
+  }
+}
+
+async function* countRowsWritten(
+  rows: Iterable<DatasetRow> | AsyncIterable<DatasetRow>,
+  signal: AbortSignal,
+  onWritten: () => void
+): AsyncIterable<DatasetRow> {
+  for await (const row of rows) {
+    throwIfAborted(signal)
+    yield row
+    onWritten()
+  }
+}

package/src/execute-sql-step.ts

@@ -0,0 +1,115 @@
+import type {
+  DatasetDefinition,
+  LakeStorage,
+  LakeStorageWithSql,
+  PipelineDefinition,
+  PipelineStepDefinition,
+} from "@sixb/core"
+import type {
+  DatasetVersion,
+  DatasetWriteMode,
+  LakeSqlTransformCapabilities,
+} from "@sixb/core/lake-storage"
+import { PipelineWorkerError, throwIfAborted } from "./errors"
+import type { ResolvedStepInput } from "./step-inputs"
+import type { PipelineJob, PipelineWorkerContext } from "./types"
+
+function hasSqlExecutor(lakeStorage: LakeStorage): lakeStorage is LakeStorageWithSql {
+  const sql = "sql" in lakeStorage ? lakeStorage.sql : null
+
+  return (
+    typeof sql === "object" && sql !== null && "execute" in sql && typeof sql.execute === "function"
+  )
+}
+
+function assertSqlModeSupported(input: {
+  readonly capabilities: LakeSqlTransformCapabilities
+  readonly dialect: string
+  readonly mode: DatasetWriteMode
+  readonly pipelineId: string
+  readonly stepId: string
+}): void {
+  const supported =
+    input.mode === "append"
+      ? input.capabilities.supportsAppend
+      : input.capabilities.supportsSnapshot
+  if (supported) return
+
+  throw new PipelineWorkerError(
+    `[SixbPipelineWorker] Pipeline '${input.pipelineId}' step '${input.stepId}' writes in ` +
+      `'${input.mode}' mode, which the ${input.dialect} SQL executor does not support.`
+  )
+}
+
+export async function executeSqlStep(input: {
+  readonly runtime: PipelineWorkerContext
+  readonly pipeline: PipelineDefinition
+  readonly step: PipelineStepDefinition
+  readonly job: PipelineJob
+  readonly signal: AbortSignal
+  readonly outputDataset: DatasetDefinition
+  readonly resolvedInputs: readonly ResolvedStepInput[]
+}): Promise<{
+  readonly version: DatasetVersion
+  readonly versionCreated: boolean
+  readonly rowsWritten?: number
+}> {
+  const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input
+
+  if (step.executor.kind !== "sql") {
+    throw new PipelineWorkerError(
+      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses run execution, which is not supported by the SQL step executor.`
+    )
+  }
+
+  if (!hasSqlExecutor(runtime.lakeStorage)) {
+    throw new PipelineWorkerError(
+      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' requires SQL transform support, but lake storage does not provide lakeStorage.sql.execute(...).`
+    )
+  }
+
+  // `LakeSqlExecutor.capabilities` is a required field on the contract that nothing read,
+  // so a provider declaring `supportsAppend: false` was asked to be honest and then
+  // ignored — the unsupported mode failed somewhere inside the provider instead, in
+  // whatever words that provider happened to use.
+  assertSqlModeSupported({
+    capabilities: runtime.lakeStorage.sql.capabilities,
+    dialect: runtime.lakeStorage.sql.dialect,
+    mode: step.mode,
+    pipelineId: pipeline.id,
+    stepId: step.id,
+  })
+
+  throwIfAborted(signal)
+
+  const commit = await runtime.lakeStorage.sql.execute({
+    sources: Object.fromEntries(
+      resolvedInputs.map((input) => [
+        input.name,
+        {
+          dataset: input.dataset,
+          versionId: input.version.versionId,
+        },
+      ])
+    ),
+    sql: step.executor.sql,
+    target: outputDataset,
+    mode: step.mode,
+    producer: {
+      kind: "pipeline",
+      id: pipeline.id,
+      runId: job.id,
+      stepId: step.id,
+    },
+    commitMessage: `pipeline ${pipeline.id} step ${step.id} run ${job.id}`,
+  })
+
+  throwIfAborted(signal)
+  const { outcome, ...version } = commit
+
+  return {
+    version,
+    versionCreated: outcome === "created",
+    rowsWritten: version.rowCount,
+  }
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export type { PipelineWorkerSixb } from "./types"
+export { PipelineWorker } from "./worker"

package/src/run-pipeline-job.ts

@@ -0,0 +1,130 @@
+import { resolveLogsRuntime } from "@sixb/core/internal/logging"
+import type { DatasetVersion } from "@sixb/core/lake-storage"
+import type { PipelineRunRecord } from "@sixb/core/storage"
+import {
+  createPipelineBookkeepingError,
+  requireFinishedAt,
+  requirePipeline,
+  statusForFailure,
+  throwIfAborted,
+  toPipelineRunFailure,
+} from "./errors"
+import { runStep } from "./run-step"
+import type { PipelineRunResult, PipelineStepRunResult, RunPipelineJobInput } from "./types"
+
+export class PipelineRunAlreadyStartedError extends Error {
+  override readonly name = "PipelineRunAlreadyStartedError"
+
+  constructor(readonly run: PipelineRunRecord) {
+    super(`[SixbPipelineWorker] Pipeline run '${run.id}' has already started.`)
+  }
+}
+
+export async function runPipelineJob(input: RunPipelineJobInput): Promise<PipelineRunResult> {
+  const { runtime, job } = input
+  const signal = input.signal ?? new AbortController().signal
+  const pipeline = requirePipeline(runtime.getPipelineById(job.pipelineId), job)
+  const logSession = resolveLogsRuntime(runtime.id, runtime.logs).startExecution({
+    kind: "pipeline",
+    id: job.id,
+  })
+  const steps: PipelineStepRunResult[] = []
+  let startedRun: PipelineRunRecord | undefined
+  let finalVersion: DatasetVersion | undefined
+  let finished = false
+
+  throwIfAborted(signal)
+
+  try {
+    try {
+      startedRun = await runtime.pipelineRunsStorage.start({
+        projectId: runtime.id,
+        id: job.id,
+        pipelineId: pipeline.id,
+      })
+    } catch (error) {
+      const existing = await runtime.pipelineRunsStorage.getById({
+        projectId: runtime.id,
+        id: job.id,
+      })
+      if (existing?.pipelineId === pipeline.id) {
+        throw new PipelineRunAlreadyStartedError(existing)
+      }
+      throw error
+    }
+    await input.onRunStarted?.(startedRun)
+
+    for (const [stepIndex, node] of pipeline.graph.nodes.entries()) {
+      const stepResult = await runStep({
+        runtime,
+        pipeline,
+        step: node.step,
+        stepIndex,
+        job,
+        signal,
+        logSession,
+        onStepStarted: input.onStepStarted,
+        onStepFinished: input.onStepFinished,
+      })
+
+      steps.push(stepResult)
+      finalVersion = stepResult.version
+      // This is the durability boundary for per-step output events.
+      await input.onStepCommitted?.(stepResult)
+    }
+
+    let run: PipelineRunRecord
+    try {
+      run = await runtime.pipelineRunsStorage.finish({
+        projectId: runtime.id,
+        id: job.id,
+        status: "succeeded",
+        output: finalVersion
+          ? {
+              datasetId: finalVersion.datasetId,
+              versionId: finalVersion.versionId,
+            }
+          : undefined,
+      })
+    } catch (error) {
+      if (finalVersion) {
+        throw createPipelineBookkeepingError({
+          pipelineId: pipeline.id,
+          runId: job.id,
+          version: finalVersion,
+          cause: error,
+        })
+      }
+      throw error
+    }
+    finished = true
+
+    return {
+      run: {
+        ...run,
+        finishedAt: requireFinishedAt(job.id, run.finishedAt),
+      },
+      steps,
+      version: finalVersion,
+    }
+  } catch (error) {
+    if (startedRun && !finished) {
+      const status = statusForFailure(signal, error)
+      try {
+        const run = await runtime.pipelineRunsStorage.finish({
+          projectId: runtime.id,
+          id: job.id,
+          status,
+          error: toPipelineRunFailure(error),
+        })
+        if (status === "failed" && run.status === "failed") input.onRunFailed?.(error, run)
+      } catch {
+        // The run did not transition to the requested terminal status.
+      }
+    }
+
+    throw error
+  } finally {
+    await logSession.flush()
+  }
+}

package/src/run-step.ts

@@ -0,0 +1,154 @@
+import type { DatasetDefinition, PipelineDefinition, PipelineStepDefinition } from "@sixb/core"
+import type { DatasetVersion } from "@sixb/core/lake-storage"
+import type { PipelineStepRunRecord } from "@sixb/core/storage"
+import {
+  createStepBookkeepingError,
+  requireRegisteredDataset,
+  statusForFailure,
+  throwIfAborted,
+  toPipelineRunFailure,
+} from "./errors"
+import { executeRunStep } from "./execute-run-step"
+import { executeSqlStep } from "./execute-sql-step"
+import { type ResolvedStepInput, resolveStepInputs } from "./step-inputs"
+import type {
+  PipelineJob,
+  PipelineLogSession,
+  PipelineStepFinishedHandler,
+  PipelineStepRunResult,
+  PipelineStepStartedHandler,
+  PipelineWorkerContext,
+} from "./types"
+
+type StepExecutionResult = {
+  readonly version: DatasetVersion
+  readonly versionCreated: boolean
+  readonly rowsWritten?: number
+}
+
+export async function runStep(input: {
+  readonly runtime: PipelineWorkerContext
+  readonly pipeline: PipelineDefinition
+  readonly step: PipelineStepDefinition
+  readonly stepIndex: number
+  readonly job: PipelineJob
+  readonly signal: AbortSignal
+  readonly logSession: PipelineLogSession
+  readonly onStepStarted?: PipelineStepStartedHandler
+  readonly onStepFinished?: PipelineStepFinishedHandler
+}): Promise<PipelineStepRunResult> {
+  const { runtime, pipeline, step, stepIndex, job, signal } = input
+  const lifecycleContext = {
+    stepIndex,
+    totalSteps: pipeline.graph.nodes.length,
+  }
+
+  throwIfAborted(signal)
+  const resolvedInputs = await resolveStepInputs({ runtime, pipeline, step })
+  const inputRefs = resolvedInputs.map((resolved) => resolved.ref)
+  const outputDataset = requireRegisteredDataset({
+    dataset: runtime.getDatasetById(step.output.id),
+    pipelineId: pipeline.id,
+    stepId: step.id,
+    role: "output",
+    datasetId: step.output.id,
+  })
+
+  await runtime.lakeStorage.createDataset(outputDataset)
+
+  const stepRunId = `${job.id}:step:${stepIndex + 1}:${step.id}`
+  const startedStepRun = await runtime.pipelineRunsStorage.startStep({
+    projectId: runtime.id,
+    id: stepRunId,
+    pipelineRunId: job.id,
+    pipelineId: pipeline.id,
+    stepId: step.id,
+    datasetId: outputDataset.id,
+    mode: step.mode,
+    inputs: inputRefs,
+  })
+  await input.onStepStarted?.(startedStepRun, lifecycleContext)
+
+  let committedVersion: DatasetVersion | undefined
+  let rowsWritten: number | undefined
+
+  try {
+    const execution = await executeStep({
+      runtime,
+      pipeline,
+      step,
+      job,
+      signal,
+      logSession: input.logSession,
+      outputDataset,
+      resolvedInputs,
+    })
+    committedVersion = execution.version
+    rowsWritten = execution.rowsWritten
+
+    let stepRun: PipelineStepRunRecord
+    try {
+      stepRun = await runtime.pipelineRunsStorage.finishStep({
+        projectId: runtime.id,
+        id: stepRunId,
+        status: "succeeded",
+        output: {
+          datasetId: committedVersion.datasetId,
+          versionId: committedVersion.versionId,
+        },
+        rowsWritten,
+      })
+    } catch (error) {
+      throw createStepBookkeepingError({
+        pipelineId: pipeline.id,
+        stepId: step.id,
+        runId: stepRunId,
+        version: committedVersion,
+        cause: error,
+      })
+    }
+    await input.onStepFinished?.(stepRun, lifecycleContext)
+
+    return {
+      run: stepRun,
+      version: committedVersion,
+      versionCreated: execution.versionCreated,
+    }
+  } catch (error) {
+    if (!committedVersion) {
+      const failedStepRun = await runtime.pipelineRunsStorage
+        .finishStep({
+          projectId: runtime.id,
+          id: stepRunId,
+          status: statusForFailure(signal, error),
+          rowsWritten,
+          error: toPipelineRunFailure(error),
+        })
+        .catch(() => null)
+
+      if (failedStepRun) {
+        await input.onStepFinished?.(failedStepRun, lifecycleContext)
+      }
+    }
+
+    throw error
+  }
+}
+
+async function executeStep(input: {
+  readonly runtime: PipelineWorkerContext
+  readonly pipeline: PipelineDefinition
+  readonly step: PipelineStepDefinition
+  readonly job: PipelineJob
+  readonly signal: AbortSignal
+  readonly logSession: PipelineLogSession
+  readonly outputDataset: DatasetDefinition
+  readonly resolvedInputs: readonly ResolvedStepInput[]
+}): Promise<StepExecutionResult> {
+  switch (input.step.executor.kind) {
+    case "run":
+      return executeRunStep(input)
+    case "sql":
+      return executeSqlStep(input)
+  }
+}

package/src/step-inputs.ts

@@ -0,0 +1,84 @@
+import type {
+  DatasetDefinition,
+  PipelineDefinition,
+  PipelineStepDefinition,
+  PipelineStepInput,
+  PipelineStepRunContext,
+} from "@sixb/core"
+import type {
+  DatasetVersion,
+  DatasetVersionRef,
+  ReadDatasetRowsInput,
+} from "@sixb/core/lake-storage"
+import { PipelineWorkerError, requireRegisteredDataset } from "./errors"
+import type { PipelineWorkerContext } from "./types"
+
+export interface ResolvedStepInput {
+  readonly name: string
+  readonly dataset: DatasetDefinition
+  readonly version: DatasetVersion
+  readonly ref: DatasetVersionRef
+}
+
+export async function resolveStepInputs(input: {
+  readonly runtime: PipelineWorkerContext
+  readonly pipeline: PipelineDefinition
+  readonly step: PipelineStepDefinition
+}): Promise<readonly ResolvedStepInput[]> {
+  const { runtime, pipeline, step } = input
+  const resolved: ResolvedStepInput[] = []
+
+  for (const [name, declaredDataset] of Object.entries(step.inputs)) {
+    const dataset = requireRegisteredDataset({
+      dataset: runtime.getDatasetById(declaredDataset.id),
+      pipelineId: pipeline.id,
+      stepId: step.id,
+      role: "input",
+      name,
+      datasetId: declaredDataset.id,
+    })
+
+    const version = await runtime.lakeStorage.getLatestVersion(dataset.id)
+    if (!version) {
+      throw new PipelineWorkerError(
+        `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' input '${name}' dataset '${dataset.id}' has no committed version.`
+      )
+    }
+
+    resolved.push({
+      name,
+      dataset,
+      version,
+      ref: {
+        datasetId: version.datasetId,
+        versionId: version.versionId,
+      },
+    })
+  }
+
+  return resolved
+}
+
+export function createStepInputs(
+  lakeStorage: PipelineWorkerContext["lakeStorage"],
+  resolved: readonly ResolvedStepInput[]
+): PipelineStepRunContext["inputs"] {
+  const inputs: Record<string, PipelineStepInput> = {}
+
+  for (const input of resolved) {
+    inputs[input.name] = {
+      dataset: input.dataset,
+      version: input.version,
+      readRows(readInput: Omit<ReadDatasetRowsInput, "datasetId" | "versionId"> = {}) {
+        // Step readers stay pinned even if newer versions commit while the handler runs.
+        return lakeStorage.readRows({
+          ...readInput,
+          datasetId: input.dataset.id,
+          versionId: input.version.versionId,
+        })
+      },
+    }
+  }
+
+  return inputs
+}

package/src/types.ts

@@ -0,0 +1,88 @@
+import type {
+  DatasetDefinition,
+  DomainEventLog,
+  LakeStorage,
+  PipelineDefinition,
+  Queues,
+  Storage,
+} from "@sixb/core"
+import type { LogsRuntime } from "@sixb/core/internal/logging"
+import type { DatasetVersion } from "@sixb/core/lake-storage"
+import type {
+  PipelineRunRecord,
+  PipelineRunStorage,
+  PipelineStepRunRecord,
+} from "@sixb/core/storage"
+
+export type PipelineLogSession = ReturnType<LogsRuntime["startExecution"]>
+
+export interface PipelineWorkerContext {
+  readonly id: string
+  readonly pipelineRunsStorage: PipelineRunStorage
+  readonly lakeStorage: LakeStorage
+  readonly logs?: LogsRuntime
+  getDatasetById(datasetId: string): DatasetDefinition | null
+  getPipelineById(pipelineId: string): PipelineDefinition | null
+}
+
+export interface PipelineWorkerSixb {
+  readonly id: string
+  readonly events?: DomainEventLog
+  readonly logs?: LogsRuntime
+  readonly lakeStorage: LakeStorage
+  readonly queues: Queues
+  readonly storage: Storage
+  listPipelines(): readonly PipelineDefinition[]
+  getPipelineById(pipelineId: string): PipelineDefinition | null
+  getDatasetById(datasetId: string): DatasetDefinition | null
+}
+
+export interface PipelineJob {
+  readonly id: string
+  readonly pipelineId: string
+}
+
+export interface RunPipelineJobInput {
+  readonly runtime: PipelineWorkerContext
+  readonly job: PipelineJob
+  readonly signal?: AbortSignal
+  readonly onRunStarted?: PipelineRunStartedHandler
+  readonly onRunFailed?: PipelineRunFailedHandler
+  readonly onStepStarted?: PipelineStepStartedHandler
+  readonly onStepFinished?: PipelineStepFinishedHandler
+  readonly onStepCommitted?: PipelineStepCommittedHandler
+}
+
+export type PipelineStepCommittedHandler = (result: PipelineStepRunResult) => Promise<void> | void
+
+export type PipelineRunStartedHandler = (run: PipelineRunRecord) => Promise<void> | void
+
+export type PipelineRunFailedHandler = (error: unknown, run: PipelineRunRecord) => void
+
+export interface PipelineStepLifecycleContext {
+  readonly stepIndex: number
+  readonly totalSteps: number
+}
+
+export type PipelineStepStartedHandler = (
+  step: PipelineStepRunRecord,
+  context: PipelineStepLifecycleContext
+) => Promise<void> | void
+
+export type PipelineStepFinishedHandler = (
+  step: PipelineStepRunRecord,
+  context: PipelineStepLifecycleContext
+) => Promise<void> | void
+
+export interface PipelineStepRunResult {
+  readonly run: PipelineStepRunRecord
+  readonly version: DatasetVersion
+  /** True only when this step created the returned dataset version. */
+  readonly versionCreated: boolean
+}
+
+export interface PipelineRunResult {
+  readonly run: PipelineRunRecord
+  readonly steps: readonly PipelineStepRunResult[]
+  readonly version?: DatasetVersion
+}