@sixb/pipeline-worker 0.0.1

package/dist/index.js.map

@@ -0,0 +1,17 @@
+{
+  "version": 3,
+  "sources": ["../src/worker.ts", "../src/events.ts", "../src/run-pipeline-job.ts", "../src/errors.ts", "../src/step-inputs.ts", "../src/execute-run-step.ts", "../src/execute-sql-step.ts", "../src/run-step.ts"],
+  "sourcesContent": [
+    "import { reportRunFailure } from \"@sixb/core/internal/error-reporting\"\nimport type { QueueWorkerFailureDecision } from \"@sixb/core/internal/workers\"\nimport { QueueWorker } from \"@sixb/core/internal/workers\"\nimport type { ClaimedQueueJob, PipelineRunRequestedQueueJob } from \"@sixb/core/queues\"\nimport type { PipelineRunStorage } from \"@sixb/core/storage\"\nimport {\n  emitDatasetVersionCommitted,\n  emitPipelineRunFinished,\n  emitPipelineRunStarted,\n  emitPipelineRunStepFinished,\n  emitPipelineRunStepStarted,\n} from \"./events\"\nimport { PipelineRunAlreadyStartedError, runPipelineJob } from \"./run-pipeline-job\"\nimport type {\n  PipelineJob,\n  PipelineRunResult,\n  PipelineWorkerContext,\n  PipelineWorkerSixb,\n} from \"./types\"\n\nexport class PipelineWorker extends QueueWorker<PipelineRunRequestedQueueJob> {\n  private readonly context: PipelineWorkerContext\n  private readonly sixb: PipelineWorkerSixb\n\n  constructor(sixb: PipelineWorkerSixb) {\n    if (sixb.listPipelines().length === 0) {\n      throw new Error(\"[SixbPipelineWorker] No pipeline definitions are registered.\")\n    }\n\n    const pipelineRunsStorage = sixb.storage.pipelineRuns\n    if (!pipelineRunsStorage) {\n      throw new Error(\"[SixbPipelineWorker] Pipeline workers require storage.pipelineRuns support.\")\n    }\n\n    super({\n      projectId: sixb.id,\n      queue: sixb.queues.pipelines,\n      workerId: `pipeline-worker-${sixb.id}`,\n    })\n\n    this.context = buildPipelineContext(sixb, pipelineRunsStorage)\n    this.sixb = sixb\n  }\n\n  protected async execute(\n    claimed: ClaimedQueueJob<PipelineRunRequestedQueueJob>,\n    signal: AbortSignal\n  ): Promise<void> {\n    const { job } = claimed\n    const pipelineJob: PipelineJob = {\n      id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,\n      pipelineId: job.payload.pipelineId,\n    }\n\n    let result: PipelineRunResult\n    try {\n      result = await runPipelineJob({\n        runtime: this.context,\n        job: pipelineJob,\n        signal,\n        onRunStarted: (run) => emitPipelineRunStarted(this.sixb.events, run),\n        onRunFailed: (error, run) => {\n          reportRunFailure(this.sixb, error, {\n            projectId: this.sixb.id,\n            occurredAt: run.finishedAt,\n            attempt: job.attempt,\n            run: {\n              kind: \"pipeline\",\n              runId: run.id,\n              pipelineId: run.pipelineId,\n            },\n          })\n        },\n        onStepStarted: (step, context) =>\n          emitPipelineRunStepStarted(this.sixb.events, step, context),\n        onStepFinished: (step, context) =>\n          emitPipelineRunStepFinished(this.sixb.events, step, context),\n        onStepCommitted: (step) => emitDatasetVersionCommitted(this.sixb.events, pipelineJob, step),\n      })\n    } catch (error) {\n      if (error instanceof PipelineRunAlreadyStartedError) return\n      throw error\n    }\n\n    await emitPipelineRunFinished(this.sixb.events, {\n      id: pipelineJob.id,\n      pipelineId: pipelineJob.pipelineId,\n      status: \"succeeded\",\n      datasetId: result.version?.datasetId,\n      versionId: result.version?.versionId,\n    })\n  }\n\n  protected override async onExecutionError(\n    claimed: ClaimedQueueJob<PipelineRunRequestedQueueJob>,\n    _error: unknown\n  ): Promise<QueueWorkerFailureDecision> {\n    const { job } = claimed\n    await emitPipelineRunFinished(this.sixb.events, {\n      id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,\n      pipelineId: job.payload.pipelineId,\n      status: \"failed\",\n    })\n\n    // Pipeline runs are not all-or-nothing in V1. A previous step may have committed an append\n    // output, so failed pipeline jobs must not be retried automatically.\n    return { kind: \"fail\" }\n  }\n\n  protected override async onAbortError(\n    claimed: ClaimedQueueJob<PipelineRunRequestedQueueJob>,\n    _error: unknown\n  ): Promise<QueueWorkerFailureDecision> {\n    const { job } = claimed\n    const pipelineJob: PipelineJob = {\n      id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,\n      pipelineId: job.payload.pipelineId,\n    }\n\n    if (!(await hasCommittedStep(this.context, pipelineJob.id))) {\n      return { kind: \"retry\" }\n    }\n\n    // After a step commit, rerunning the whole pipeline could duplicate append outputs.\n    await emitPipelineRunFinished(this.sixb.events, {\n      id: pipelineJob.id,\n      pipelineId: pipelineJob.pipelineId,\n      status: \"cancelled\",\n    })\n\n    return { kind: \"fail\" }\n  }\n}\n\nfunction buildPipelineContext(\n  sixb: PipelineWorkerSixb,\n  pipelineRunsStorage: PipelineRunStorage\n): PipelineWorkerContext {\n  return {\n    id: sixb.id,\n    pipelineRunsStorage,\n    lakeStorage: sixb.lakeStorage,\n    logs: sixb.logs,\n    getPipelineById(pipelineId) {\n      return sixb.getPipelineById(pipelineId)\n    },\n    getDatasetById(datasetId) {\n      return sixb.getDatasetById(datasetId)\n    },\n  }\n}\n\nasync function hasCommittedStep(\n  context: PipelineWorkerContext,\n  pipelineRunId: string\n): Promise<boolean> {\n  const result = await context.pipelineRunsStorage.listSteps({\n    projectId: context.id,\n    pipelineRunId,\n    statuses: [\"succeeded\"],\n    limit: 1,\n  })\n\n  return result.steps.some((step) => Boolean(step.output))\n}\n",
+    "import type { DomainEventLog } from \"@sixb/core\"\nimport type {\n  PipelineRunRecord,\n  PipelineRunStatus,\n  PipelineStepRunRecord,\n} from \"@sixb/core/storage\"\nimport type { PipelineJob, PipelineStepLifecycleContext, PipelineStepRunResult } from \"./types\"\n\nconst SOURCE = \"SixbPipelineWorker\"\n\nexport async function emitPipelineRunStarted(\n  events: DomainEventLog | undefined,\n  run: Pick<PipelineRunRecord, \"id\" | \"pipelineId\" | \"startedAt\">\n): Promise<void> {\n  await events?.emit(\n    {\n      events: [\n        {\n          type: \"pipeline.run.started\",\n          payload: {\n            pipelineId: run.pipelineId,\n            runId: run.id,\n            startedAt: run.startedAt.toISOString(),\n          },\n        },\n      ],\n    },\n    { source: SOURCE }\n  )\n}\n\nexport async function emitPipelineRunStepStarted(\n  events: DomainEventLog | undefined,\n  step: Pick<\n    PipelineStepRunRecord,\n    \"id\" | \"pipelineRunId\" | \"pipelineId\" | \"stepId\" | \"datasetId\" | \"startedAt\"\n  >,\n  context: PipelineStepLifecycleContext\n): Promise<void> {\n  await events?.emit(\n    {\n      events: [\n        {\n          type: \"pipeline.run.step.started\",\n          payload: {\n            pipelineId: step.pipelineId,\n            runId: step.pipelineRunId,\n            stepRunId: step.id,\n            stepId: step.stepId,\n            stepIndex: context.stepIndex,\n            totalSteps: context.totalSteps,\n            datasetId: step.datasetId,\n            startedAt: step.startedAt.toISOString(),\n          },\n        },\n      ],\n    },\n    { source: SOURCE }\n  )\n}\n\nexport async function emitPipelineRunStepFinished(\n  events: DomainEventLog | undefined,\n  step: Pick<\n    PipelineStepRunRecord,\n    | \"id\"\n    | \"pipelineRunId\"\n    | \"pipelineId\"\n    | \"stepId\"\n    | \"datasetId\"\n    | \"status\"\n    | \"finishedAt\"\n    | \"output\"\n    | \"rowsWritten\"\n    | \"error\"\n  >,\n  context: PipelineStepLifecycleContext\n): Promise<void> {\n  await events?.emit(\n    {\n      events: [\n        {\n          type: \"pipeline.run.step.finished\",\n          payload: {\n            pipelineId: step.pipelineId,\n            runId: step.pipelineRunId,\n            stepRunId: step.id,\n            stepId: step.stepId,\n            stepIndex: context.stepIndex,\n            totalSteps: context.totalSteps,\n            datasetId: step.datasetId,\n            status: requireTerminalStatus(step.status, `Pipeline step run '${step.id}'`),\n            finishedAt: requireFinishedAt(step.id, step.finishedAt).toISOString(),\n            ...(step.output ? { versionId: step.output.versionId } : {}),\n            ...(step.rowsWritten !== undefined ? { rowsWritten: step.rowsWritten } : {}),\n            ...(step.error ? { error: step.error.message } : {}),\n          },\n        },\n      ],\n    },\n    { source: SOURCE }\n  )\n}\n\nexport async function emitDatasetVersionCommitted(\n  events: DomainEventLog | undefined,\n  job: PipelineJob,\n  step: PipelineStepRunResult\n): Promise<void> {\n  if (!step.versionCreated) return\n\n  // Emit step outputs immediately so dataset event schedules can react before later steps finish.\n  await events?.emit(\n    {\n      events: [\n        {\n          type: \"dataset.version.committed\" as const,\n          payload: {\n            datasetId: step.version.datasetId,\n            versionId: step.version.versionId,\n            createdAt: step.version.createdAt.toISOString(),\n            producer: {\n              kind: \"pipeline\" as const,\n              id: job.pipelineId,\n              runId: job.id,\n              stepId: step.run.stepId,\n            },\n          },\n        },\n      ],\n    },\n    { source: SOURCE }\n  )\n}\n\nexport async function emitPipelineRunFinished(\n  events: DomainEventLog | undefined,\n  job: {\n    readonly id: string\n    readonly pipelineId: string\n    readonly status: \"succeeded\" | \"failed\" | \"cancelled\"\n    readonly datasetId?: string\n    readonly versionId?: string\n  }\n): Promise<void> {\n  await events?.emit(\n    { events: [{ type: \"pipeline.run.finished\", payload: buildPipelineRunFinishedPayload(job) }] },\n    { source: SOURCE }\n  )\n}\n\nfunction buildPipelineRunFinishedPayload(job: {\n  readonly id: string\n  readonly pipelineId: string\n  readonly status: \"succeeded\" | \"failed\" | \"cancelled\"\n  readonly datasetId?: string\n  readonly versionId?: string\n}) {\n  return {\n    pipelineId: job.pipelineId,\n    runId: job.id,\n    status: job.status,\n    ...(job.datasetId && job.versionId\n      ? {\n          datasetId: job.datasetId,\n          versionId: job.versionId,\n        }\n      : {}),\n  }\n}\n\nfunction requireFinishedAt(stepRunId: string, finishedAt: Date | undefined): Date {\n  if (finishedAt) {\n    return finishedAt\n  }\n\n  throw new Error(`[SixbPipelineWorker] Pipeline step run '${stepRunId}' has no finishedAt.`)\n}\n\nfunction requireTerminalStatus(\n  status: PipelineRunStatus,\n  context: string\n): Exclude<PipelineRunStatus, \"running\"> {\n  if (status === \"running\") {\n    throw new Error(`[SixbPipelineWorker] ${context} is still running.`)\n  }\n\n  return status\n}\n",
+    "import { resolveLogsRuntime } from \"@sixb/core/internal/logging\"\nimport type { DatasetVersion } from \"@sixb/core/lake-storage\"\nimport type { PipelineRunRecord } from \"@sixb/core/storage\"\nimport {\n  createPipelineBookkeepingError,\n  requireFinishedAt,\n  requirePipeline,\n  statusForFailure,\n  throwIfAborted,\n  toPipelineRunFailure,\n} from \"./errors\"\nimport { runStep } from \"./run-step\"\nimport type { PipelineRunResult, PipelineStepRunResult, RunPipelineJobInput } from \"./types\"\n\nexport class PipelineRunAlreadyStartedError extends Error {\n  override readonly name = \"PipelineRunAlreadyStartedError\"\n\n  constructor(readonly run: PipelineRunRecord) {\n    super(`[SixbPipelineWorker] Pipeline run '${run.id}' has already started.`)\n  }\n}\n\nexport async function runPipelineJob(input: RunPipelineJobInput): Promise<PipelineRunResult> {\n  const { runtime, job } = input\n  const signal = input.signal ?? new AbortController().signal\n  const pipeline = requirePipeline(runtime.getPipelineById(job.pipelineId), job)\n  const logSession = resolveLogsRuntime(runtime.id, runtime.logs).startExecution({\n    kind: \"pipeline\",\n    id: job.id,\n  })\n  const steps: PipelineStepRunResult[] = []\n  let startedRun: PipelineRunRecord | undefined\n  let finalVersion: DatasetVersion | undefined\n  let finished = false\n\n  throwIfAborted(signal)\n\n  try {\n    try {\n      startedRun = await runtime.pipelineRunsStorage.start({\n        projectId: runtime.id,\n        id: job.id,\n        pipelineId: pipeline.id,\n      })\n    } catch (error) {\n      const existing = await runtime.pipelineRunsStorage.getById({\n        projectId: runtime.id,\n        id: job.id,\n      })\n      if (existing?.pipelineId === pipeline.id) {\n        throw new PipelineRunAlreadyStartedError(existing)\n      }\n      throw error\n    }\n    await input.onRunStarted?.(startedRun)\n\n    for (const [stepIndex, node] of pipeline.graph.nodes.entries()) {\n      const stepResult = await runStep({\n        runtime,\n        pipeline,\n        step: node.step,\n        stepIndex,\n        job,\n        signal,\n        logSession,\n        onStepStarted: input.onStepStarted,\n        onStepFinished: input.onStepFinished,\n      })\n\n      steps.push(stepResult)\n      finalVersion = stepResult.version\n      // This is the durability boundary for per-step output events.\n      await input.onStepCommitted?.(stepResult)\n    }\n\n    let run: PipelineRunRecord\n    try {\n      run = await runtime.pipelineRunsStorage.finish({\n        projectId: runtime.id,\n        id: job.id,\n        status: \"succeeded\",\n        output: finalVersion\n          ? {\n              datasetId: finalVersion.datasetId,\n              versionId: finalVersion.versionId,\n            }\n          : undefined,\n      })\n    } catch (error) {\n      if (finalVersion) {\n        throw createPipelineBookkeepingError({\n          pipelineId: pipeline.id,\n          runId: job.id,\n          version: finalVersion,\n          cause: error,\n        })\n      }\n      throw error\n    }\n    finished = true\n\n    return {\n      run: {\n        ...run,\n        finishedAt: requireFinishedAt(job.id, run.finishedAt),\n      },\n      steps,\n      version: finalVersion,\n    }\n  } catch (error) {\n    if (startedRun && !finished) {\n      const status = statusForFailure(signal, error)\n      try {\n        const run = await runtime.pipelineRunsStorage.finish({\n          projectId: runtime.id,\n          id: job.id,\n          status,\n          error: toPipelineRunFailure(error),\n        })\n        if (status === \"failed\" && run.status === \"failed\") input.onRunFailed?.(error, run)\n      } catch {\n        // The run did not transition to the requested terminal status.\n      }\n    }\n\n    throw error\n  } finally {\n    await logSession.flush()\n  }\n}\n",
+    "import type { DatasetDefinition, PipelineDefinition } from \"@sixb/core\"\nimport type { DatasetVersion } from \"@sixb/core/lake-storage\"\nimport type { PipelineRunFailure, PipelineRunStatus } from \"@sixb/core/storage\"\nimport type { PipelineJob } from \"./types\"\n\nexport class PipelineWorkerError extends Error {\n  readonly name = \"PipelineWorkerError\"\n}\n\nexport function toPipelineRunFailure(error: unknown): PipelineRunFailure {\n  if (error instanceof Error) {\n    return {\n      name: error.name,\n      message: error.message,\n    }\n  }\n\n  return {\n    message: String(error),\n  }\n}\n\nexport function throwIfAborted(signal: AbortSignal): void {\n  if (signal.aborted) {\n    throw createAbortError()\n  }\n}\n\nexport function statusForFailure(\n  signal: AbortSignal,\n  error: unknown\n): Extract<PipelineRunStatus, \"failed\" | \"cancelled\"> {\n  return signal.aborted || isAbortError(error) ? \"cancelled\" : \"failed\"\n}\n\nexport function requireFinishedAt(runId: string, finishedAt: Date | undefined): Date {\n  if (finishedAt) {\n    return finishedAt\n  }\n\n  throw new PipelineWorkerError(\n    `[SixbPipelineWorker] Pipeline run '${runId}' finished without a finishedAt timestamp.`\n  )\n}\n\nexport function createStepBookkeepingError(options: {\n  readonly pipelineId: string\n  readonly stepId: string\n  readonly runId: string\n  readonly version: DatasetVersion\n  readonly cause: unknown\n}): Error {\n  // The dataset version is already durable; retrying the whole pipeline could duplicate appends.\n  return new PipelineWorkerError(\n    `[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' committed dataset version '${options.version.versionId}', but failed to finalize step run '${options.runId}'. The dataset commit may already have succeeded and the step run record may need repair.`,\n    { cause: options.cause }\n  )\n}\n\nexport function createPipelineBookkeepingError(options: {\n  readonly pipelineId: string\n  readonly runId: string\n  readonly version: DatasetVersion\n  readonly cause: unknown\n}): Error {\n  return new PipelineWorkerError(\n    `[SixbPipelineWorker] Pipeline '${options.pipelineId}' committed final dataset version '${options.version.versionId}', but failed to finalize pipeline run '${options.runId}'. The dataset commit may already have succeeded and the pipeline run record may need repair.`,\n    { cause: options.cause }\n  )\n}\n\nexport function requirePipeline(\n  pipeline: PipelineDefinition | null,\n  job: PipelineJob\n): PipelineDefinition {\n  if (!pipeline) {\n    throw new PipelineWorkerError(`[SixbPipelineWorker] Unknown pipeline '${job.pipelineId}'.`)\n  }\n\n  return pipeline\n}\n\nexport function requireRegisteredDataset(options: {\n  readonly dataset: DatasetDefinition | null\n  readonly pipelineId: string\n  readonly stepId: string\n  readonly role: \"input\" | \"output\"\n  readonly name?: string\n  readonly datasetId: string\n}): DatasetDefinition {\n  if (options.dataset) {\n    return options.dataset\n  }\n\n  const namedRole = options.role === \"input\" ? `input '${options.name ?? \"unknown\"}'` : \"output\"\n\n  throw new PipelineWorkerError(\n    `[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' ${namedRole} references unknown dataset '${options.datasetId}'.`\n  )\n}\n\nfunction createAbortError(): Error {\n  const error = new Error(\"Worker runtime aborted.\")\n  error.name = \"AbortError\"\n  return error\n}\n\nfunction isAbortError(error: unknown): boolean {\n  return error instanceof Error && error.name === \"AbortError\"\n}\n",
+    "import type {\n  DatasetDefinition,\n  PipelineDefinition,\n  PipelineStepDefinition,\n  PipelineStepInput,\n  PipelineStepRunContext,\n} from \"@sixb/core\"\nimport type {\n  DatasetVersion,\n  DatasetVersionRef,\n  ReadDatasetRowsInput,\n} from \"@sixb/core/lake-storage\"\nimport { PipelineWorkerError, requireRegisteredDataset } from \"./errors\"\nimport type { PipelineWorkerContext } from \"./types\"\n\nexport interface ResolvedStepInput {\n  readonly name: string\n  readonly dataset: DatasetDefinition\n  readonly version: DatasetVersion\n  readonly ref: DatasetVersionRef\n}\n\nexport async function resolveStepInputs(input: {\n  readonly runtime: PipelineWorkerContext\n  readonly pipeline: PipelineDefinition\n  readonly step: PipelineStepDefinition\n}): Promise<readonly ResolvedStepInput[]> {\n  const { runtime, pipeline, step } = input\n  const resolved: ResolvedStepInput[] = []\n\n  for (const [name, declaredDataset] of Object.entries(step.inputs)) {\n    const dataset = requireRegisteredDataset({\n      dataset: runtime.getDatasetById(declaredDataset.id),\n      pipelineId: pipeline.id,\n      stepId: step.id,\n      role: \"input\",\n      name,\n      datasetId: declaredDataset.id,\n    })\n\n    const version = await runtime.lakeStorage.getLatestVersion(dataset.id)\n    if (!version) {\n      throw new PipelineWorkerError(\n        `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' input '${name}' dataset '${dataset.id}' has no committed version.`\n      )\n    }\n\n    resolved.push({\n      name,\n      dataset,\n      version,\n      ref: {\n        datasetId: version.datasetId,\n        versionId: version.versionId,\n      },\n    })\n  }\n\n  return resolved\n}\n\nexport function createStepInputs(\n  lakeStorage: PipelineWorkerContext[\"lakeStorage\"],\n  resolved: readonly ResolvedStepInput[]\n): PipelineStepRunContext[\"inputs\"] {\n  const inputs: Record<string, PipelineStepInput> = {}\n\n  for (const input of resolved) {\n    inputs[input.name] = {\n      dataset: input.dataset,\n      version: input.version,\n      readRows(readInput: Omit<ReadDatasetRowsInput, \"datasetId\" | \"versionId\"> = {}) {\n        // Step readers stay pinned even if newer versions commit while the handler runs.\n        return lakeStorage.readRows({\n          ...readInput,\n          datasetId: input.dataset.id,\n          versionId: input.version.versionId,\n        })\n      },\n    }\n  }\n\n  return inputs\n}\n",
+    "import type {\n  DatasetDefinition,\n  DatasetRow,\n  PipelineDefinition,\n  PipelineStepDefinition,\n  PipelineStepRunContext,\n} from \"@sixb/core\"\nimport type { DatasetVersion } from \"@sixb/core/lake-storage\"\nimport { PipelineWorkerError, throwIfAborted } from \"./errors\"\nimport { createStepInputs, type ResolvedStepInput } from \"./step-inputs\"\nimport type { PipelineJob, PipelineLogSession, PipelineWorkerContext } from \"./types\"\n\nexport async function executeRunStep(input: {\n  readonly runtime: PipelineWorkerContext\n  readonly pipeline: PipelineDefinition\n  readonly step: PipelineStepDefinition\n  readonly job: PipelineJob\n  readonly signal: AbortSignal\n  readonly logSession: PipelineLogSession\n  readonly outputDataset: DatasetDefinition\n  readonly resolvedInputs: readonly ResolvedStepInput[]\n}): Promise<{\n  readonly version: DatasetVersion\n  readonly versionCreated: boolean\n  readonly rowsWritten: number\n}> {\n  const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input\n\n  if (step.executor.kind !== \"run\") {\n    throw new PipelineWorkerError(\n      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses SQL execution, which is not supported by the run step executor.`\n    )\n  }\n\n  const logger = input.logSession.withContext({ stepId: step.id })\n  let rowsWritten = 0\n  const write = await runtime.lakeStorage.beginWrite({\n    dataset: outputDataset,\n    mode: step.mode,\n    inputs: resolvedInputs.map((resolved) => resolved.ref),\n    producer: {\n      kind: \"pipeline\",\n      id: pipeline.id,\n      runId: job.id,\n      stepId: step.id,\n    },\n  })\n\n  try {\n    const context: PipelineStepRunContext = {\n      projectId: runtime.id,\n      pipelineId: pipeline.id,\n      stepId: step.id,\n      runId: job.id,\n      signal,\n      logger,\n      inputs: createStepInputs(runtime.lakeStorage, resolvedInputs),\n      output: {\n        async writeRows(rows) {\n          await write.writeRows(\n            countRowsWritten(rows, signal, () => {\n              rowsWritten += 1\n            })\n          )\n        },\n      },\n    }\n\n    await step.executor.handler(context)\n    throwIfAborted(signal)\n\n    const commit = await write.commit({\n      commitMessage: `pipeline ${pipeline.id} step ${step.id} run ${job.id}`,\n    })\n    const { outcome, ...version } = commit\n\n    return {\n      version,\n      versionCreated: outcome === \"created\",\n      rowsWritten,\n    }\n  } catch (error) {\n    await write.abort().catch(() => {})\n    throw error\n  }\n}\n\nasync function* countRowsWritten(\n  rows: Iterable<DatasetRow> | AsyncIterable<DatasetRow>,\n  signal: AbortSignal,\n  onWritten: () => void\n): AsyncIterable<DatasetRow> {\n  for await (const row of rows) {\n    throwIfAborted(signal)\n    yield row\n    onWritten()\n  }\n}\n",
+    "import type {\n  DatasetDefinition,\n  LakeStorage,\n  LakeStorageWithSql,\n  PipelineDefinition,\n  PipelineStepDefinition,\n} from \"@sixb/core\"\nimport type {\n  DatasetVersion,\n  DatasetWriteMode,\n  LakeSqlTransformCapabilities,\n} from \"@sixb/core/lake-storage\"\nimport { PipelineWorkerError, throwIfAborted } from \"./errors\"\nimport type { ResolvedStepInput } from \"./step-inputs\"\nimport type { PipelineJob, PipelineWorkerContext } from \"./types\"\n\nfunction hasSqlExecutor(lakeStorage: LakeStorage): lakeStorage is LakeStorageWithSql {\n  const sql = \"sql\" in lakeStorage ? lakeStorage.sql : null\n\n  return (\n    typeof sql === \"object\" && sql !== null && \"execute\" in sql && typeof sql.execute === \"function\"\n  )\n}\n\nfunction assertSqlModeSupported(input: {\n  readonly capabilities: LakeSqlTransformCapabilities\n  readonly dialect: string\n  readonly mode: DatasetWriteMode\n  readonly pipelineId: string\n  readonly stepId: string\n}): void {\n  const supported =\n    input.mode === \"append\"\n      ? input.capabilities.supportsAppend\n      : input.capabilities.supportsSnapshot\n  if (supported) return\n\n  throw new PipelineWorkerError(\n    `[SixbPipelineWorker] Pipeline '${input.pipelineId}' step '${input.stepId}' writes in ` +\n      `'${input.mode}' mode, which the ${input.dialect} SQL executor does not support.`\n  )\n}\n\nexport async function executeSqlStep(input: {\n  readonly runtime: PipelineWorkerContext\n  readonly pipeline: PipelineDefinition\n  readonly step: PipelineStepDefinition\n  readonly job: PipelineJob\n  readonly signal: AbortSignal\n  readonly outputDataset: DatasetDefinition\n  readonly resolvedInputs: readonly ResolvedStepInput[]\n}): Promise<{\n  readonly version: DatasetVersion\n  readonly versionCreated: boolean\n  readonly rowsWritten?: number\n}> {\n  const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input\n\n  if (step.executor.kind !== \"sql\") {\n    throw new PipelineWorkerError(\n      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses run execution, which is not supported by the SQL step executor.`\n    )\n  }\n\n  if (!hasSqlExecutor(runtime.lakeStorage)) {\n    throw new PipelineWorkerError(\n      `[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' requires SQL transform support, but lake storage does not provide lakeStorage.sql.execute(...).`\n    )\n  }\n\n  // `LakeSqlExecutor.capabilities` is a required field on the contract that nothing read,\n  // so a provider declaring `supportsAppend: false` was asked to be honest and then\n  // ignored — the unsupported mode failed somewhere inside the provider instead, in\n  // whatever words that provider happened to use.\n  assertSqlModeSupported({\n    capabilities: runtime.lakeStorage.sql.capabilities,\n    dialect: runtime.lakeStorage.sql.dialect,\n    mode: step.mode,\n    pipelineId: pipeline.id,\n    stepId: step.id,\n  })\n\n  throwIfAborted(signal)\n\n  const commit = await runtime.lakeStorage.sql.execute({\n    sources: Object.fromEntries(\n      resolvedInputs.map((input) => [\n        input.name,\n        {\n          dataset: input.dataset,\n          versionId: input.version.versionId,\n        },\n      ])\n    ),\n    sql: step.executor.sql,\n    target: outputDataset,\n    mode: step.mode,\n    producer: {\n      kind: \"pipeline\",\n      id: pipeline.id,\n      runId: job.id,\n      stepId: step.id,\n    },\n    commitMessage: `pipeline ${pipeline.id} step ${step.id} run ${job.id}`,\n  })\n\n  throwIfAborted(signal)\n  const { outcome, ...version } = commit\n\n  return {\n    version,\n    versionCreated: outcome === \"created\",\n    rowsWritten: version.rowCount,\n  }\n}\n",
+    "import type { DatasetDefinition, PipelineDefinition, PipelineStepDefinition } from \"@sixb/core\"\nimport type { DatasetVersion } from \"@sixb/core/lake-storage\"\nimport type { PipelineStepRunRecord } from \"@sixb/core/storage\"\nimport {\n  createStepBookkeepingError,\n  requireRegisteredDataset,\n  statusForFailure,\n  throwIfAborted,\n  toPipelineRunFailure,\n} from \"./errors\"\nimport { executeRunStep } from \"./execute-run-step\"\nimport { executeSqlStep } from \"./execute-sql-step\"\nimport { type ResolvedStepInput, resolveStepInputs } from \"./step-inputs\"\nimport type {\n  PipelineJob,\n  PipelineLogSession,\n  PipelineStepFinishedHandler,\n  PipelineStepRunResult,\n  PipelineStepStartedHandler,\n  PipelineWorkerContext,\n} from \"./types\"\n\ntype StepExecutionResult = {\n  readonly version: DatasetVersion\n  readonly versionCreated: boolean\n  readonly rowsWritten?: number\n}\n\nexport async function runStep(input: {\n  readonly runtime: PipelineWorkerContext\n  readonly pipeline: PipelineDefinition\n  readonly step: PipelineStepDefinition\n  readonly stepIndex: number\n  readonly job: PipelineJob\n  readonly signal: AbortSignal\n  readonly logSession: PipelineLogSession\n  readonly onStepStarted?: PipelineStepStartedHandler\n  readonly onStepFinished?: PipelineStepFinishedHandler\n}): Promise<PipelineStepRunResult> {\n  const { runtime, pipeline, step, stepIndex, job, signal } = input\n  const lifecycleContext = {\n    stepIndex,\n    totalSteps: pipeline.graph.nodes.length,\n  }\n\n  throwIfAborted(signal)\n  const resolvedInputs = await resolveStepInputs({ runtime, pipeline, step })\n  const inputRefs = resolvedInputs.map((resolved) => resolved.ref)\n  const outputDataset = requireRegisteredDataset({\n    dataset: runtime.getDatasetById(step.output.id),\n    pipelineId: pipeline.id,\n    stepId: step.id,\n    role: \"output\",\n    datasetId: step.output.id,\n  })\n\n  await runtime.lakeStorage.createDataset(outputDataset)\n\n  const stepRunId = `${job.id}:step:${stepIndex + 1}:${step.id}`\n  const startedStepRun = await runtime.pipelineRunsStorage.startStep({\n    projectId: runtime.id,\n    id: stepRunId,\n    pipelineRunId: job.id,\n    pipelineId: pipeline.id,\n    stepId: step.id,\n    datasetId: outputDataset.id,\n    mode: step.mode,\n    inputs: inputRefs,\n  })\n  await input.onStepStarted?.(startedStepRun, lifecycleContext)\n\n  let committedVersion: DatasetVersion | undefined\n  let rowsWritten: number | undefined\n\n  try {\n    const execution = await executeStep({\n      runtime,\n      pipeline,\n      step,\n      job,\n      signal,\n      logSession: input.logSession,\n      outputDataset,\n      resolvedInputs,\n    })\n    committedVersion = execution.version\n    rowsWritten = execution.rowsWritten\n\n    let stepRun: PipelineStepRunRecord\n    try {\n      stepRun = await runtime.pipelineRunsStorage.finishStep({\n        projectId: runtime.id,\n        id: stepRunId,\n        status: \"succeeded\",\n        output: {\n          datasetId: committedVersion.datasetId,\n          versionId: committedVersion.versionId,\n        },\n        rowsWritten,\n      })\n    } catch (error) {\n      throw createStepBookkeepingError({\n        pipelineId: pipeline.id,\n        stepId: step.id,\n        runId: stepRunId,\n        version: committedVersion,\n        cause: error,\n      })\n    }\n    await input.onStepFinished?.(stepRun, lifecycleContext)\n\n    return {\n      run: stepRun,\n      version: committedVersion,\n      versionCreated: execution.versionCreated,\n    }\n  } catch (error) {\n    if (!committedVersion) {\n      const failedStepRun = await runtime.pipelineRunsStorage\n        .finishStep({\n          projectId: runtime.id,\n          id: stepRunId,\n          status: statusForFailure(signal, error),\n          rowsWritten,\n          error: toPipelineRunFailure(error),\n        })\n        .catch(() => null)\n\n      if (failedStepRun) {\n        await input.onStepFinished?.(failedStepRun, lifecycleContext)\n      }\n    }\n\n    throw error\n  }\n}\n\nasync function executeStep(input: {\n  readonly runtime: PipelineWorkerContext\n  readonly pipeline: PipelineDefinition\n  readonly step: PipelineStepDefinition\n  readonly job: PipelineJob\n  readonly signal: AbortSignal\n  readonly logSession: PipelineLogSession\n  readonly outputDataset: DatasetDefinition\n  readonly resolvedInputs: readonly ResolvedStepInput[]\n}): Promise<StepExecutionResult> {\n  switch (input.step.executor.kind) {\n    case \"run\":\n      return executeRunStep(input)\n    case \"sql\":\n      return executeSqlStep(input)\n  }\n}\n"
+  ],
+  "mappings": ";;AAAA;AAEA;;;ACMA,IAAM,SAAS;AAEf,eAAsB,sBAAsB,CAC1C,QACA,KACe;AAAA,EACf,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,YAAY,IAAI;AAAA,UAChB,OAAO,IAAI;AAAA,UACX,WAAW,IAAI,UAAU,YAAY;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,0BAA0B,CAC9C,QACA,MAIA,SACe;AAAA,EACf,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,YAAY,KAAK;AAAA,UACjB,OAAO,KAAK;AAAA,UACZ,WAAW,KAAK;AAAA,UAChB,QAAQ,KAAK;AAAA,UACb,WAAW,QAAQ;AAAA,UACnB,YAAY,QAAQ;AAAA,UACpB,WAAW,KAAK;AAAA,UAChB,WAAW,KAAK,UAAU,YAAY;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,2BAA2B,CAC/C,QACA,MAaA,SACe;AAAA,EACf,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,YAAY,KAAK;AAAA,UACjB,OAAO,KAAK;AAAA,UACZ,WAAW,KAAK;AAAA,UAChB,QAAQ,KAAK;AAAA,UACb,WAAW,QAAQ;AAAA,UACnB,YAAY,QAAQ;AAAA,UACpB,WAAW,KAAK;AAAA,UAChB,QAAQ,sBAAsB,KAAK,QAAQ,sBAAsB,KAAK,KAAK;AAAA,UAC3E,YAAY,kBAAkB,KAAK,IAAI,KAAK,UAAU,EAAE,YAAY;AAAA,aAChE,KAAK,SAAS,EAAE,WAAW,KAAK,OAAO,UAAU,IAAI,CAAC;AAAA,aACtD,KAAK,gBAAgB,YAAY,EAAE,aAAa,KAAK,YAAY,IAAI,CAAC;AAAA,aACtE,KAAK,QAAQ,EAAE,OAAO,KAAK,MAAM,QAAQ,IAAI,CAAC;AAAA,QACpD;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,2BAA2B,CAC/C,QACA,KACA,MACe;AAAA,EACf,IAAI,CAAC,KAAK;AAAA,IAAgB;AAAA,EAG1B,MAAM,QAAQ,KACZ;AAAA,IACE,QAAQ;AAAA,MACN;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,WAAW,KAAK,QAAQ;AAAA,UACxB,WAAW,KAAK,QAAQ;AAAA,UACxB,WAAW,KAAK,QAAQ,UAAU,YAAY;AAAA,UAC9C,UAAU;AAAA,YACR,MAAM;AAAA,YACN,IAAI,IAAI;AAAA,YACR,OAAO,IAAI;AAAA,YACX,QAAQ,KAAK,IAAI;AAAA,UACnB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,GACA,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,eAAsB,uBAAuB,CAC3C,QACA,KAOe;AAAA,EACf,MAAM,QAAQ,KACZ,EAAE,QAAQ,CAAC,EAAE,MAAM,yBAAyB,SAAS,gCAAgC,GAAG,EAAE,CAAC,EAAE,GAC7F,EAAE,QAAQ,OAAO,CACnB;AAAA;AAGF,SAAS,+BAA+B,CAAC,KAMtC;AAAA,EACD,OAAO;AAAA,IACL,YAAY,IAAI;AAAA,IAChB,OAAO,IAAI;AAAA,IACX,QAAQ,IAAI;AAAA,OACR,IAAI,aAAa,IAAI,YACrB;AAAA,MACE,WAAW,IAAI;AAAA,MACf,WAAW,IAAI;AAAA,IACjB,IACA,CAAC;AAAA,EACP;AAAA;AAGF,SAAS,iBAAiB,CAAC,WAAmB,YAAoC;AAAA,EAChF,IAAI,YAAY;AAAA,IACd,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,MAAM,2CAA2C,+BAA+B;AAAA;AAG5F,SAAS,qBAAqB,CAC5B,QACA,SACuC;AAAA,EACvC,IAAI,WAAW,WAAW;AAAA,IACxB,MAAM,IAAI,MAAM,wBAAwB,2BAA2B;AAAA,EACrE;AAAA,EAEA,OAAO;AAAA;;;AC3LT;;;ACKO,MAAM,4BAA4B,MAAM;AAAA,EACpC,OAAO;AAClB;AAEO,SAAS,oBAAoB,CAAC,OAAoC;AAAA,EACvE,IAAI,iBAAiB,OAAO;AAAA,IAC1B,OAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,OAAO;AAAA,IACL,SAAS,OAAO,KAAK;AAAA,EACvB;AAAA;AAGK,SAAS,cAAc,CAAC,QAA2B;AAAA,EACxD,IAAI,OAAO,SAAS;AAAA,IAClB,MAAM,iBAAiB;AAAA,EACzB;AAAA;AAGK,SAAS,gBAAgB,CAC9B,QACA,OACoD;AAAA,EACpD,OAAO,OAAO,WAAW,aAAa,KAAK,IAAI,cAAc;AAAA;AAGxD,SAAS,kBAAiB,CAAC,OAAe,YAAoC;AAAA,EACnF,IAAI,YAAY;AAAA,IACd,OAAO;AAAA,EACT;AAAA,EAEA,MAAM,IAAI,oBACR,sCAAsC,iDACxC;AAAA;AAGK,SAAS,0BAA0B,CAAC,SAMjC;AAAA,EAER,OAAO,IAAI,oBACT,kCAAkC,QAAQ,qBAAqB,QAAQ,sCAAsC,QAAQ,QAAQ,gDAAgD,QAAQ,kGACrL,EAAE,OAAO,QAAQ,MAAM,CACzB;AAAA;AAGK,SAAS,8BAA8B,CAAC,SAKrC;AAAA,EACR,OAAO,IAAI,oBACT,kCAAkC,QAAQ,gDAAgD,QAAQ,QAAQ,oDAAoD,QAAQ,sGACtK,EAAE,OAAO,QAAQ,MAAM,CACzB;AAAA;AAGK,SAAS,eAAe,CAC7B,UACA,KACoB;AAAA,EACpB,IAAI,CAAC,UAAU;AAAA,IACb,MAAM,IAAI,oBAAoB,0CAA0C,IAAI,cAAc;AAAA,EAC5F;AAAA,EAEA,OAAO;AAAA;AAGF,SAAS,wBAAwB,CAAC,SAOnB;AAAA,EACpB,IAAI,QAAQ,SAAS;AAAA,IACnB,OAAO,QAAQ;AAAA,EACjB;AAAA,EAEA,MAAM,YAAY,QAAQ,SAAS,UAAU,UAAU,QAAQ,QAAQ,eAAe;AAAA,EAEtF,MAAM,IAAI,oBACR,kCAAkC,QAAQ,qBAAqB,QAAQ,WAAW,yCAAyC,QAAQ,aACrI;AAAA;AAGF,SAAS,gBAAgB,GAAU;AAAA,EACjC,MAAM,QAAQ,IAAI,MAAM,yBAAyB;AAAA,EACjD,MAAM,OAAO;AAAA,EACb,OAAO;AAAA;AAGT,SAAS,YAAY,CAAC,OAAyB;AAAA,EAC7C,OAAO,iBAAiB,SAAS,MAAM,SAAS;AAAA;;;ACtFlD,eAAsB,iBAAiB,CAAC,OAIE;AAAA,EACxC,QAAQ,SAAS,UAAU,SAAS;AAAA,EACpC,MAAM,WAAgC,CAAC;AAAA,EAEvC,YAAY,MAAM,oBAAoB,OAAO,QAAQ,KAAK,MAAM,GAAG;AAAA,IACjE,MAAM,UAAU,yBAAyB;AAAA,MACvC,SAAS,QAAQ,eAAe,gBAAgB,EAAE;AAAA,MAClD,YAAY,SAAS;AAAA,MACrB,QAAQ,KAAK;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,WAAW,gBAAgB;AAAA,IAC7B,CAAC;AAAA,IAED,MAAM,UAAU,MAAM,QAAQ,YAAY,iBAAiB,QAAQ,EAAE;AAAA,IACrE,IAAI,CAAC,SAAS;AAAA,MACZ,MAAM,IAAI,oBACR,kCAAkC,SAAS,aAAa,KAAK,cAAc,kBAAkB,QAAQ,+BACvG;AAAA,IACF;AAAA,IAEA,SAAS,KAAK;AAAA,MACZ;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK;AAAA,QACH,WAAW,QAAQ;AAAA,QACnB,WAAW,QAAQ;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,OAAO;AAAA;AAGF,SAAS,gBAAgB,CAC9B,aACA,UACkC;AAAA,EAClC,MAAM,SAA4C,CAAC;AAAA,EAEnD,WAAW,SAAS,UAAU;AAAA,IAC5B,OAAO,MAAM,QAAQ;AAAA,MACnB,SAAS,MAAM;AAAA,MACf,SAAS,MAAM;AAAA,MACf,QAAQ,CAAC,YAAmE,CAAC,GAAG;AAAA,QAE9E,OAAO,YAAY,SAAS;AAAA,aACvB;AAAA,UACH,WAAW,MAAM,QAAQ;AAAA,UACzB,WAAW,MAAM,QAAQ;AAAA,QAC3B,CAAC;AAAA;AAAA,IAEL;AAAA,EACF;AAAA,EAEA,OAAO;AAAA;;;ACtET,eAAsB,cAAc,CAAC,OAalC;AAAA,EACD,QAAQ,SAAS,UAAU,MAAM,KAAK,QAAQ,eAAe,mBAAmB;AAAA,EAEhF,IAAI,KAAK,SAAS,SAAS,OAAO;AAAA,IAChC,MAAM,IAAI,oBACR,kCAAkC,SAAS,aAAa,KAAK,0EAC/D;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,MAAM,WAAW,YAAY,EAAE,QAAQ,KAAK,GAAG,CAAC;AAAA,EAC/D,IAAI,cAAc;AAAA,EAClB,MAAM,QAAQ,MAAM,QAAQ,YAAY,WAAW;AAAA,IACjD,SAAS;AAAA,IACT,MAAM,KAAK;AAAA,IACX,QAAQ,eAAe,IAAI,CAAC,aAAa,SAAS,GAAG;AAAA,IACrD,UAAU;AAAA,MACR,MAAM;AAAA,MACN,IAAI,SAAS;AAAA,MACb,OAAO,IAAI;AAAA,MACX,QAAQ,KAAK;AAAA,IACf;AAAA,EACF,CAAC;AAAA,EAED,IAAI;AAAA,IACF,MAAM,UAAkC;AAAA,MACtC,WAAW,QAAQ;AAAA,MACnB,YAAY,SAAS;AAAA,MACrB,QAAQ,KAAK;AAAA,MACb,OAAO,IAAI;AAAA,MACX;AAAA,MACA;AAAA,MACA,QAAQ,iBAAiB,QAAQ,aAAa,cAAc;AAAA,MAC5D,QAAQ;AAAA,aACA,UAAS,CAAC,MAAM;AAAA,UACpB,MAAM,MAAM,UACV,iBAAiB,MAAM,QAAQ,MAAM;AAAA,YACnC,eAAe;AAAA,WAChB,CACH;AAAA;AAAA,MAEJ;AAAA,IACF;AAAA,IAEA,MAAM,KAAK,SAAS,QAAQ,OAAO;AAAA,IACnC,eAAe,MAAM;AAAA,IAErB,MAAM,SAAS,MAAM,MAAM,OAAO;AAAA,MAChC,eAAe,YAAY,SAAS,WAAW,KAAK,UAAU,IAAI;AAAA,IACpE,CAAC;AAAA,IACD,QAAQ,YAAY,YAAY;AAAA,IAEhC,OAAO;AAAA,MACL;AAAA,MACA,gBAAgB,YAAY;AAAA,MAC5B;AAAA,IACF;AAAA,IACA,OAAO,OAAO;AAAA,IACd,MAAM,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,IAClC,MAAM;AAAA;AAAA;AAIV,gBAAgB,gBAAgB,CAC9B,MACA,QACA,WAC2B;AAAA,EAC3B,iBAAiB,OAAO,MAAM;AAAA,IAC5B,eAAe,MAAM;AAAA,IACrB,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA;;;AChFF,SAAS,cAAc,CAAC,aAA6D;AAAA,EACnF,MAAM,MAAM,SAAS,cAAc,YAAY,MAAM;AAAA,EAErD,OACE,OAAO,QAAQ,YAAY,QAAQ,QAAQ,aAAa,OAAO,OAAO,IAAI,YAAY;AAAA;AAI1F,SAAS,sBAAsB,CAAC,OAMvB;AAAA,EACP,MAAM,YACJ,MAAM,SAAS,WACX,MAAM,aAAa,iBACnB,MAAM,aAAa;AAAA,EACzB,IAAI;AAAA,IAAW;AAAA,EAEf,MAAM,IAAI,oBACR,kCAAkC,MAAM,qBAAqB,MAAM,uBACjE,IAAI,MAAM,yBAAyB,MAAM,wCAC7C;AAAA;AAGF,eAAsB,cAAc,CAAC,OAYlC;AAAA,EACD,QAAQ,SAAS,UAAU,MAAM,KAAK,QAAQ,eAAe,mBAAmB;AAAA,EAEhF,IAAI,KAAK,SAAS,SAAS,OAAO;AAAA,IAChC,MAAM,IAAI,oBACR,kCAAkC,SAAS,aAAa,KAAK,0EAC/D;AAAA,EACF;AAAA,EAEA,IAAI,CAAC,eAAe,QAAQ,WAAW,GAAG;AAAA,IACxC,MAAM,IAAI,oBACR,kCAAkC,SAAS,aAAa,KAAK,qGAC/D;AAAA,EACF;AAAA,EAMA,uBAAuB;AAAA,IACrB,cAAc,QAAQ,YAAY,IAAI;AAAA,IACtC,SAAS,QAAQ,YAAY,IAAI;AAAA,IACjC,MAAM,KAAK;AAAA,IACX,YAAY,SAAS;AAAA,IACrB,QAAQ,KAAK;AAAA,EACf,CAAC;AAAA,EAED,eAAe,MAAM;AAAA,EAErB,MAAM,SAAS,MAAM,QAAQ,YAAY,IAAI,QAAQ;AAAA,IACnD,SAAS,OAAO,YACd,eAAe,IAAI,CAAC,WAAU;AAAA,MAC5B,OAAM;AAAA,MACN;AAAA,QACE,SAAS,OAAM;AAAA,QACf,WAAW,OAAM,QAAQ;AAAA,MAC3B;AAAA,IACF,CAAC,CACH;AAAA,IACA,KAAK,KAAK,SAAS;AAAA,IACnB,QAAQ;AAAA,IACR,MAAM,KAAK;AAAA,IACX,UAAU;AAAA,MACR,MAAM;AAAA,MACN,IAAI,SAAS;AAAA,MACb,OAAO,IAAI;AAAA,MACX,QAAQ,KAAK;AAAA,IACf;AAAA,IACA,eAAe,YAAY,SAAS,WAAW,KAAK,UAAU,IAAI;AAAA,EACpE,CAAC;AAAA,EAED,eAAe,MAAM;AAAA,EACrB,QAAQ,YAAY,YAAY;AAAA,EAEhC,OAAO;AAAA,IACL;AAAA,IACA,gBAAgB,YAAY;AAAA,IAC5B,aAAa,QAAQ;AAAA,EACvB;AAAA;;;ACrFF,eAAsB,OAAO,CAAC,OAUK;AAAA,EACjC,QAAQ,SAAS,UAAU,MAAM,WAAW,KAAK,WAAW;AAAA,EAC5D,MAAM,mBAAmB;AAAA,IACvB;AAAA,IACA,YAAY,SAAS,MAAM,MAAM;AAAA,EACnC;AAAA,EAEA,eAAe,MAAM;AAAA,EACrB,MAAM,iBAAiB,MAAM,kBAAkB,EAAE,SAAS,UAAU,KAAK,CAAC;AAAA,EAC1E,MAAM,YAAY,eAAe,IAAI,CAAC,aAAa,SAAS,GAAG;AAAA,EAC/D,MAAM,gBAAgB,yBAAyB;AAAA,IAC7C,SAAS,QAAQ,eAAe,KAAK,OAAO,EAAE;AAAA,IAC9C,YAAY,SAAS;AAAA,IACrB,QAAQ,KAAK;AAAA,IACb,MAAM;AAAA,IACN,WAAW,KAAK,OAAO;AAAA,EACzB,CAAC;AAAA,EAED,MAAM,QAAQ,YAAY,cAAc,aAAa;AAAA,EAErD,MAAM,YAAY,GAAG,IAAI,WAAW,YAAY,KAAK,KAAK;AAAA,EAC1D,MAAM,iBAAiB,MAAM,QAAQ,oBAAoB,UAAU;AAAA,IACjE,WAAW,QAAQ;AAAA,IACnB,IAAI;AAAA,IACJ,eAAe,IAAI;AAAA,IACnB,YAAY,SAAS;AAAA,IACrB,QAAQ,KAAK;AAAA,IACb,WAAW,cAAc;AAAA,IACzB,MAAM,KAAK;AAAA,IACX,QAAQ;AAAA,EACV,CAAC;AAAA,EACD,MAAM,MAAM,gBAAgB,gBAAgB,gBAAgB;AAAA,EAE5D,IAAI;AAAA,EACJ,IAAI;AAAA,EAEJ,IAAI;AAAA,IACF,MAAM,YAAY,MAAM,YAAY;AAAA,MAClC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,MAAM;AAAA,MAClB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,IACD,mBAAmB,UAAU;AAAA,IAC7B,cAAc,UAAU;AAAA,IAExB,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,UAAU,MAAM,QAAQ,oBAAoB,WAAW;AAAA,QACrD,WAAW,QAAQ;AAAA,QACnB,IAAI;AAAA,QACJ,QAAQ;AAAA,QACR,QAAQ;AAAA,UACN,WAAW,iBAAiB;AAAA,UAC5B,WAAW,iBAAiB;AAAA,QAC9B;AAAA,QACA;AAAA,MACF,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,MAAM,2BAA2B;AAAA,QAC/B,YAAY,SAAS;AAAA,QACrB,QAAQ,KAAK;AAAA,QACb,OAAO;AAAA,QACP,SAAS;AAAA,QACT,OAAO;AAAA,MACT,CAAC;AAAA;AAAA,IAEH,MAAM,MAAM,iBAAiB,SAAS,gBAAgB;AAAA,IAEtD,OAAO;AAAA,MACL,KAAK;AAAA,MACL,SAAS;AAAA,MACT,gBAAgB,UAAU;AAAA,IAC5B;AAAA,IACA,OAAO,OAAO;AAAA,IACd,IAAI,CAAC,kBAAkB;AAAA,MACrB,MAAM,gBAAgB,MAAM,QAAQ,oBACjC,WAAW;AAAA,QACV,WAAW,QAAQ;AAAA,QACnB,IAAI;AAAA,QACJ,QAAQ,iBAAiB,QAAQ,KAAK;AAAA,QACtC;AAAA,QACA,OAAO,qBAAqB,KAAK;AAAA,MACnC,CAAC,EACA,MAAM,MAAM,IAAI;AAAA,MAEnB,IAAI,eAAe;AAAA,QACjB,MAAM,MAAM,iBAAiB,eAAe,gBAAgB;AAAA,MAC9D;AAAA,IACF;AAAA,IAEA,MAAM;AAAA;AAAA;AAIV,eAAe,WAAW,CAAC,OASM;AAAA,EAC/B,QAAQ,MAAM,KAAK,SAAS;AAAA,SACrB;AAAA,MACH,OAAO,eAAe,KAAK;AAAA,SACxB;AAAA,MACH,OAAO,eAAe,KAAK;AAAA;AAAA;;;ALzI1B,MAAM,uCAAuC,MAAM;AAAA,EAGnC;AAAA,EAFH,OAAO;AAAA,EAEzB,WAAW,CAAU,KAAwB;AAAA,IAC3C,MAAM,sCAAsC,IAAI,0BAA0B;AAAA,IADvD;AAAA;AAGvB;AAEA,eAAsB,cAAc,CAAC,OAAwD;AAAA,EAC3F,QAAQ,SAAS,QAAQ;AAAA,EACzB,MAAM,SAAS,MAAM,UAAU,IAAI,gBAAgB,EAAE;AAAA,EACrD,MAAM,WAAW,gBAAgB,QAAQ,gBAAgB,IAAI,UAAU,GAAG,GAAG;AAAA,EAC7E,MAAM,aAAa,mBAAmB,QAAQ,IAAI,QAAQ,IAAI,EAAE,eAAe;AAAA,IAC7E,MAAM;AAAA,IACN,IAAI,IAAI;AAAA,EACV,CAAC;AAAA,EACD,MAAM,QAAiC,CAAC;AAAA,EACxC,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI,WAAW;AAAA,EAEf,eAAe,MAAM;AAAA,EAErB,IAAI;AAAA,IACF,IAAI;AAAA,MACF,aAAa,MAAM,QAAQ,oBAAoB,MAAM;AAAA,QACnD,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,QACR,YAAY,SAAS;AAAA,MACvB,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,MAAM,WAAW,MAAM,QAAQ,oBAAoB,QAAQ;AAAA,QACzD,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,MACV,CAAC;AAAA,MACD,IAAI,UAAU,eAAe,SAAS,IAAI;AAAA,QACxC,MAAM,IAAI,+BAA+B,QAAQ;AAAA,MACnD;AAAA,MACA,MAAM;AAAA;AAAA,IAER,MAAM,MAAM,eAAe,UAAU;AAAA,IAErC,YAAY,WAAW,SAAS,SAAS,MAAM,MAAM,QAAQ,GAAG;AAAA,MAC9D,MAAM,aAAa,MAAM,QAAQ;AAAA,QAC/B;AAAA,QACA;AAAA,QACA,MAAM,KAAK;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,eAAe,MAAM;AAAA,QACrB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AAAA,MAED,MAAM,KAAK,UAAU;AAAA,MACrB,eAAe,WAAW;AAAA,MAE1B,MAAM,MAAM,kBAAkB,UAAU;AAAA,IAC1C;AAAA,IAEA,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,MAAM,MAAM,QAAQ,oBAAoB,OAAO;AAAA,QAC7C,WAAW,QAAQ;AAAA,QACnB,IAAI,IAAI;AAAA,QACR,QAAQ;AAAA,QACR,QAAQ,eACJ;AAAA,UACE,WAAW,aAAa;AAAA,UACxB,WAAW,aAAa;AAAA,QAC1B,IACA;AAAA,MACN,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,cAAc;AAAA,QAChB,MAAM,+BAA+B;AAAA,UACnC,YAAY,SAAS;AAAA,UACrB,OAAO,IAAI;AAAA,UACX,SAAS;AAAA,UACT,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,MACA,MAAM;AAAA;AAAA,IAER,WAAW;AAAA,IAEX,OAAO;AAAA,MACL,KAAK;AAAA,WACA;AAAA,QACH,YAAY,mBAAkB,IAAI,IAAI,IAAI,UAAU;AAAA,MACtD;AAAA,MACA;AAAA,MACA,SAAS;AAAA,IACX;AAAA,IACA,OAAO,OAAO;AAAA,IACd,IAAI,cAAc,CAAC,UAAU;AAAA,MAC3B,MAAM,SAAS,iBAAiB,QAAQ,KAAK;AAAA,MAC7C,IAAI;AAAA,QACF,MAAM,MAAM,MAAM,QAAQ,oBAAoB,OAAO;AAAA,UACnD,WAAW,QAAQ;AAAA,UACnB,IAAI,IAAI;AAAA,UACR;AAAA,UACA,OAAO,qBAAqB,KAAK;AAAA,QACnC,CAAC;AAAA,QACD,IAAI,WAAW,YAAY,IAAI,WAAW;AAAA,UAAU,MAAM,cAAc,OAAO,GAAG;AAAA,QAClF,MAAM;AAAA,IAGV;AAAA,IAEA,MAAM;AAAA,YACN;AAAA,IACA,MAAM,WAAW,MAAM;AAAA;AAAA;;;AF3GpB,MAAM,uBAAuB,YAA0C;AAAA,EAC3D;AAAA,EACA;AAAA,EAEjB,WAAW,CAAC,MAA0B;AAAA,IACpC,IAAI,KAAK,cAAc,EAAE,WAAW,GAAG;AAAA,MACrC,MAAM,IAAI,MAAM,8DAA8D;AAAA,IAChF;AAAA,IAEA,MAAM,sBAAsB,KAAK,QAAQ;AAAA,IACzC,IAAI,CAAC,qBAAqB;AAAA,MACxB,MAAM,IAAI,MAAM,6EAA6E;AAAA,IAC/F;AAAA,IAEA,MAAM;AAAA,MACJ,WAAW,KAAK;AAAA,MAChB,OAAO,KAAK,OAAO;AAAA,MACnB,UAAU,mBAAmB,KAAK;AAAA,IACpC,CAAC;AAAA,IAED,KAAK,UAAU,qBAAqB,MAAM,mBAAmB;AAAA,IAC7D,KAAK,OAAO;AAAA;AAAA,OAGE,QAAO,CACrB,SACA,QACe;AAAA,IACf,QAAQ,QAAQ;AAAA,IAChB,MAAM,cAA2B;AAAA,MAC/B,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,YAAY,IAAI,QAAQ;AAAA,IAC1B;AAAA,IAEA,IAAI;AAAA,IACJ,IAAI;AAAA,MACF,SAAS,MAAM,eAAe;AAAA,QAC5B,SAAS,KAAK;AAAA,QACd,KAAK;AAAA,QACL;AAAA,QACA,cAAc,CAAC,QAAQ,uBAAuB,KAAK,KAAK,QAAQ,GAAG;AAAA,QACnE,aAAa,CAAC,OAAO,QAAQ;AAAA,UAC3B,iBAAiB,KAAK,MAAM,OAAO;AAAA,YACjC,WAAW,KAAK,KAAK;AAAA,YACrB,YAAY,IAAI;AAAA,YAChB,SAAS,IAAI;AAAA,YACb,KAAK;AAAA,cACH,MAAM;AAAA,cACN,OAAO,IAAI;AAAA,cACX,YAAY,IAAI;AAAA,YAClB;AAAA,UACF,CAAC;AAAA;AAAA,QAEH,eAAe,CAAC,MAAM,YACpB,2BAA2B,KAAK,KAAK,QAAQ,MAAM,OAAO;AAAA,QAC5D,gBAAgB,CAAC,MAAM,YACrB,4BAA4B,KAAK,KAAK,QAAQ,MAAM,OAAO;AAAA,QAC7D,iBAAiB,CAAC,SAAS,4BAA4B,KAAK,KAAK,QAAQ,aAAa,IAAI;AAAA,MAC5F,CAAC;AAAA,MACD,OAAO,OAAO;AAAA,MACd,IAAI,iBAAiB;AAAA,QAAgC;AAAA,MACrD,MAAM;AAAA;AAAA,IAGR,MAAM,wBAAwB,KAAK,KAAK,QAAQ;AAAA,MAC9C,IAAI,YAAY;AAAA,MAChB,YAAY,YAAY;AAAA,MACxB,QAAQ;AAAA,MACR,WAAW,OAAO,SAAS;AAAA,MAC3B,WAAW,OAAO,SAAS;AAAA,IAC7B,CAAC;AAAA;AAAA,OAGsB,iBAAgB,CACvC,SACA,QACqC;AAAA,IACrC,QAAQ,QAAQ;AAAA,IAChB,MAAM,wBAAwB,KAAK,KAAK,QAAQ;AAAA,MAC9C,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,YAAY,IAAI,QAAQ;AAAA,MACxB,QAAQ;AAAA,IACV,CAAC;AAAA,IAID,OAAO,EAAE,MAAM,OAAO;AAAA;AAAA,OAGC,aAAY,CACnC,SACA,QACqC;AAAA,IACrC,QAAQ,QAAQ;AAAA,IAChB,MAAM,cAA2B;AAAA,MAC/B,IAAI,IAAI,QAAQ,SAAS,GAAG,IAAI,cAAc,IAAI;AAAA,MAClD,YAAY,IAAI,QAAQ;AAAA,IAC1B;AAAA,IAEA,IAAI,CAAE,MAAM,iBAAiB,KAAK,SAAS,YAAY,EAAE,GAAI;AAAA,MAC3D,OAAO,EAAE,MAAM,QAAQ;AAAA,IACzB;AAAA,IAGA,MAAM,wBAAwB,KAAK,KAAK,QAAQ;AAAA,MAC9C,IAAI,YAAY;AAAA,MAChB,YAAY,YAAY;AAAA,MACxB,QAAQ;AAAA,IACV,CAAC;AAAA,IAED,OAAO,EAAE,MAAM,OAAO;AAAA;AAE1B;AAEA,SAAS,oBAAoB,CAC3B,MACA,qBACuB;AAAA,EACvB,OAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT;AAAA,IACA,aAAa,KAAK;AAAA,IAClB,MAAM,KAAK;AAAA,IACX,eAAe,CAAC,YAAY;AAAA,MAC1B,OAAO,KAAK,gBAAgB,UAAU;AAAA;AAAA,IAExC,cAAc,CAAC,WAAW;AAAA,MACxB,OAAO,KAAK,eAAe,SAAS;AAAA;AAAA,EAExC;AAAA;AAGF,eAAe,gBAAgB,CAC7B,SACA,eACkB;AAAA,EAClB,MAAM,SAAS,MAAM,QAAQ,oBAAoB,UAAU;AAAA,IACzD,WAAW,QAAQ;AAAA,IACnB;AAAA,IACA,UAAU,CAAC,WAAW;AAAA,IACtB,OAAO;AAAA,EACT,CAAC;AAAA,EAED,OAAO,OAAO,MAAM,KAAK,CAAC,SAAS,QAAQ,KAAK,MAAM,CAAC;AAAA;",
+  "debugId": "068611D3BE49C18D64756E2164756E21",
+  "names": []
+}

package/dist/run-pipeline-job.d.ts

@@ -0,0 +1,9 @@
+import type { PipelineRunRecord } from "@sixb/core/storage";
+import type { PipelineRunResult, RunPipelineJobInput } from "./types";
+export declare class PipelineRunAlreadyStartedError extends Error {
+    readonly run: PipelineRunRecord;
+    readonly name = "PipelineRunAlreadyStartedError";
+    constructor(run: PipelineRunRecord);
+}
+export declare function runPipelineJob(input: RunPipelineJobInput): Promise<PipelineRunResult>;
+//# sourceMappingURL=run-pipeline-job.d.ts.map

package/dist/run-pipeline-job.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-pipeline-job.d.ts","sourceRoot":"","sources":["../src/run-pipeline-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AAU3D,OAAO,KAAK,EAAE,iBAAiB,EAAyB,mBAAmB,EAAE,MAAM,SAAS,CAAA;AAE5F,qBAAa,8BAA+B,SAAQ,KAAK;IAG3C,QAAQ,CAAC,GAAG,EAAE,iBAAiB;IAF3C,SAAkB,IAAI,oCAAmC;gBAEpC,GAAG,EAAE,iBAAiB;CAG5C;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CA2G3F"}

package/dist/run-step.d.ts

@@ -0,0 +1,14 @@
+import type { PipelineDefinition, PipelineStepDefinition } from "@sixb/core";
+import type { PipelineJob, PipelineLogSession, PipelineStepFinishedHandler, PipelineStepRunResult, PipelineStepStartedHandler, PipelineWorkerContext } from "./types";
+export declare function runStep(input: {
+    readonly runtime: PipelineWorkerContext;
+    readonly pipeline: PipelineDefinition;
+    readonly step: PipelineStepDefinition;
+    readonly stepIndex: number;
+    readonly job: PipelineJob;
+    readonly signal: AbortSignal;
+    readonly logSession: PipelineLogSession;
+    readonly onStepStarted?: PipelineStepStartedHandler;
+    readonly onStepFinished?: PipelineStepFinishedHandler;
+}): Promise<PipelineStepRunResult>;
+//# sourceMappingURL=run-step.d.ts.map

package/dist/run-step.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-step.d.ts","sourceRoot":"","sources":["../src/run-step.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAqB,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AAa/F,OAAO,KAAK,EACV,WAAW,EACX,kBAAkB,EAClB,2BAA2B,EAC3B,qBAAqB,EACrB,0BAA0B,EAC1B,qBAAqB,EACtB,MAAM,SAAS,CAAA;AAQhB,wBAAsB,OAAO,CAAC,KAAK,EAAE;IACnC,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAA;IACvC,QAAQ,CAAC,aAAa,CAAC,EAAE,0BAA0B,CAAA;IACnD,QAAQ,CAAC,cAAc,CAAC,EAAE,2BAA2B,CAAA;CACtD,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAiGjC"}

package/dist/step-inputs.d.ts

@@ -0,0 +1,16 @@
+import type { DatasetDefinition, PipelineDefinition, PipelineStepDefinition, PipelineStepRunContext } from "@sixb/core";
+import type { DatasetVersion, DatasetVersionRef } from "@sixb/core/lake-storage";
+import type { PipelineWorkerContext } from "./types";
+export interface ResolvedStepInput {
+    readonly name: string;
+    readonly dataset: DatasetDefinition;
+    readonly version: DatasetVersion;
+    readonly ref: DatasetVersionRef;
+}
+export declare function resolveStepInputs(input: {
+    readonly runtime: PipelineWorkerContext;
+    readonly pipeline: PipelineDefinition;
+    readonly step: PipelineStepDefinition;
+}): Promise<readonly ResolvedStepInput[]>;
+export declare function createStepInputs(lakeStorage: PipelineWorkerContext["lakeStorage"], resolved: readonly ResolvedStepInput[]): PipelineStepRunContext["inputs"];
+//# sourceMappingURL=step-inputs.d.ts.map

package/dist/step-inputs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"step-inputs.d.ts","sourceRoot":"","sources":["../src/step-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EAEtB,sBAAsB,EACvB,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EACV,cAAc,EACd,iBAAiB,EAElB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAEpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,GAAG,EAAE,iBAAiB,CAAA;CAChC;AAED,wBAAsB,iBAAiB,CAAC,KAAK,EAAE;IAC7C,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;CACtC,GAAG,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC,CAiCxC;AAED,wBAAgB,gBAAgB,CAC9B,WAAW,EAAE,qBAAqB,CAAC,aAAa,CAAC,EACjD,QAAQ,EAAE,SAAS,iBAAiB,EAAE,GACrC,sBAAsB,CAAC,QAAQ,CAAC,CAmBlC"}

package/dist/tsconfig.json

@@ -0,0 +1,5 @@
+// Written by scripts/build-package.ts, and published on purpose.
+// Bun applies the nearest tsconfig paths map to every import it resolves, node_modules
+// included. An empty map here keeps a consumer's aliases from sending a module in this
+// directory back into a second copy of its own source.
+{ "compilerOptions": { "paths": {} } }

package/dist/types.d.ts

@@ -0,0 +1,59 @@
+import type { DatasetDefinition, DomainEventLog, LakeStorage, PipelineDefinition, Queues, Storage } from "@sixb/core";
+import type { LogsRuntime } from "@sixb/core/internal/logging";
+import type { DatasetVersion } from "@sixb/core/lake-storage";
+import type { PipelineRunRecord, PipelineRunStorage, PipelineStepRunRecord } from "@sixb/core/storage";
+export type PipelineLogSession = ReturnType<LogsRuntime["startExecution"]>;
+export interface PipelineWorkerContext {
+    readonly id: string;
+    readonly pipelineRunsStorage: PipelineRunStorage;
+    readonly lakeStorage: LakeStorage;
+    readonly logs?: LogsRuntime;
+    getDatasetById(datasetId: string): DatasetDefinition | null;
+    getPipelineById(pipelineId: string): PipelineDefinition | null;
+}
+export interface PipelineWorkerSixb {
+    readonly id: string;
+    readonly events?: DomainEventLog;
+    readonly logs?: LogsRuntime;
+    readonly lakeStorage: LakeStorage;
+    readonly queues: Queues;
+    readonly storage: Storage;
+    listPipelines(): readonly PipelineDefinition[];
+    getPipelineById(pipelineId: string): PipelineDefinition | null;
+    getDatasetById(datasetId: string): DatasetDefinition | null;
+}
+export interface PipelineJob {
+    readonly id: string;
+    readonly pipelineId: string;
+}
+export interface RunPipelineJobInput {
+    readonly runtime: PipelineWorkerContext;
+    readonly job: PipelineJob;
+    readonly signal?: AbortSignal;
+    readonly onRunStarted?: PipelineRunStartedHandler;
+    readonly onRunFailed?: PipelineRunFailedHandler;
+    readonly onStepStarted?: PipelineStepStartedHandler;
+    readonly onStepFinished?: PipelineStepFinishedHandler;
+    readonly onStepCommitted?: PipelineStepCommittedHandler;
+}
+export type PipelineStepCommittedHandler = (result: PipelineStepRunResult) => Promise<void> | void;
+export type PipelineRunStartedHandler = (run: PipelineRunRecord) => Promise<void> | void;
+export type PipelineRunFailedHandler = (error: unknown, run: PipelineRunRecord) => void;
+export interface PipelineStepLifecycleContext {
+    readonly stepIndex: number;
+    readonly totalSteps: number;
+}
+export type PipelineStepStartedHandler = (step: PipelineStepRunRecord, context: PipelineStepLifecycleContext) => Promise<void> | void;
+export type PipelineStepFinishedHandler = (step: PipelineStepRunRecord, context: PipelineStepLifecycleContext) => Promise<void> | void;
+export interface PipelineStepRunResult {
+    readonly run: PipelineStepRunRecord;
+    readonly version: DatasetVersion;
+    /** True only when this step created the returned dataset version. */
+    readonly versionCreated: boolean;
+}
+export interface PipelineRunResult {
+    readonly run: PipelineRunRecord;
+    readonly steps: readonly PipelineStepRunResult[];
+    readonly version?: DatasetVersion;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EACjB,cAAc,EACd,WAAW,EACX,kBAAkB,EAClB,MAAM,EACN,OAAO,EACR,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAC7D,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAClB,qBAAqB,EACtB,MAAM,oBAAoB,CAAA;AAE3B,MAAM,MAAM,kBAAkB,GAAG,UAAU,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC,CAAA;AAE1E,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,mBAAmB,EAAE,kBAAkB,CAAA;IAChD,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAA;IAC3B,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,CAAA;IAC3D,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAAA;CAC/D;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,aAAa,IAAI,SAAS,kBAAkB,EAAE,CAAA;IAC9C,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAAA;IAC9D,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,CAAA;CAC5D;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAA;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAA;IAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,yBAAyB,CAAA;IACjD,QAAQ,CAAC,WAAW,CAAC,EAAE,wBAAwB,CAAA;IAC/C,QAAQ,CAAC,aAAa,CAAC,EAAE,0BAA0B,CAAA;IACnD,QAAQ,CAAC,cAAc,CAAC,EAAE,2BAA2B,CAAA;IACrD,QAAQ,CAAC,eAAe,CAAC,EAAE,4BAA4B,CAAA;CACxD;AAED,MAAM,MAAM,4BAA4B,GAAG,CAAC,MAAM,EAAE,qBAAqB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;AAElG,MAAM,MAAM,yBAAyB,GAAG,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;AAExF,MAAM,MAAM,wBAAwB,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,iBAAiB,KAAK,IAAI,CAAA;AAEvF,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,MAAM,0BAA0B,GAAG,CACvC,IAAI,EAAE,qBAAqB,EAC3B,OAAO,EAAE,4BAA4B,KAClC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;AAEzB,MAAM,MAAM,2BAA2B,GAAG,CACxC,IAAI,EAAE,qBAAqB,EAC3B,OAAO,EAAE,4BAA4B,KAClC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;AAEzB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,GAAG,EAAE,qBAAqB,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,qEAAqE;IACrE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,GAAG,EAAE,iBAAiB,CAAA;IAC/B,QAAQ,CAAC,KAAK,EAAE,SAAS,qBAAqB,EAAE,CAAA;IAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,cAAc,CAAA;CAClC"}

package/dist/worker.d.ts

@@ -0,0 +1,13 @@
+import type { QueueWorkerFailureDecision } from "@sixb/core/internal/workers";
+import { QueueWorker } from "@sixb/core/internal/workers";
+import type { ClaimedQueueJob, PipelineRunRequestedQueueJob } from "@sixb/core/queues";
+import type { PipelineWorkerSixb } from "./types";
+export declare class PipelineWorker extends QueueWorker<PipelineRunRequestedQueueJob> {
+    private readonly context;
+    private readonly sixb;
+    constructor(sixb: PipelineWorkerSixb);
+    protected execute(claimed: ClaimedQueueJob<PipelineRunRequestedQueueJob>, signal: AbortSignal): Promise<void>;
+    protected onExecutionError(claimed: ClaimedQueueJob<PipelineRunRequestedQueueJob>, _error: unknown): Promise<QueueWorkerFailureDecision>;
+    protected onAbortError(claimed: ClaimedQueueJob<PipelineRunRequestedQueueJob>, _error: unknown): Promise<QueueWorkerFailureDecision>;
+}
+//# sourceMappingURL=worker.d.ts.map

package/dist/worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAA;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAA;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAUtF,OAAO,KAAK,EAIV,kBAAkB,EACnB,MAAM,SAAS,CAAA;AAEhB,qBAAa,cAAe,SAAQ,WAAW,CAAC,4BAA4B,CAAC;IAC3E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAC/C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAoB;gBAE7B,IAAI,EAAE,kBAAkB;cAoBpB,OAAO,CACrB,OAAO,EAAE,eAAe,CAAC,4BAA4B,CAAC,EACtD,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,IAAI,CAAC;cA8CS,gBAAgB,CACvC,OAAO,EAAE,eAAe,CAAC,4BAA4B,CAAC,EACtD,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,0BAA0B,CAAC;cAab,YAAY,CACnC,OAAO,EAAE,eAAe,CAAC,4BAA4B,CAAC,EACtD,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,0BAA0B,CAAC;CAoBvC"}

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@sixb/pipeline-worker",
+  "version": "0.0.1",
+  "description": "Worker that runs Sixb pipelines",
+  "keywords": [
+    "sixb",
+    "bun",
+    "worker"
+  ],
+  "homepage": "https://sixb.ai",
+  "bugs": {
+    "url": "https://github.com/sixb-ai/sixb/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sixb-ai/sixb.git",
+    "directory": "packages/pipeline-worker"
+  },
+  "author": "Sixb contributors",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "bun": "./src/index.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "bun ../../scripts/build-package.ts",
+    "typecheck": "tsc --noEmit",
+    "test": "bun test",
+    "prepack": "bun run build"
+  },
+  "dependencies": {
+    "@sixb/core": "0.0.1"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "typescript": "^5.7.0"
+  },
+  "license": "MIT",
+  "engines": {
+    "bun": ">=1.3.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "dist",
+    "src",
+    "README.md",
+    "LICENSE"
+  ]
+}

package/src/errors.ts

@@ -0,0 +1,110 @@
+import type { DatasetDefinition, PipelineDefinition } from "@sixb/core"
+import type { DatasetVersion } from "@sixb/core/lake-storage"
+import type { PipelineRunFailure, PipelineRunStatus } from "@sixb/core/storage"
+import type { PipelineJob } from "./types"
+
+export class PipelineWorkerError extends Error {
+  readonly name = "PipelineWorkerError"
+}
+
+export function toPipelineRunFailure(error: unknown): PipelineRunFailure {
+  if (error instanceof Error) {
+    return {
+      name: error.name,
+      message: error.message,
+    }
+  }
+
+  return {
+    message: String(error),
+  }
+}
+
+export function throwIfAborted(signal: AbortSignal): void {
+  if (signal.aborted) {
+    throw createAbortError()
+  }
+}
+
+export function statusForFailure(
+  signal: AbortSignal,
+  error: unknown
+): Extract<PipelineRunStatus, "failed" | "cancelled"> {
+  return signal.aborted || isAbortError(error) ? "cancelled" : "failed"
+}
+
+export function requireFinishedAt(runId: string, finishedAt: Date | undefined): Date {
+  if (finishedAt) {
+    return finishedAt
+  }
+
+  throw new PipelineWorkerError(
+    `[SixbPipelineWorker] Pipeline run '${runId}' finished without a finishedAt timestamp.`
+  )
+}
+
+export function createStepBookkeepingError(options: {
+  readonly pipelineId: string
+  readonly stepId: string
+  readonly runId: string
+  readonly version: DatasetVersion
+  readonly cause: unknown
+}): Error {
+  // The dataset version is already durable; retrying the whole pipeline could duplicate appends.
+  return new PipelineWorkerError(
+    `[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' committed dataset version '${options.version.versionId}', but failed to finalize step run '${options.runId}'. The dataset commit may already have succeeded and the step run record may need repair.`,
+    { cause: options.cause }
+  )
+}
+
+export function createPipelineBookkeepingError(options: {
+  readonly pipelineId: string
+  readonly runId: string
+  readonly version: DatasetVersion
+  readonly cause: unknown
+}): Error {
+  return new PipelineWorkerError(
+    `[SixbPipelineWorker] Pipeline '${options.pipelineId}' committed final dataset version '${options.version.versionId}', but failed to finalize pipeline run '${options.runId}'. The dataset commit may already have succeeded and the pipeline run record may need repair.`,
+    { cause: options.cause }
+  )
+}
+
+export function requirePipeline(
+  pipeline: PipelineDefinition | null,
+  job: PipelineJob
+): PipelineDefinition {
+  if (!pipeline) {
+    throw new PipelineWorkerError(`[SixbPipelineWorker] Unknown pipeline '${job.pipelineId}'.`)
+  }
+
+  return pipeline
+}
+
+export function requireRegisteredDataset(options: {
+  readonly dataset: DatasetDefinition | null
+  readonly pipelineId: string
+  readonly stepId: string
+  readonly role: "input" | "output"
+  readonly name?: string
+  readonly datasetId: string
+}): DatasetDefinition {
+  if (options.dataset) {
+    return options.dataset
+  }
+
+  const namedRole = options.role === "input" ? `input '${options.name ?? "unknown"}'` : "output"
+
+  throw new PipelineWorkerError(
+    `[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' ${namedRole} references unknown dataset '${options.datasetId}'.`
+  )
+}
+
+function createAbortError(): Error {
+  const error = new Error("Worker runtime aborted.")
+  error.name = "AbortError"
+  return error
+}
+
+function isAbortError(error: unknown): boolean {
+  return error instanceof Error && error.name === "AbortError"
+}

package/src/events.ts

@@ -0,0 +1,189 @@
+import type { DomainEventLog } from "@sixb/core"
+import type {
+  PipelineRunRecord,
+  PipelineRunStatus,
+  PipelineStepRunRecord,
+} from "@sixb/core/storage"
+import type { PipelineJob, PipelineStepLifecycleContext, PipelineStepRunResult } from "./types"
+
+const SOURCE = "SixbPipelineWorker"
+
+export async function emitPipelineRunStarted(
+  events: DomainEventLog | undefined,
+  run: Pick<PipelineRunRecord, "id" | "pipelineId" | "startedAt">
+): Promise<void> {
+  await events?.emit(
+    {
+      events: [
+        {
+          type: "pipeline.run.started",
+          payload: {
+            pipelineId: run.pipelineId,
+            runId: run.id,
+            startedAt: run.startedAt.toISOString(),
+          },
+        },
+      ],
+    },
+    { source: SOURCE }
+  )
+}
+
+export async function emitPipelineRunStepStarted(
+  events: DomainEventLog | undefined,
+  step: Pick<
+    PipelineStepRunRecord,
+    "id" | "pipelineRunId" | "pipelineId" | "stepId" | "datasetId" | "startedAt"
+  >,
+  context: PipelineStepLifecycleContext
+): Promise<void> {
+  await events?.emit(
+    {
+      events: [
+        {
+          type: "pipeline.run.step.started",
+          payload: {
+            pipelineId: step.pipelineId,
+            runId: step.pipelineRunId,
+            stepRunId: step.id,
+            stepId: step.stepId,
+            stepIndex: context.stepIndex,
+            totalSteps: context.totalSteps,
+            datasetId: step.datasetId,
+            startedAt: step.startedAt.toISOString(),
+          },
+        },
+      ],
+    },
+    { source: SOURCE }
+  )
+}
+
+export async function emitPipelineRunStepFinished(
+  events: DomainEventLog | undefined,
+  step: Pick<
+    PipelineStepRunRecord,
+    | "id"
+    | "pipelineRunId"
+    | "pipelineId"
+    | "stepId"
+    | "datasetId"
+    | "status"
+    | "finishedAt"
+    | "output"
+    | "rowsWritten"
+    | "error"
+  >,
+  context: PipelineStepLifecycleContext
+): Promise<void> {
+  await events?.emit(
+    {
+      events: [
+        {
+          type: "pipeline.run.step.finished",
+          payload: {
+            pipelineId: step.pipelineId,
+            runId: step.pipelineRunId,
+            stepRunId: step.id,
+            stepId: step.stepId,
+            stepIndex: context.stepIndex,
+            totalSteps: context.totalSteps,
+            datasetId: step.datasetId,
+            status: requireTerminalStatus(step.status, `Pipeline step run '${step.id}'`),
+            finishedAt: requireFinishedAt(step.id, step.finishedAt).toISOString(),
+            ...(step.output ? { versionId: step.output.versionId } : {}),
+            ...(step.rowsWritten !== undefined ? { rowsWritten: step.rowsWritten } : {}),
+            ...(step.error ? { error: step.error.message } : {}),
+          },
+        },
+      ],
+    },
+    { source: SOURCE }
+  )
+}
+
+export async function emitDatasetVersionCommitted(
+  events: DomainEventLog | undefined,
+  job: PipelineJob,
+  step: PipelineStepRunResult
+): Promise<void> {
+  if (!step.versionCreated) return
+
+  // Emit step outputs immediately so dataset event schedules can react before later steps finish.
+  await events?.emit(
+    {
+      events: [
+        {
+          type: "dataset.version.committed" as const,
+          payload: {
+            datasetId: step.version.datasetId,
+            versionId: step.version.versionId,
+            createdAt: step.version.createdAt.toISOString(),
+            producer: {
+              kind: "pipeline" as const,
+              id: job.pipelineId,
+              runId: job.id,
+              stepId: step.run.stepId,
+            },
+          },
+        },
+      ],
+    },
+    { source: SOURCE }
+  )
+}
+
+export async function emitPipelineRunFinished(
+  events: DomainEventLog | undefined,
+  job: {
+    readonly id: string
+    readonly pipelineId: string
+    readonly status: "succeeded" | "failed" | "cancelled"
+    readonly datasetId?: string
+    readonly versionId?: string
+  }
+): Promise<void> {
+  await events?.emit(
+    { events: [{ type: "pipeline.run.finished", payload: buildPipelineRunFinishedPayload(job) }] },
+    { source: SOURCE }
+  )
+}
+
+function buildPipelineRunFinishedPayload(job: {
+  readonly id: string
+  readonly pipelineId: string
+  readonly status: "succeeded" | "failed" | "cancelled"
+  readonly datasetId?: string
+  readonly versionId?: string
+}) {
+  return {
+    pipelineId: job.pipelineId,
+    runId: job.id,
+    status: job.status,
+    ...(job.datasetId && job.versionId
+      ? {
+          datasetId: job.datasetId,
+          versionId: job.versionId,
+        }
+      : {}),
+  }
+}
+
+function requireFinishedAt(stepRunId: string, finishedAt: Date | undefined): Date {
+  if (finishedAt) {
+    return finishedAt
+  }
+
+  throw new Error(`[SixbPipelineWorker] Pipeline step run '${stepRunId}' has no finishedAt.`)
+}
+
+function requireTerminalStatus(
+  status: PipelineRunStatus,
+  context: string
+): Exclude<PipelineRunStatus, "running"> {
+  if (status === "running") {
+    throw new Error(`[SixbPipelineWorker] ${context} is still running.`)
+  }
+
+  return status
+}