@sixb/pipeline-worker 0.0.1

package/dist/index.js

@@ -0,0 +1,654 @@
+// @bun
+// src/worker.ts
+import { reportRunFailure } from "@sixb/core/internal/error-reporting";
+import { QueueWorker } from "@sixb/core/internal/workers";
+
+// src/events.ts
+var SOURCE = "SixbPipelineWorker";
+async function emitPipelineRunStarted(events, run) {
+  await events?.emit({
+    events: [
+      {
+        type: "pipeline.run.started",
+        payload: {
+          pipelineId: run.pipelineId,
+          runId: run.id,
+          startedAt: run.startedAt.toISOString()
+        }
+      }
+    ]
+  }, { source: SOURCE });
+}
+async function emitPipelineRunStepStarted(events, step, context) {
+  await events?.emit({
+    events: [
+      {
+        type: "pipeline.run.step.started",
+        payload: {
+          pipelineId: step.pipelineId,
+          runId: step.pipelineRunId,
+          stepRunId: step.id,
+          stepId: step.stepId,
+          stepIndex: context.stepIndex,
+          totalSteps: context.totalSteps,
+          datasetId: step.datasetId,
+          startedAt: step.startedAt.toISOString()
+        }
+      }
+    ]
+  }, { source: SOURCE });
+}
+async function emitPipelineRunStepFinished(events, step, context) {
+  await events?.emit({
+    events: [
+      {
+        type: "pipeline.run.step.finished",
+        payload: {
+          pipelineId: step.pipelineId,
+          runId: step.pipelineRunId,
+          stepRunId: step.id,
+          stepId: step.stepId,
+          stepIndex: context.stepIndex,
+          totalSteps: context.totalSteps,
+          datasetId: step.datasetId,
+          status: requireTerminalStatus(step.status, `Pipeline step run '${step.id}'`),
+          finishedAt: requireFinishedAt(step.id, step.finishedAt).toISOString(),
+          ...step.output ? { versionId: step.output.versionId } : {},
+          ...step.rowsWritten !== undefined ? { rowsWritten: step.rowsWritten } : {},
+          ...step.error ? { error: step.error.message } : {}
+        }
+      }
+    ]
+  }, { source: SOURCE });
+}
+async function emitDatasetVersionCommitted(events, job, step) {
+  if (!step.versionCreated)
+    return;
+  await events?.emit({
+    events: [
+      {
+        type: "dataset.version.committed",
+        payload: {
+          datasetId: step.version.datasetId,
+          versionId: step.version.versionId,
+          createdAt: step.version.createdAt.toISOString(),
+          producer: {
+            kind: "pipeline",
+            id: job.pipelineId,
+            runId: job.id,
+            stepId: step.run.stepId
+          }
+        }
+      }
+    ]
+  }, { source: SOURCE });
+}
+async function emitPipelineRunFinished(events, job) {
+  await events?.emit({ events: [{ type: "pipeline.run.finished", payload: buildPipelineRunFinishedPayload(job) }] }, { source: SOURCE });
+}
+function buildPipelineRunFinishedPayload(job) {
+  return {
+    pipelineId: job.pipelineId,
+    runId: job.id,
+    status: job.status,
+    ...job.datasetId && job.versionId ? {
+      datasetId: job.datasetId,
+      versionId: job.versionId
+    } : {}
+  };
+}
+function requireFinishedAt(stepRunId, finishedAt) {
+  if (finishedAt) {
+    return finishedAt;
+  }
+  throw new Error(`[SixbPipelineWorker] Pipeline step run '${stepRunId}' has no finishedAt.`);
+}
+function requireTerminalStatus(status, context) {
+  if (status === "running") {
+    throw new Error(`[SixbPipelineWorker] ${context} is still running.`);
+  }
+  return status;
+}
+
+// src/run-pipeline-job.ts
+import { resolveLogsRuntime } from "@sixb/core/internal/logging";
+
+// src/errors.ts
+class PipelineWorkerError extends Error {
+  name = "PipelineWorkerError";
+}
+function toPipelineRunFailure(error) {
+  if (error instanceof Error) {
+    return {
+      name: error.name,
+      message: error.message
+    };
+  }
+  return {
+    message: String(error)
+  };
+}
+function throwIfAborted(signal) {
+  if (signal.aborted) {
+    throw createAbortError();
+  }
+}
+function statusForFailure(signal, error) {
+  return signal.aborted || isAbortError(error) ? "cancelled" : "failed";
+}
+function requireFinishedAt2(runId, finishedAt) {
+  if (finishedAt) {
+    return finishedAt;
+  }
+  throw new PipelineWorkerError(`[SixbPipelineWorker] Pipeline run '${runId}' finished without a finishedAt timestamp.`);
+}
+function createStepBookkeepingError(options) {
+  return new PipelineWorkerError(`[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' committed dataset version '${options.version.versionId}', but failed to finalize step run '${options.runId}'. The dataset commit may already have succeeded and the step run record may need repair.`, { cause: options.cause });
+}
+function createPipelineBookkeepingError(options) {
+  return new PipelineWorkerError(`[SixbPipelineWorker] Pipeline '${options.pipelineId}' committed final dataset version '${options.version.versionId}', but failed to finalize pipeline run '${options.runId}'. The dataset commit may already have succeeded and the pipeline run record may need repair.`, { cause: options.cause });
+}
+function requirePipeline(pipeline, job) {
+  if (!pipeline) {
+    throw new PipelineWorkerError(`[SixbPipelineWorker] Unknown pipeline '${job.pipelineId}'.`);
+  }
+  return pipeline;
+}
+function requireRegisteredDataset(options) {
+  if (options.dataset) {
+    return options.dataset;
+  }
+  const namedRole = options.role === "input" ? `input '${options.name ?? "unknown"}'` : "output";
+  throw new PipelineWorkerError(`[SixbPipelineWorker] Pipeline '${options.pipelineId}' step '${options.stepId}' ${namedRole} references unknown dataset '${options.datasetId}'.`);
+}
+function createAbortError() {
+  const error = new Error("Worker runtime aborted.");
+  error.name = "AbortError";
+  return error;
+}
+function isAbortError(error) {
+  return error instanceof Error && error.name === "AbortError";
+}
+
+// src/step-inputs.ts
+async function resolveStepInputs(input) {
+  const { runtime, pipeline, step } = input;
+  const resolved = [];
+  for (const [name, declaredDataset] of Object.entries(step.inputs)) {
+    const dataset = requireRegisteredDataset({
+      dataset: runtime.getDatasetById(declaredDataset.id),
+      pipelineId: pipeline.id,
+      stepId: step.id,
+      role: "input",
+      name,
+      datasetId: declaredDataset.id
+    });
+    const version = await runtime.lakeStorage.getLatestVersion(dataset.id);
+    if (!version) {
+      throw new PipelineWorkerError(`[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' input '${name}' dataset '${dataset.id}' has no committed version.`);
+    }
+    resolved.push({
+      name,
+      dataset,
+      version,
+      ref: {
+        datasetId: version.datasetId,
+        versionId: version.versionId
+      }
+    });
+  }
+  return resolved;
+}
+function createStepInputs(lakeStorage, resolved) {
+  const inputs = {};
+  for (const input of resolved) {
+    inputs[input.name] = {
+      dataset: input.dataset,
+      version: input.version,
+      readRows(readInput = {}) {
+        return lakeStorage.readRows({
+          ...readInput,
+          datasetId: input.dataset.id,
+          versionId: input.version.versionId
+        });
+      }
+    };
+  }
+  return inputs;
+}
+
+// src/execute-run-step.ts
+async function executeRunStep(input) {
+  const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input;
+  if (step.executor.kind !== "run") {
+    throw new PipelineWorkerError(`[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses SQL execution, which is not supported by the run step executor.`);
+  }
+  const logger = input.logSession.withContext({ stepId: step.id });
+  let rowsWritten = 0;
+  const write = await runtime.lakeStorage.beginWrite({
+    dataset: outputDataset,
+    mode: step.mode,
+    inputs: resolvedInputs.map((resolved) => resolved.ref),
+    producer: {
+      kind: "pipeline",
+      id: pipeline.id,
+      runId: job.id,
+      stepId: step.id
+    }
+  });
+  try {
+    const context = {
+      projectId: runtime.id,
+      pipelineId: pipeline.id,
+      stepId: step.id,
+      runId: job.id,
+      signal,
+      logger,
+      inputs: createStepInputs(runtime.lakeStorage, resolvedInputs),
+      output: {
+        async writeRows(rows) {
+          await write.writeRows(countRowsWritten(rows, signal, () => {
+            rowsWritten += 1;
+          }));
+        }
+      }
+    };
+    await step.executor.handler(context);
+    throwIfAborted(signal);
+    const commit = await write.commit({
+      commitMessage: `pipeline ${pipeline.id} step ${step.id} run ${job.id}`
+    });
+    const { outcome, ...version } = commit;
+    return {
+      version,
+      versionCreated: outcome === "created",
+      rowsWritten
+    };
+  } catch (error) {
+    await write.abort().catch(() => {});
+    throw error;
+  }
+}
+async function* countRowsWritten(rows, signal, onWritten) {
+  for await (const row of rows) {
+    throwIfAborted(signal);
+    yield row;
+    onWritten();
+  }
+}
+
+// src/execute-sql-step.ts
+function hasSqlExecutor(lakeStorage) {
+  const sql = "sql" in lakeStorage ? lakeStorage.sql : null;
+  return typeof sql === "object" && sql !== null && "execute" in sql && typeof sql.execute === "function";
+}
+function assertSqlModeSupported(input) {
+  const supported = input.mode === "append" ? input.capabilities.supportsAppend : input.capabilities.supportsSnapshot;
+  if (supported)
+    return;
+  throw new PipelineWorkerError(`[SixbPipelineWorker] Pipeline '${input.pipelineId}' step '${input.stepId}' writes in ` + `'${input.mode}' mode, which the ${input.dialect} SQL executor does not support.`);
+}
+async function executeSqlStep(input) {
+  const { runtime, pipeline, step, job, signal, outputDataset, resolvedInputs } = input;
+  if (step.executor.kind !== "sql") {
+    throw new PipelineWorkerError(`[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' uses run execution, which is not supported by the SQL step executor.`);
+  }
+  if (!hasSqlExecutor(runtime.lakeStorage)) {
+    throw new PipelineWorkerError(`[SixbPipelineWorker] Pipeline '${pipeline.id}' step '${step.id}' requires SQL transform support, but lake storage does not provide lakeStorage.sql.execute(...).`);
+  }
+  assertSqlModeSupported({
+    capabilities: runtime.lakeStorage.sql.capabilities,
+    dialect: runtime.lakeStorage.sql.dialect,
+    mode: step.mode,
+    pipelineId: pipeline.id,
+    stepId: step.id
+  });
+  throwIfAborted(signal);
+  const commit = await runtime.lakeStorage.sql.execute({
+    sources: Object.fromEntries(resolvedInputs.map((input2) => [
+      input2.name,
+      {
+        dataset: input2.dataset,
+        versionId: input2.version.versionId
+      }
+    ])),
+    sql: step.executor.sql,
+    target: outputDataset,
+    mode: step.mode,
+    producer: {
+      kind: "pipeline",
+      id: pipeline.id,
+      runId: job.id,
+      stepId: step.id
+    },
+    commitMessage: `pipeline ${pipeline.id} step ${step.id} run ${job.id}`
+  });
+  throwIfAborted(signal);
+  const { outcome, ...version } = commit;
+  return {
+    version,
+    versionCreated: outcome === "created",
+    rowsWritten: version.rowCount
+  };
+}
+
+// src/run-step.ts
+async function runStep(input) {
+  const { runtime, pipeline, step, stepIndex, job, signal } = input;
+  const lifecycleContext = {
+    stepIndex,
+    totalSteps: pipeline.graph.nodes.length
+  };
+  throwIfAborted(signal);
+  const resolvedInputs = await resolveStepInputs({ runtime, pipeline, step });
+  const inputRefs = resolvedInputs.map((resolved) => resolved.ref);
+  const outputDataset = requireRegisteredDataset({
+    dataset: runtime.getDatasetById(step.output.id),
+    pipelineId: pipeline.id,
+    stepId: step.id,
+    role: "output",
+    datasetId: step.output.id
+  });
+  await runtime.lakeStorage.createDataset(outputDataset);
+  const stepRunId = `${job.id}:step:${stepIndex + 1}:${step.id}`;
+  const startedStepRun = await runtime.pipelineRunsStorage.startStep({
+    projectId: runtime.id,
+    id: stepRunId,
+    pipelineRunId: job.id,
+    pipelineId: pipeline.id,
+    stepId: step.id,
+    datasetId: outputDataset.id,
+    mode: step.mode,
+    inputs: inputRefs
+  });
+  await input.onStepStarted?.(startedStepRun, lifecycleContext);
+  let committedVersion;
+  let rowsWritten;
+  try {
+    const execution = await executeStep({
+      runtime,
+      pipeline,
+      step,
+      job,
+      signal,
+      logSession: input.logSession,
+      outputDataset,
+      resolvedInputs
+    });
+    committedVersion = execution.version;
+    rowsWritten = execution.rowsWritten;
+    let stepRun;
+    try {
+      stepRun = await runtime.pipelineRunsStorage.finishStep({
+        projectId: runtime.id,
+        id: stepRunId,
+        status: "succeeded",
+        output: {
+          datasetId: committedVersion.datasetId,
+          versionId: committedVersion.versionId
+        },
+        rowsWritten
+      });
+    } catch (error) {
+      throw createStepBookkeepingError({
+        pipelineId: pipeline.id,
+        stepId: step.id,
+        runId: stepRunId,
+        version: committedVersion,
+        cause: error
+      });
+    }
+    await input.onStepFinished?.(stepRun, lifecycleContext);
+    return {
+      run: stepRun,
+      version: committedVersion,
+      versionCreated: execution.versionCreated
+    };
+  } catch (error) {
+    if (!committedVersion) {
+      const failedStepRun = await runtime.pipelineRunsStorage.finishStep({
+        projectId: runtime.id,
+        id: stepRunId,
+        status: statusForFailure(signal, error),
+        rowsWritten,
+        error: toPipelineRunFailure(error)
+      }).catch(() => null);
+      if (failedStepRun) {
+        await input.onStepFinished?.(failedStepRun, lifecycleContext);
+      }
+    }
+    throw error;
+  }
+}
+async function executeStep(input) {
+  switch (input.step.executor.kind) {
+    case "run":
+      return executeRunStep(input);
+    case "sql":
+      return executeSqlStep(input);
+  }
+}
+
+// src/run-pipeline-job.ts
+class PipelineRunAlreadyStartedError extends Error {
+  run;
+  name = "PipelineRunAlreadyStartedError";
+  constructor(run) {
+    super(`[SixbPipelineWorker] Pipeline run '${run.id}' has already started.`);
+    this.run = run;
+  }
+}
+async function runPipelineJob(input) {
+  const { runtime, job } = input;
+  const signal = input.signal ?? new AbortController().signal;
+  const pipeline = requirePipeline(runtime.getPipelineById(job.pipelineId), job);
+  const logSession = resolveLogsRuntime(runtime.id, runtime.logs).startExecution({
+    kind: "pipeline",
+    id: job.id
+  });
+  const steps = [];
+  let startedRun;
+  let finalVersion;
+  let finished = false;
+  throwIfAborted(signal);
+  try {
+    try {
+      startedRun = await runtime.pipelineRunsStorage.start({
+        projectId: runtime.id,
+        id: job.id,
+        pipelineId: pipeline.id
+      });
+    } catch (error) {
+      const existing = await runtime.pipelineRunsStorage.getById({
+        projectId: runtime.id,
+        id: job.id
+      });
+      if (existing?.pipelineId === pipeline.id) {
+        throw new PipelineRunAlreadyStartedError(existing);
+      }
+      throw error;
+    }
+    await input.onRunStarted?.(startedRun);
+    for (const [stepIndex, node] of pipeline.graph.nodes.entries()) {
+      const stepResult = await runStep({
+        runtime,
+        pipeline,
+        step: node.step,
+        stepIndex,
+        job,
+        signal,
+        logSession,
+        onStepStarted: input.onStepStarted,
+        onStepFinished: input.onStepFinished
+      });
+      steps.push(stepResult);
+      finalVersion = stepResult.version;
+      await input.onStepCommitted?.(stepResult);
+    }
+    let run;
+    try {
+      run = await runtime.pipelineRunsStorage.finish({
+        projectId: runtime.id,
+        id: job.id,
+        status: "succeeded",
+        output: finalVersion ? {
+          datasetId: finalVersion.datasetId,
+          versionId: finalVersion.versionId
+        } : undefined
+      });
+    } catch (error) {
+      if (finalVersion) {
+        throw createPipelineBookkeepingError({
+          pipelineId: pipeline.id,
+          runId: job.id,
+          version: finalVersion,
+          cause: error
+        });
+      }
+      throw error;
+    }
+    finished = true;
+    return {
+      run: {
+        ...run,
+        finishedAt: requireFinishedAt2(job.id, run.finishedAt)
+      },
+      steps,
+      version: finalVersion
+    };
+  } catch (error) {
+    if (startedRun && !finished) {
+      const status = statusForFailure(signal, error);
+      try {
+        const run = await runtime.pipelineRunsStorage.finish({
+          projectId: runtime.id,
+          id: job.id,
+          status,
+          error: toPipelineRunFailure(error)
+        });
+        if (status === "failed" && run.status === "failed")
+          input.onRunFailed?.(error, run);
+      } catch {}
+    }
+    throw error;
+  } finally {
+    await logSession.flush();
+  }
+}
+
+// src/worker.ts
+class PipelineWorker extends QueueWorker {
+  context;
+  sixb;
+  constructor(sixb) {
+    if (sixb.listPipelines().length === 0) {
+      throw new Error("[SixbPipelineWorker] No pipeline definitions are registered.");
+    }
+    const pipelineRunsStorage = sixb.storage.pipelineRuns;
+    if (!pipelineRunsStorage) {
+      throw new Error("[SixbPipelineWorker] Pipeline workers require storage.pipelineRuns support.");
+    }
+    super({
+      projectId: sixb.id,
+      queue: sixb.queues.pipelines,
+      workerId: `pipeline-worker-${sixb.id}`
+    });
+    this.context = buildPipelineContext(sixb, pipelineRunsStorage);
+    this.sixb = sixb;
+  }
+  async execute(claimed, signal) {
+    const { job } = claimed;
+    const pipelineJob = {
+      id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,
+      pipelineId: job.payload.pipelineId
+    };
+    let result;
+    try {
+      result = await runPipelineJob({
+        runtime: this.context,
+        job: pipelineJob,
+        signal,
+        onRunStarted: (run) => emitPipelineRunStarted(this.sixb.events, run),
+        onRunFailed: (error, run) => {
+          reportRunFailure(this.sixb, error, {
+            projectId: this.sixb.id,
+            occurredAt: run.finishedAt,
+            attempt: job.attempt,
+            run: {
+              kind: "pipeline",
+              runId: run.id,
+              pipelineId: run.pipelineId
+            }
+          });
+        },
+        onStepStarted: (step, context) => emitPipelineRunStepStarted(this.sixb.events, step, context),
+        onStepFinished: (step, context) => emitPipelineRunStepFinished(this.sixb.events, step, context),
+        onStepCommitted: (step) => emitDatasetVersionCommitted(this.sixb.events, pipelineJob, step)
+      });
+    } catch (error) {
+      if (error instanceof PipelineRunAlreadyStartedError)
+        return;
+      throw error;
+    }
+    await emitPipelineRunFinished(this.sixb.events, {
+      id: pipelineJob.id,
+      pipelineId: pipelineJob.pipelineId,
+      status: "succeeded",
+      datasetId: result.version?.datasetId,
+      versionId: result.version?.versionId
+    });
+  }
+  async onExecutionError(claimed, _error) {
+    const { job } = claimed;
+    await emitPipelineRunFinished(this.sixb.events, {
+      id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,
+      pipelineId: job.payload.pipelineId,
+      status: "failed"
+    });
+    return { kind: "fail" };
+  }
+  async onAbortError(claimed, _error) {
+    const { job } = claimed;
+    const pipelineJob = {
+      id: job.payload.runId ?? `${job.id}:attempt:${job.attempt}`,
+      pipelineId: job.payload.pipelineId
+    };
+    if (!await hasCommittedStep(this.context, pipelineJob.id)) {
+      return { kind: "retry" };
+    }
+    await emitPipelineRunFinished(this.sixb.events, {
+      id: pipelineJob.id,
+      pipelineId: pipelineJob.pipelineId,
+      status: "cancelled"
+    });
+    return { kind: "fail" };
+  }
+}
+function buildPipelineContext(sixb, pipelineRunsStorage) {
+  return {
+    id: sixb.id,
+    pipelineRunsStorage,
+    lakeStorage: sixb.lakeStorage,
+    logs: sixb.logs,
+    getPipelineById(pipelineId) {
+      return sixb.getPipelineById(pipelineId);
+    },
+    getDatasetById(datasetId) {
+      return sixb.getDatasetById(datasetId);
+    }
+  };
+}
+async function hasCommittedStep(context, pipelineRunId) {
+  const result = await context.pipelineRunsStorage.listSteps({
+    projectId: context.id,
+    pipelineRunId,
+    statuses: ["succeeded"],
+    limit: 1
+  });
+  return result.steps.some((step) => Boolean(step.output));
+}
+export {
+  PipelineWorker
+};
+
+//# debugId=068611D3BE49C18D64756E2164756E21