@sixb/pipeline-worker 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sixb contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,34 @@
+# @sixb/pipeline-worker
+
+Worker that runs Sixb pipelines.
+
+Dequeues pipeline runs, executes each step in order, and records the run in `storage.pipelineRuns`.
+Pipelines are how dataset rows become objects, links, and telemetry, so without this worker running a
+pipeline request simply waits in the queue.
+
+## Install
+
+```bash
+bun add @sixb/pipeline-worker
+```
+
+You normally do not import it. `sixb worker pipeline` and `sixb worker-group` construct it for you,
+and that is the supported way to run it.
+
+## Running it yourself
+
+Construct it directly only when you host workers inside your own process:
+
+```ts
+import { PipelineWorker } from "@sixb/pipeline-worker"
+import { sixb } from "./sixb.config"
+
+const worker = new PipelineWorker(sixb)
+await worker.start()
+// on shutdown
+await worker.stop()
+```
+
+Requires a lake storage provider — pipeline steps read dataset versions from it — and a storage
+provider with `pipelineRuns` support. Both are checked at construction, so a missing provider fails at
+boot rather than on the first job.

package/dist/errors.d.ts

@@ -0,0 +1,34 @@
+import type { DatasetDefinition, PipelineDefinition } from "@sixb/core";
+import type { DatasetVersion } from "@sixb/core/lake-storage";
+import type { PipelineRunFailure, PipelineRunStatus } from "@sixb/core/storage";
+import type { PipelineJob } from "./types";
+export declare class PipelineWorkerError extends Error {
+    readonly name = "PipelineWorkerError";
+}
+export declare function toPipelineRunFailure(error: unknown): PipelineRunFailure;
+export declare function throwIfAborted(signal: AbortSignal): void;
+export declare function statusForFailure(signal: AbortSignal, error: unknown): Extract<PipelineRunStatus, "failed" | "cancelled">;
+export declare function requireFinishedAt(runId: string, finishedAt: Date | undefined): Date;
+export declare function createStepBookkeepingError(options: {
+    readonly pipelineId: string;
+    readonly stepId: string;
+    readonly runId: string;
+    readonly version: DatasetVersion;
+    readonly cause: unknown;
+}): Error;
+export declare function createPipelineBookkeepingError(options: {
+    readonly pipelineId: string;
+    readonly runId: string;
+    readonly version: DatasetVersion;
+    readonly cause: unknown;
+}): Error;
+export declare function requirePipeline(pipeline: PipelineDefinition | null, job: PipelineJob): PipelineDefinition;
+export declare function requireRegisteredDataset(options: {
+    readonly dataset: DatasetDefinition | null;
+    readonly pipelineId: string;
+    readonly stepId: string;
+    readonly role: "input" | "output";
+    readonly name?: string;
+    readonly datasetId: string;
+}): DatasetDefinition;
+//# sourceMappingURL=errors.d.ts.map

package/dist/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACvE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAC7D,OAAO,KAAK,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AAC/E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAE1C,qBAAa,mBAAoB,SAAQ,KAAK;IAC5C,QAAQ,CAAC,IAAI,yBAAwB;CACtC;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,kBAAkB,CAWvE;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAIxD;AAED,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,WAAW,EACnB,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,iBAAiB,EAAE,QAAQ,GAAG,WAAW,CAAC,CAEpD;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,GAAG,SAAS,GAAG,IAAI,CAQnF;AAED,wBAAgB,0BAA0B,CAAC,OAAO,EAAE;IAClD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CACxB,GAAG,KAAK,CAMR;AAED,wBAAgB,8BAA8B,CAAC,OAAO,EAAE;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CACxB,GAAG,KAAK,CAKR;AAED,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,kBAAkB,GAAG,IAAI,EACnC,GAAG,EAAE,WAAW,GACf,kBAAkB,CAMpB;AAED,wBAAgB,wBAAwB,CAAC,OAAO,EAAE;IAChD,QAAQ,CAAC,OAAO,EAAE,iBAAiB,GAAG,IAAI,CAAA;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,QAAQ,CAAA;IACjC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;CAC3B,GAAG,iBAAiB,CAUpB"}

package/dist/events.d.ts

@@ -0,0 +1,15 @@
+import type { DomainEventLog } from "@sixb/core";
+import type { PipelineRunRecord, PipelineStepRunRecord } from "@sixb/core/storage";
+import type { PipelineJob, PipelineStepLifecycleContext, PipelineStepRunResult } from "./types";
+export declare function emitPipelineRunStarted(events: DomainEventLog | undefined, run: Pick<PipelineRunRecord, "id" | "pipelineId" | "startedAt">): Promise<void>;
+export declare function emitPipelineRunStepStarted(events: DomainEventLog | undefined, step: Pick<PipelineStepRunRecord, "id" | "pipelineRunId" | "pipelineId" | "stepId" | "datasetId" | "startedAt">, context: PipelineStepLifecycleContext): Promise<void>;
+export declare function emitPipelineRunStepFinished(events: DomainEventLog | undefined, step: Pick<PipelineStepRunRecord, "id" | "pipelineRunId" | "pipelineId" | "stepId" | "datasetId" | "status" | "finishedAt" | "output" | "rowsWritten" | "error">, context: PipelineStepLifecycleContext): Promise<void>;
+export declare function emitDatasetVersionCommitted(events: DomainEventLog | undefined, job: PipelineJob, step: PipelineStepRunResult): Promise<void>;
+export declare function emitPipelineRunFinished(events: DomainEventLog | undefined, job: {
+    readonly id: string;
+    readonly pipelineId: string;
+    readonly status: "succeeded" | "failed" | "cancelled";
+    readonly datasetId?: string;
+    readonly versionId?: string;
+}): Promise<void>;
+//# sourceMappingURL=events.d.ts.map

package/dist/events.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../src/events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAChD,OAAO,KAAK,EACV,iBAAiB,EAEjB,qBAAqB,EACtB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,KAAK,EAAE,WAAW,EAAE,4BAA4B,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAI/F,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,cAAc,GAAG,SAAS,EAClC,GAAG,EAAE,IAAI,CAAC,iBAAiB,EAAE,IAAI,GAAG,YAAY,GAAG,WAAW,CAAC,GAC9D,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED,wBAAsB,0BAA0B,CAC9C,MAAM,EAAE,cAAc,GAAG,SAAS,EAClC,IAAI,EAAE,IAAI,CACR,qBAAqB,EACrB,IAAI,GAAG,eAAe,GAAG,YAAY,GAAG,QAAQ,GAAG,WAAW,GAAG,WAAW,CAC7E,EACD,OAAO,EAAE,4BAA4B,GACpC,OAAO,CAAC,IAAI,CAAC,CAqBf;AAED,wBAAsB,2BAA2B,CAC/C,MAAM,EAAE,cAAc,GAAG,SAAS,EAClC,IAAI,EAAE,IAAI,CACR,qBAAqB,EACnB,IAAI,GACJ,eAAe,GACf,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,QAAQ,GACR,aAAa,GACb,OAAO,CACV,EACD,OAAO,EAAE,4BAA4B,GACpC,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,2BAA2B,CAC/C,MAAM,EAAE,cAAc,GAAG,SAAS,EAClC,GAAG,EAAE,WAAW,EAChB,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,cAAc,GAAG,SAAS,EAClC,GAAG,EAAE;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAA;IACrD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAC5B,GACA,OAAO,CAAC,IAAI,CAAC,CAKf"}

package/dist/execute-run-step.d.ts

@@ -0,0 +1,19 @@
+import type { DatasetDefinition, PipelineDefinition, PipelineStepDefinition } from "@sixb/core";
+import type { DatasetVersion } from "@sixb/core/lake-storage";
+import { type ResolvedStepInput } from "./step-inputs";
+import type { PipelineJob, PipelineLogSession, PipelineWorkerContext } from "./types";
+export declare function executeRunStep(input: {
+    readonly runtime: PipelineWorkerContext;
+    readonly pipeline: PipelineDefinition;
+    readonly step: PipelineStepDefinition;
+    readonly job: PipelineJob;
+    readonly signal: AbortSignal;
+    readonly logSession: PipelineLogSession;
+    readonly outputDataset: DatasetDefinition;
+    readonly resolvedInputs: readonly ResolvedStepInput[];
+}): Promise<{
+    readonly version: DatasetVersion;
+    readonly versionCreated: boolean;
+    readonly rowsWritten: number;
+}>;
+//# sourceMappingURL=execute-run-step.d.ts.map

package/dist/execute-run-step.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execute-run-step.d.ts","sourceRoot":"","sources":["../src/execute-run-step.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EAEjB,kBAAkB,EAClB,sBAAsB,EAEvB,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAE7D,OAAO,EAAoB,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACxE,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AAErF,wBAAsB,cAAc,CAAC,KAAK,EAAE;IAC1C,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAA;IACvC,QAAQ,CAAC,aAAa,EAAE,iBAAiB,CAAA;IACzC,QAAQ,CAAC,cAAc,EAAE,SAAS,iBAAiB,EAAE,CAAA;CACtD,GAAG,OAAO,CAAC;IACV,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B,CAAC,CA4DD"}

package/dist/execute-sql-step.d.ts

@@ -0,0 +1,18 @@
+import type { DatasetDefinition, PipelineDefinition, PipelineStepDefinition } from "@sixb/core";
+import type { DatasetVersion } from "@sixb/core/lake-storage";
+import type { ResolvedStepInput } from "./step-inputs";
+import type { PipelineJob, PipelineWorkerContext } from "./types";
+export declare function executeSqlStep(input: {
+    readonly runtime: PipelineWorkerContext;
+    readonly pipeline: PipelineDefinition;
+    readonly step: PipelineStepDefinition;
+    readonly job: PipelineJob;
+    readonly signal: AbortSignal;
+    readonly outputDataset: DatasetDefinition;
+    readonly resolvedInputs: readonly ResolvedStepInput[];
+}): Promise<{
+    readonly version: DatasetVersion;
+    readonly versionCreated: boolean;
+    readonly rowsWritten?: number;
+}>;
+//# sourceMappingURL=execute-sql-step.d.ts.map

package/dist/execute-sql-step.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execute-sql-step.d.ts","sourceRoot":"","sources":["../src/execute-sql-step.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EAGjB,kBAAkB,EAClB,sBAAsB,EACvB,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EACV,cAAc,EAGf,MAAM,yBAAyB,CAAA;AAEhC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACtD,OAAO,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAA;AA6BjE,wBAAsB,cAAc,CAAC,KAAK,EAAE;IAC1C,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,QAAQ,CAAC,aAAa,EAAE,iBAAiB,CAAA;IACzC,QAAQ,CAAC,cAAc,EAAE,SAAS,iBAAiB,EAAE,CAAA;CACtD,GAAG,OAAO,CAAC;IACV,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAA;IAChC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAC9B,CAAC,CA2DD"}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export type { PipelineWorkerSixb } from "./types";
+export { PipelineWorker } from "./worker";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAA;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA"}