@lde/pipeline 0.6.24 → 0.6.25

package/dist/index.d.ts

@@ -1,12 +1,10 @@
 export * from './asyncQueue.js';
 export * from './batch.js';
 export * from './pipeline.js';
+export * from './progressReporter.js';
 export * from './selector.js';
 export * from './stage.js';
-export * from './step.js';
-export * from './step/sparqlQuery.js';
-export * from './builder.js';
-export * from './config.js';
+export * from './stageOutputResolver.js';
 export * from './sparql/index.js';
 export * from './distribution/index.js';
 export * from './writer/index.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC"}

package/dist/index.js

@@ -1,12 +1,10 @@
 export * from './asyncQueue.js';
 export * from './batch.js';
 export * from './pipeline.js';
+export * from './progressReporter.js';
 export * from './selector.js';
 export * from './stage.js';
-export * from './step.js';
-export * from './step/sparqlQuery.js';
-export * from './builder.js';
-export * from './config.js';
+export * from './stageOutputResolver.js';
 export * from './sparql/index.js';
 export * from './distribution/index.js';
 export * from './writer/index.js';

package/dist/pipeline.d.ts

@@ -1,11 +1,28 @@
-import { Selector } from './selector.js';
-import { Step } from './step.js';
+import type { Selector } from './selector.js';
+import { Stage } from './stage.js';
+import type { Writer } from './writer/writer.js';
+import { type DistributionResolver } from './distribution/resolver.js';
+import type { StageOutputResolver } from './stageOutputResolver.js';
+import type { ProgressReporter } from './progressReporter.js';
+export interface PipelineOptions {
+    name: string;
+    selector: Selector;
+    stages: Stage[];
+    writer: Writer;
+    distributionResolver: DistributionResolver;
+    stageOutputResolver?: StageOutputResolver;
+    outputDir?: string;
+    outputFormat?: 'turtle' | 'n-triples' | 'n-quads';
+    reporter?: ProgressReporter;
+}
 export declare class Pipeline {
-    private readonly config;
-    constructor(config: {
-        selector: Selector;
-        steps: Step[];
-    });
+    private readonly options;
+    constructor(options: PipelineOptions);
     run(): Promise<void>;
+    private processDataset;
+    private runStage;
+    private runChain;
+    private runChainedStage;
+    private readFiles;
 }
 //# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAc,IAAI,EAAE,MAAM,WAAW,CAAC;AAI7C,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE;QAAE,QAAQ,EAAE,QAAQ,CAAC;QAAC,KAAK,EAAE,IAAI,EAAE,CAAA;KAAE;IAE7D,GAAG;CAiCjB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,oBAAoB,CAAC;IAC3C,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAClD,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,eAAe;IAe9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAcZ,cAAc;YA2Bd,QAAQ;YAgCR,QAAQ;YAoDR,eAAe;YAoCd,SAAS;CAUzB"}

package/dist/pipeline.js

@@ -1,42 +1,141 @@
-import { NotSupported } from './step.js';
-import { Readable } from 'node:stream';
+import { createReadStream } from 'node:fs';
+import { StreamParser } from 'n3';
+import { FileWriter } from './writer/fileWriter.js';
+import { NoDistributionAvailable, } from './distribution/resolver.js';
+import { NotSupported } from './sparql/executor.js';
 export class Pipeline {
-    config;
-    constructor(config) {
-        this.config = config;
+    options;
+    constructor(options) {
+        const hasSubStages = options.stages.some((stage) => stage.stages.length > 0);
+        if (hasSubStages && !options.stageOutputResolver) {
+            throw new Error('stageOutputResolver is required when any stage has sub-stages');
+        }
+        if (hasSubStages && !options.outputDir) {
+            throw new Error('outputDir is required when any stage has sub-stages');
+        }
+        this.options = options;
     }
     async run() {
-        const datasets = await this.config.selector.select();
+        const { selector, reporter, name } = this.options;
+        const start = Date.now();
+        reporter?.pipelineStart(name);
+        const datasets = await selector.select();
         for await (const dataset of datasets) {
-            const distribution = dataset.getSparqlDistribution() ?? undefined;
-            for (const step of this.config.steps) {
-                const result = await step.execute(dataset, distribution);
-                if (result instanceof NotSupported) {
-                    console.error(result);
+            await this.processDataset(dataset);
+        }
+        reporter?.pipelineComplete({ duration: Date.now() - start });
+    }
+    async processDataset(dataset) {
+        const { distributionResolver, reporter } = this.options;
+        const datasetIri = dataset.iri.toString();
+        reporter?.datasetStart(datasetIri);
+        const resolved = await distributionResolver.resolve(dataset);
+        if (resolved instanceof NoDistributionAvailable) {
+            reporter?.datasetSkipped(datasetIri, resolved.message);
+            return;
+        }
+        try {
+            for (const stage of this.options.stages) {
+                if (stage.stages.length > 0) {
+                    await this.runChain(dataset, resolved.distribution, stage);
                 }
-                else if (result instanceof Readable) {
-                    const promise = new Promise((resolve, reject) => {
-                        result.on('data', (data) => {
-                            // TODO: pipe to writers.
-                            console.log('Data:', data);
-                        });
-                        result.on('error', (error) => {
-                            console.error('rejecting');
-                            reject(error);
-                        });
-                        result.on('end', resolve);
-                    });
-                    await promise;
+                else {
+                    await this.runStage(dataset, resolved.distribution, stage);
                 }
             }
-            for (const step of this.config.steps) {
-                if (isFinishable(step)) {
-                    await step.finish();
+        }
+        catch {
+            // Stage error for this dataset; continue to next dataset.
+        }
+        reporter?.datasetComplete(datasetIri);
+    }
+    async runStage(dataset, distribution, stage) {
+        const { writer, reporter } = this.options;
+        reporter?.stageStart(stage.name);
+        const stageStart = Date.now();
+        let elementsProcessed = 0;
+        let quadsGenerated = 0;
+        const result = await stage.run(dataset, distribution, writer, {
+            onProgress: (elements, quads) => {
+                elementsProcessed = elements;
+                quadsGenerated = quads;
+                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+            },
+        });
+        if (result instanceof NotSupported) {
+            reporter?.stageSkipped(stage.name, result.message);
+        }
+        else {
+            reporter?.stageComplete(stage.name, {
+                elementsProcessed,
+                quadsGenerated,
+                duration: Date.now() - stageStart,
+            });
+        }
+    }
+    async runChain(dataset, distribution, stage) {
+        const { writer, stageOutputResolver, outputDir, outputFormat } = this.options;
+        const outputFiles = [];
+        try {
+            // 1. Run parent stage → FileWriter.
+            const parentWriter = new FileWriter({
+                outputDir: `${outputDir}/${stage.name}`,
+                format: outputFormat,
+            });
+            await this.runChainedStage(dataset, distribution, stage, parentWriter);
+            outputFiles.push(parentWriter.getOutputPath(dataset));
+            // 2. Chain through children.
+            let currentDistribution = await stageOutputResolver.resolve(parentWriter.getOutputPath(dataset));
+            for (let i = 0; i < stage.stages.length; i++) {
+                const child = stage.stages[i];
+                const childWriter = new FileWriter({
+                    outputDir: `${outputDir}/${child.name}`,
+                    format: outputFormat,
+                });
+                await this.runChainedStage(dataset, currentDistribution, child, childWriter);
+                outputFiles.push(childWriter.getOutputPath(dataset));
+                if (i < stage.stages.length - 1) {
+                    currentDistribution = await stageOutputResolver.resolve(childWriter.getOutputPath(dataset));
                 }
             }
+            // 3. Concatenate all output files → user writer.
+            await writer.write(dataset, this.readFiles(outputFiles));
+        }
+        finally {
+            await stageOutputResolver.cleanup();
+        }
+    }
+    async runChainedStage(dataset, distribution, stage, stageWriter) {
+        const { reporter } = this.options;
+        reporter?.stageStart(stage.name);
+        const stageStart = Date.now();
+        let elementsProcessed = 0;
+        let quadsGenerated = 0;
+        const result = await stage.run(dataset, distribution, stageWriter, {
+            onProgress: (elements, quads) => {
+                elementsProcessed = elements;
+                quadsGenerated = quads;
+                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+            },
+        });
+        if (result instanceof NotSupported) {
+            reporter?.stageSkipped(stage.name, result.message);
+            throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
+        }
+        reporter?.stageComplete(stage.name, {
+            elementsProcessed,
+            quadsGenerated,
+            duration: Date.now() - stageStart,
+        });
+    }
+    async *readFiles(paths) {
+        for (const path of paths) {
+            const stream = createReadStream(path);
+            const parser = new StreamParser();
+            stream.pipe(parser);
+            for await (const quad of parser) {
+                yield quad;
+            }
         }
     }
 }
-const isFinishable = (step) => {
-    return typeof step.finish === 'function';
-};

package/dist/progressReporter.d.ts

@@ -0,0 +1,21 @@
+export interface ProgressReporter {
+    pipelineStart(name: string): void;
+    datasetStart(dataset: string): void;
+    stageStart(stage: string): void;
+    stageProgress(update: {
+        elementsProcessed: number;
+        quadsGenerated: number;
+    }): void;
+    stageComplete(stage: string, result: {
+        elementsProcessed: number;
+        quadsGenerated: number;
+        duration: number;
+    }): void;
+    stageSkipped(stage: string, reason: string): void;
+    datasetComplete(dataset: string): void;
+    datasetSkipped(dataset: string, reason: string): void;
+    pipelineComplete(result: {
+        duration: number;
+    }): void;
+}
+//# sourceMappingURL=progressReporter.d.ts.map

package/dist/progressReporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,aAAa,CAAC,MAAM,EAAE;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC;IACT,aAAa,CACX,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAClD,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtD,gBAAgB,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACtD"}

package/dist/progressReporter.js

@@ -0,0 +1 @@
+export {};

package/dist/sparql/executor.d.ts

@@ -2,8 +2,13 @@ import { Dataset, Distribution } from '@lde/dataset';
 import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import type { NamedNode, Quad, Stream } from '@rdfjs/types';
 import type { Readable } from 'node:stream';
-import { NotSupported } from '../step.js';
-export { NotSupported } from '../step.js';
+/**
+ * An executor could not run because the dataset lacks a supported distribution.
+ */
+export declare class NotSupported {
+    readonly message: string;
+    constructor(message: string);
+}
 /** A single row of variable bindings (variable name → NamedNode). */
 export type VariableBindings = Record<string, NamedNode>;
 export interface ExecuteOptions {

package/dist/sparql/executor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAI5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAK1C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAcnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,UAAU,CAAC;IAoBtB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,OAAO,EAAE,OAAO,GACf,MAAM,CAWR;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAO5C;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAcnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,UAAU,CAAC;IAoBtB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,OAAO,EAAE,OAAO,GACf,MAAM,CAWR;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}

package/dist/sparql/executor.js

@@ -4,8 +4,15 @@ import { resolve } from 'node:path';
 import { Generator, Parser } from 'sparqljs';
 import { withDefaultGraph } from './graph.js';
 import { injectValues } from './values.js';
-// Re-export for convenience
-export { NotSupported } from '../step.js';
+/**
+ * An executor could not run because the dataset lacks a supported distribution.
+ */
+export class NotSupported {
+    message;
+    constructor(message) {
+        this.message = message;
+    }
+}
 /**
  * A streaming SPARQL CONSTRUCT executor that parses the query once (in the
  * constructor) and operates on the AST for graph and VALUES injection.

package/dist/stage.d.ts

@@ -10,12 +10,15 @@ export interface StageOptions {
     batchSize?: number;
     /** Maximum concurrent in-flight executor batches. @default 10 */
     maxConcurrency?: number;
+    /** Child stages that chain off this stage's output. */
+    stages?: Stage[];
 }
 export interface RunOptions {
     onProgress?: (elementsProcessed: number, quadsGenerated: number) => void;
 }
 export declare class Stage {
     readonly name: string;
+    readonly stages: readonly Stage[];
     private readonly executors;
     private readonly selector?;
     private readonly batchSize;

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAU3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAajB,eAAe;YA8Gf,UAAU;CAqBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAW3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAajB,eAAe;YA8Gf,UAAU;CAqBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}

package/dist/stage.js

@@ -3,12 +3,14 @@ import { batch } from './batch.js';
 import { AsyncQueue } from './asyncQueue.js';
 export class Stage {
     name;
+    stages;
     executors;
     selector;
     batchSize;
     maxConcurrency;
     constructor(options) {
         this.name = options.name;
+        this.stages = options.stages ?? [];
         this.executors = Array.isArray(options.executors)
             ? options.executors
             : [options.executors];

package/dist/stageOutputResolver.d.ts

@@ -0,0 +1,6 @@
+import { Distribution } from '@lde/dataset';
+export interface StageOutputResolver {
+    resolve(outputPath: string): Promise<Distribution>;
+    cleanup(): Promise<void>;
+}
+//# sourceMappingURL=stageOutputResolver.d.ts.map

package/dist/stageOutputResolver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stageOutputResolver.d.ts","sourceRoot":"","sources":["../src/stageOutputResolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACnD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B"}

package/dist/stageOutputResolver.js

@@ -0,0 +1 @@
+export {};

package/dist/writer/fileWriter.d.ts

@@ -14,10 +14,11 @@ export interface FileWriterOptions {
 }
 export declare class FileWriter implements Writer {
     private readonly outputDir;
-    private readonly format;
+    readonly format: 'turtle' | 'n-triples' | 'n-quads';
     constructor(options: FileWriterOptions);
     write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
-    private getFilename;
+    getOutputPath(dataset: Dataset): string;
+    getFilename(dataset: Dataset): string;
     private getExtension;
 }
 //# sourceMappingURL=fileWriter.d.ts.map

package/dist/writer/fileWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;CAC7C;AAaD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqC;gBAEhD,OAAO,EAAE,iBAAiB;IAKhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBxE,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,YAAY;CAUrB"}
+{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;CAC7C;AAaD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;gBAExC,OAAO,EAAE,iBAAiB;IAKhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBxE,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,YAAY;CAUrB"}

package/dist/writer/fileWriter.js

@@ -43,6 +43,9 @@ export class FileWriter {
             });
         });
     }
+    getOutputPath(dataset) {
+        return join(this.outputDir, this.getFilename(dataset));
+    }
     getFilename(dataset) {
         const extension = this.getExtension();
         const baseName = filenamifyUrl(dataset.iri.toString(), {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.24",
+  "version": "0.6.25",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"
@@ -23,12 +23,10 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "0.6.8",
-    "@lde/dataset-registry-client": "0.6.15",
-    "@lde/sparql-importer": "0.2.8",
-    "@lde/sparql-server": "0.4.8",
+    "@lde/dataset": "0.6.9",
+    "@lde/dataset-registry-client": "0.6.16",
+    "@lde/sparql-importer": "0.2.9",
     "@rdfjs/types": "^2.0.1",
-    "c12": "^3.0.2",
     "fetch-sparql-endpoint": "^7.1.0",
     "filenamify-url": "^3.0.0",
     "n3": "^1.17.0",

package/dist/builder.d.ts

@@ -1,120 +0,0 @@
-import { Selector, ManualDatasetSelection, RegistrySelector } from './selector.js';
-import { Step } from './step.js';
-/**
- * Configuration for QLever SPARQL server.
- */
-export interface QleverConfig {
-    /**
-     * Execution mode: 'docker' for containerized, 'native' for local binary.
-     */
-    mode: 'docker' | 'native';
-    /**
-     * Docker image to use (for docker mode).
-     * @default 'adfreiburg/qlever'
-     */
-    image?: string;
-    /**
-     * Port for the SPARQL endpoint.
-     * @default 7001
-     */
-    port?: number;
-    /**
-     * Working directory for imports.
-     */
-    workingDir?: string;
-}
-/**
- * Writer configuration.
- */
-export interface WriterConfig {
-    type: 'file' | 'sparql';
-    outputDir?: string;
-    endpoint?: URL;
-    /**
-     * Value for the Authorization header sent with SPARQL UPDATE requests, e.g.
-     * `"Basic dXNlcjpwYXNz"`, `"Bearer my-token"`, or `"GDB eyJ…"`.
-     */
-    auth?: string;
-}
-/**
- * Complete pipeline configuration.
- */
-export interface PipelineConfig {
-    selector: Selector;
-    steps: Step[];
-    writers?: WriterConfig[];
-    qlever?: QleverConfig;
-}
-/**
- * Fluent builder for creating pipeline configurations.
- *
- * @example
- * ```typescript
- * const config = PipelineBuilder.create()
- *   .withSelector(registry('https://example.com/sparql'))
- *   .addStep(sparqlQuery('queries/triples.rq'))
- *   .addWriter(fileWriter({ outputDir: 'output' }))
- *   .build();
- * ```
- */
-export declare class PipelineBuilder {
-    private selector?;
-    private steps;
-    private writers;
-    private qleverConfig?;
-    /**
-     * Create a new PipelineBuilder instance.
-     */
-    static create(): PipelineBuilder;
-    /**
-     * Set the dataset selector.
-     */
-    withSelector(selector: Selector): this;
-    /**
-     * Configure QLever for local SPARQL imports.
-     */
-    withQlever(config: QleverConfig): this;
-    /**
-     * Add a single step to the pipeline.
-     */
-    addStep(step: Step): this;
-    /**
-     * Add multiple steps to the pipeline.
-     */
-    addSteps(...steps: Step[]): this;
-    /**
-     * Add a writer for pipeline output.
-     */
-    addWriter(writer: WriterConfig): this;
-    /**
-     * Build the final pipeline configuration.
-     * @throws Error if selector is not set
-     */
-    build(): PipelineConfig;
-}
-/**
- * Create a selector that queries a Dataset Registry.
- *
- * @param endpoint SPARQL endpoint URL of the registry
- */
-export declare function registry(endpoint: string | URL): RegistrySelector;
-/**
- * Create a selector for manually specified datasets.
- *
- * @param datasets Array of dataset IRIs
- */
-export declare function manual(...datasetIris: URL[]): ManualDatasetSelection;
-/**
- * Create a file writer configuration.
- */
-export declare function fileWriter(options: {
-    outputDir: string;
-}): WriterConfig;
-/**
- * Create a SPARQL UPDATE writer configuration.
- */
-export declare function sparqlWriter(options: {
-    endpoint: URL;
-    auth?: string;
-}): WriterConfig;
-//# sourceMappingURL=builder.d.ts.map

package/dist/builder.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../src/builder.ts"],"names":[],"mappings":"AACA,OAAO,EACL,QAAQ,EACR,sBAAsB,EACtB,gBAAgB,EACjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAW;IAC5B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,YAAY,CAAC,CAAe;IAEpC;;OAEG;IACH,MAAM,CAAC,MAAM,IAAI,eAAe;IAIhC;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAKtC;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAKtC;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAKzB;;OAEG;IACH,QAAQ,CAAC,GAAG,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI;IAKhC;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAKrC;;;OAGG;IACH,KAAK,IAAI,cAAc;CAYxB;AAID;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,GAAG,gBAAgB,CAMjE;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,GAAG,WAAW,EAAE,GAAG,EAAE,GAAG,sBAAsB,CAKpE;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,YAAY,CAKvE;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE;IACpC,QAAQ,EAAE,GAAG,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,YAAY,CAMf"}

package/dist/builder.js

@@ -1,116 +0,0 @@
-import { Dataset } from '@lde/dataset';
-import { ManualDatasetSelection, RegistrySelector, } from './selector.js';
-import { Client } from '@lde/dataset-registry-client';
-/**
- * Fluent builder for creating pipeline configurations.
- *
- * @example
- * ```typescript
- * const config = PipelineBuilder.create()
- *   .withSelector(registry('https://example.com/sparql'))
- *   .addStep(sparqlQuery('queries/triples.rq'))
- *   .addWriter(fileWriter({ outputDir: 'output' }))
- *   .build();
- * ```
- */
-export class PipelineBuilder {
-    selector;
-    steps = [];
-    writers = [];
-    qleverConfig;
-    /**
-     * Create a new PipelineBuilder instance.
-     */
-    static create() {
-        return new PipelineBuilder();
-    }
-    /**
-     * Set the dataset selector.
-     */
-    withSelector(selector) {
-        this.selector = selector;
-        return this;
-    }
-    /**
-     * Configure QLever for local SPARQL imports.
-     */
-    withQlever(config) {
-        this.qleverConfig = config;
-        return this;
-    }
-    /**
-     * Add a single step to the pipeline.
-     */
-    addStep(step) {
-        this.steps.push(step);
-        return this;
-    }
-    /**
-     * Add multiple steps to the pipeline.
-     */
-    addSteps(...steps) {
-        this.steps.push(...steps);
-        return this;
-    }
-    /**
-     * Add a writer for pipeline output.
-     */
-    addWriter(writer) {
-        this.writers.push(writer);
-        return this;
-    }
-    /**
-     * Build the final pipeline configuration.
-     * @throws Error if selector is not set
-     */
-    build() {
-        if (!this.selector) {
-            throw new Error('Selector is required. Use withSelector() to set it.');
-        }
-        return {
-            selector: this.selector,
-            steps: this.steps,
-            writers: this.writers.length > 0 ? this.writers : undefined,
-            qlever: this.qleverConfig,
-        };
-    }
-}
-// Helper functions for fluent construction.
-/**
- * Create a selector that queries a Dataset Registry.
- *
- * @param endpoint SPARQL endpoint URL of the registry
- */
-export function registry(endpoint) {
-    return new RegistrySelector({
-        registry: new Client(typeof endpoint === 'string' ? new URL(endpoint) : endpoint),
-    });
-}
-/**
- * Create a selector for manually specified datasets.
- *
- * @param datasets Array of dataset IRIs
- */
-export function manual(...datasetIris) {
-    const datasets = datasetIris.map((iri) => new Dataset({ iri, distributions: [] }));
-    return new ManualDatasetSelection(datasets);
-}
-/**
- * Create a file writer configuration.
- */
-export function fileWriter(options) {
-    return {
-        type: 'file',
-        outputDir: options.outputDir,
-    };
-}
-/**
- * Create a SPARQL UPDATE writer configuration.
- */
-export function sparqlWriter(options) {
-    return {
-        type: 'sparql',
-        endpoint: options.endpoint,
-        auth: options.auth,
-    };
-}

package/dist/config.d.ts

@@ -1,71 +0,0 @@
-import { PipelineConfig, QleverConfig } from './builder.js';
-/**
- * Raw configuration schema from YAML/JSON files.
- */
-export interface RawPipelineConfig {
-    selector?: {
-        type: 'registry' | 'manual';
-        endpoint?: string;
-        datasets?: string[];
-    };
-    qlever?: QleverConfig;
-    steps?: Array<{
-        type: 'sparql-query';
-        query: string;
-    }>;
-    writers?: Array<{
-        type: 'file' | 'sparql';
-        outputDir?: string;
-        endpoint?: string;
-        auth?: string;
-    }>;
-}
-/**
- * Options for loading pipeline configuration.
- */
-export interface LoadConfigOptions {
-    /**
-     * Configuration file name (without extension).
-     * @default 'pipeline.config'
-     */
-    name?: string;
-    /**
-     * Working directory to search for config files.
-     * @default process.cwd()
-     */
-    cwd?: string;
-}
-/**
- * Define a pipeline configuration with TypeScript type checking.
- *
- * @example
- * ```typescript
- * // pipeline.config.ts
- * import { defineConfig } from '@lde/pipeline';
- *
- * export default defineConfig({
- *   selector: { type: 'registry', endpoint: 'https://example.com/sparql' },
- *   steps: [{ type: 'sparql-query', query: 'queries/triples.rq' }],
- * });
- * ```
- */
-export declare function defineConfig(config: RawPipelineConfig): RawPipelineConfig;
-/**
- * Load pipeline configuration from files.
- *
- * Searches for configuration files in the following order:
- * - pipeline.config.ts
- * - pipeline.config.js
- * - pipeline.config.yaml
- * - pipeline.config.yml
- * - pipeline.config.json
- *
- * @param options Load options
- * @returns Resolved pipeline configuration
- */
-export declare function loadPipelineConfig(options?: LoadConfigOptions): Promise<PipelineConfig>;
-/**
- * Normalize raw configuration into a typed PipelineConfig.
- */
-export declare function normalizeConfig(raw: RawPipelineConfig): PipelineConfig;
-//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,EACL,cAAc,EACd,YAAY,EAIb,MAAM,cAAc,CAAC;AAKtB;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE;QACT,IAAI,EAAE,UAAU,GAAG,QAAQ,CAAC;QAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,cAAc,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,iBAAiB,GAAG,iBAAiB,CAEzE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,cAAc,CAAC,CAWzB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,iBAAiB,GAAG,cAAc,CAOtE"}

package/dist/config.js

@@ -1,114 +0,0 @@
-import { loadConfig } from 'c12';
-import { registry, manual, } from './builder.js';
-import { SparqlQuery } from './step/sparqlQuery.js';
-/**
- * Define a pipeline configuration with TypeScript type checking.
- *
- * @example
- * ```typescript
- * // pipeline.config.ts
- * import { defineConfig } from '@lde/pipeline';
- *
- * export default defineConfig({
- *   selector: { type: 'registry', endpoint: 'https://example.com/sparql' },
- *   steps: [{ type: 'sparql-query', query: 'queries/triples.rq' }],
- * });
- * ```
- */
-export function defineConfig(config) {
-    return config;
-}
-/**
- * Load pipeline configuration from files.
- *
- * Searches for configuration files in the following order:
- * - pipeline.config.ts
- * - pipeline.config.js
- * - pipeline.config.yaml
- * - pipeline.config.yml
- * - pipeline.config.json
- *
- * @param options Load options
- * @returns Resolved pipeline configuration
- */
-export async function loadPipelineConfig(options) {
-    const { config } = await loadConfig({
-        name: options?.name ?? 'pipeline.config',
-        cwd: options?.cwd,
-    });
-    if (!config) {
-        throw new Error('No pipeline configuration found');
-    }
-    return normalizeConfig(config);
-}
-/**
- * Normalize raw configuration into a typed PipelineConfig.
- */
-export function normalizeConfig(raw) {
-    return {
-        selector: normalizeSelector(raw.selector),
-        steps: normalizeSteps(raw.steps),
-        writers: normalizeWriters(raw.writers),
-        qlever: raw.qlever,
-    };
-}
-function normalizeSelector(raw) {
-    if (!raw) {
-        throw new Error('Selector configuration is required');
-    }
-    switch (raw.type) {
-        case 'registry':
-            if (!raw.endpoint) {
-                throw new Error('Registry selector requires endpoint');
-            }
-            return registry(raw.endpoint);
-        case 'manual':
-            if (!raw.datasets || raw.datasets.length === 0) {
-                throw new Error('Manual selector requires datasets');
-            }
-            return manual(...raw.datasets.map((d) => new URL(d)));
-        default:
-            throw new Error(`Unknown selector type: ${raw.type}`);
-    }
-}
-function normalizeSteps(raw) {
-    if (!raw) {
-        return [];
-    }
-    return raw.map((step) => {
-        switch (step.type) {
-            case 'sparql-query':
-                return new SparqlQuery({
-                    identifier: step.query,
-                    query: step.query, // Will be loaded from file by SparqlQuery.fromFile if path
-                });
-            default:
-                throw new Error(`Unknown step type: ${step.type}`);
-        }
-    });
-}
-function normalizeWriters(raw) {
-    if (!raw || raw.length === 0) {
-        return undefined;
-    }
-    return raw.map((writer) => {
-        switch (writer.type) {
-            case 'file':
-                if (!writer.outputDir) {
-                    throw new Error('File writer requires outputDir');
-                }
-                return { type: 'file', outputDir: writer.outputDir };
-            case 'sparql':
-                if (!writer.endpoint) {
-                    throw new Error('SPARQL writer requires endpoint');
-                }
-                return {
-                    type: 'sparql',
-                    endpoint: new URL(writer.endpoint),
-                    auth: writer.auth,
-                };
-            default:
-                throw new Error(`Unknown writer type: ${writer.type}`);
-        }
-    });
-}

package/dist/import.d.ts

@@ -1,30 +0,0 @@
-import { Dataset } from '@lde/dataset';
-import { Failure, Finishable, NotSupported, SingleStep, Success } from './step.js';
-import { Importer } from '@lde/sparql-importer';
-import { SparqlServer } from '@lde/sparql-server';
-/**
- * A pipeline step that imports a database using an {@link Importer} and makes
- * the import available at a local SPARQL endpoint.
- */
-export declare class Import implements SingleStep, Finishable {
-    readonly identifier = "import";
-    private readonly importer;
-    private readonly server;
-    private readonly forceImport;
-    /**
-     * Create a Pipeline ImportStep.
-     *
-     * @param {object} args
-     * @param args.importer A concrete importer that will import the distribution if needed.
-     * @param args.server SPARQL server that will be started to serve the imported data.
-     * @param args.forceImport Whether to force an import even if the dataset already has a SPARQL distribution.
-     */
-    constructor({ importer, server, forceImport, }: {
-        importer: Importer;
-        server: SparqlServer;
-        forceImport?: boolean;
-    });
-    execute(dataset: Dataset): Promise<NotSupported | Failure | Success>;
-    finish(): Promise<void>;
-}
-//# sourceMappingURL=import.d.ts.map

package/dist/import.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"import.d.ts","sourceRoot":"","sources":["../src/import.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,OAAO,EACP,UAAU,EACV,YAAY,EACZ,UAAU,EACV,OAAO,EACR,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,QAAQ,EAGT,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD;;;GAGG;AACH,qBAAa,MAAO,YAAW,UAAU,EAAE,UAAU;IACnD,SAAgB,UAAU,YAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IAEtC;;;;;;;OAOG;gBACS,EACV,QAAQ,EACR,MAAM,EACN,WAAW,GACZ,EAAE;QACD,QAAQ,EAAE,QAAQ,CAAC;QACnB,MAAM,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAMY,OAAO,CAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,GAAG,OAAO,GAAG,OAAO,CAAC;IAuB/B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAGrC"}

package/dist/import.js

@@ -1,44 +0,0 @@
-import { Distribution } from '@lde/dataset';
-import { Failure, NotSupported, Success, } from './step.js';
-import { ImportFailed, NotSupported as ImporterNotSupported, } from '@lde/sparql-importer';
-/**
- * A pipeline step that imports a database using an {@link Importer} and makes
- * the import available at a local SPARQL endpoint.
- */
-export class Import {
-    identifier = 'import';
-    importer;
-    server;
-    forceImport;
-    /**
-     * Create a Pipeline ImportStep.
-     *
-     * @param {object} args
-     * @param args.importer A concrete importer that will import the distribution if needed.
-     * @param args.server SPARQL server that will be started to serve the imported data.
-     * @param args.forceImport Whether to force an import even if the dataset already has a SPARQL distribution.
-     */
-    constructor({ importer, server, forceImport, }) {
-        this.importer = importer;
-        this.server = server;
-        this.forceImport = forceImport ?? false;
-    }
-    async execute(dataset) {
-        if (dataset.getSparqlDistribution()?.isValid && !this.forceImport) {
-            return new NotSupported('A valid SPARQL distribution is available so no import needed');
-        }
-        const result = await this.importer.import(dataset);
-        if (result instanceof ImporterNotSupported) {
-            return new NotSupported('No download distribution available');
-        }
-        if (result instanceof ImportFailed) {
-            return new Failure(result.distribution, result.error);
-        }
-        await this.server.start();
-        dataset.distributions.push(Distribution.sparql(this.server.queryEndpoint));
-        return new Success(dataset, result.distribution);
-    }
-    async finish() {
-        await this.server.stop();
-    }
-}

package/dist/step/sparqlQuery.d.ts

@@ -1,35 +0,0 @@
-import { DataEmittingStep } from './../step.js';
-import { Dataset, Distribution } from '@lde/dataset';
-import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
-/**
- * Arguments for the SparqlQuery step.
- *
- * @param identifier Unique identifier for the step.
- * @param query: SPARQL CONSTRUCT query to execute.
- * @param fetcher Optional SPARQL endpoint fetcher; defaults to SparqlEndpointFetcher.
- */
-export interface Args {
-    identifier: string;
-    query: string;
-    fetcher?: SparqlEndpointFetcher;
-}
-/**
- * Executes a SPARQL CONSTRUCT query and emits the resulting quads.
- *
- * This step wraps the SparqlConstructExecutor to provide the DataEmittingStep interface
- * for use in pipelines. Supports legacy template substitution (`#namedGraph#`,
- * `#subjectFilter#`, `?dataset`); for new code prefer the AST-based executor directly.
- */
-export declare class SparqlQuery implements DataEmittingStep {
-    readonly identifier: string;
-    private readonly query;
-    private readonly fetcher?;
-    constructor({ identifier, query, fetcher }: Args);
-    execute(dataset: Dataset, distribution: Distribution): Promise<import("../sparql/executor.js").QuadStream>;
-    static fromFile(filename: string): Promise<SparqlQuery>;
-}
-/**
- * @deprecated Use readQueryFile from '@lde/pipeline/sparql' instead.
- */
-export declare function fromFile(filename: string): Promise<string>;
-//# sourceMappingURL=sparqlQuery.d.ts.map

package/dist/step/sparqlQuery.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"sparqlQuery.d.ts","sourceRoot":"","sources":["../../src/step/sparqlQuery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAO9D;;;;;;GAMG;AACH,MAAM,WAAW,IAAI;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;GAMG;AACH,qBAAa,WAAY,YAAW,gBAAgB;IAClD,SAAgB,UAAU,SAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAwB;gBAErC,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,IAAI;IAM1C,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY;WAatC,QAAQ,CAAC,QAAQ,EAAE,MAAM;CAM9C;AAED;;GAEG;AACH,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,MAAM,mBAE9C"}

package/dist/step/sparqlQuery.js

@@ -1,38 +0,0 @@
-import { SparqlConstructExecutor, substituteQueryTemplates, readQueryFile, } from '../sparql/index.js';
-/**
- * Executes a SPARQL CONSTRUCT query and emits the resulting quads.
- *
- * This step wraps the SparqlConstructExecutor to provide the DataEmittingStep interface
- * for use in pipelines. Supports legacy template substitution (`#namedGraph#`,
- * `#subjectFilter#`, `?dataset`); for new code prefer the AST-based executor directly.
- */
-export class SparqlQuery {
-    identifier;
-    query;
-    fetcher;
-    constructor({ identifier, query, fetcher }) {
-        this.identifier = identifier;
-        this.query = query;
-        this.fetcher = fetcher;
-    }
-    async execute(dataset, distribution) {
-        const substituted = substituteQueryTemplates(this.query, distribution, dataset);
-        const executor = new SparqlConstructExecutor({
-            query: substituted,
-            fetcher: this.fetcher,
-        });
-        return await executor.execute(dataset, distribution);
-    }
-    static async fromFile(filename) {
-        return new this({
-            identifier: filename,
-            query: await readQueryFile(filename),
-        });
-    }
-}
-/**
- * @deprecated Use readQueryFile from '@lde/pipeline/sparql' instead.
- */
-export async function fromFile(filename) {
-    return readQueryFile(filename);
-}

package/dist/step.d.ts

@@ -1,55 +0,0 @@
-import { Dataset, Distribution } from '@lde/dataset';
-import type { Stream } from '@rdfjs/types';
-interface AbstractStep {
-    readonly identifier: string;
-}
-export type Step = DataEmittingStep | SingleStep;
-/**
- * A pipeline step that returns a data-emitting stream of RDF quads.
- * Failure is expressed by emitting an error event; success by the end event.
- */
-export interface DataEmittingStep extends AbstractStep {
-    execute(dataset: Dataset, distribution: Distribution): Promise<Stream | NotSupported>;
-}
-/**
- * A pipeline step that executes an operation without emitting data.
- */
-export interface SingleStep extends AbstractStep {
-    execute(dataset: Dataset, distribution?: Distribution): Promise<NotSupported | Failure | Success>;
-}
-export interface Finishable {
-    finish(): Promise<void>;
-}
-/**
- * A pipeline step failed to run.
- *
- * @param distribution The distribution that was processed.
- * @param message Optional error message.
- */
-export declare class Failure {
-    readonly distribution: Distribution;
-    readonly message?: string | undefined;
-    constructor(distribution: Distribution, message?: string | undefined);
-}
-/**
- * A pipeline ran successfully.
- *
- * @param dataset: The dataset, with possible modifications, that was processed.
- * @param distribution The distribution that was processed.
- */
-export declare class Success {
-    readonly dataset: Dataset;
-    readonly distribution: Distribution;
-    constructor(dataset: Dataset, distribution: Distribution);
-}
-/**
- * A pipeline step could not be run because the dataset lacks a distribution supported by the step.
- *
- * @param message: A message explaining why the step is not supported.
- */
-export declare class NotSupported {
-    readonly message: string;
-    constructor(message: string);
-}
-export {};
-//# sourceMappingURL=step.d.ts.map

package/dist/step.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"step.d.ts","sourceRoot":"","sources":["../src/step.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE3C,UAAU,YAAY;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,MAAM,IAAI,GAAG,gBAAgB,GAAG,UAAU,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,CAAC,EAAE,YAAY,GAC1B,OAAO,CAAC,YAAY,GAAG,OAAO,GAAG,OAAO,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;aAEA,YAAY,EAAE,YAAY;aAC1B,OAAO,CAAC,EAAE,MAAM;gBADhB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,MAAM,YAAA;CAEnC;AAED;;;;;GAKG;AACH,qBAAa,OAAO;aAEA,OAAO,EAAE,OAAO;aAChB,YAAY,EAAE,YAAY;gBAD1B,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY;CAE7C;AAED;;;;GAIG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C"}

package/dist/step.js

@@ -1,39 +0,0 @@
-/**
- * A pipeline step failed to run.
- *
- * @param distribution The distribution that was processed.
- * @param message Optional error message.
- */
-export class Failure {
-    distribution;
-    message;
-    constructor(distribution, message) {
-        this.distribution = distribution;
-        this.message = message;
-    }
-}
-/**
- * A pipeline ran successfully.
- *
- * @param dataset: The dataset, with possible modifications, that was processed.
- * @param distribution The distribution that was processed.
- */
-export class Success {
-    dataset;
-    distribution;
-    constructor(dataset, distribution) {
-        this.dataset = dataset;
-        this.distribution = distribution;
-    }
-}
-/**
- * A pipeline step could not be run because the dataset lacks a distribution supported by the step.
- *
- * @param message: A message explaining why the step is not supported.
- */
-export class NotSupported {
-    message;
-    constructor(message) {
-        this.message = message;
-    }
-}