@lde/pipeline 0.6.24 → 0.6.26

package/dist/index.d.ts

@@ -1,12 +1,10 @@
 export * from './asyncQueue.js';
 export * from './batch.js';
 export * from './pipeline.js';
+export * from './progressReporter.js';
 export * from './selector.js';
 export * from './stage.js';
-export * from './step.js';
-export * from './step/sparqlQuery.js';
-export * from './builder.js';
-export * from './config.js';
+export * from './stageOutputResolver.js';
 export * from './sparql/index.js';
 export * from './distribution/index.js';
 export * from './writer/index.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC"}

package/dist/index.js

@@ -1,12 +1,10 @@
 export * from './asyncQueue.js';
 export * from './batch.js';
 export * from './pipeline.js';
+export * from './progressReporter.js';
 export * from './selector.js';
 export * from './stage.js';
-export * from './step.js';
-export * from './step/sparqlQuery.js';
-export * from './builder.js';
-export * from './config.js';
+export * from './stageOutputResolver.js';
 export * from './sparql/index.js';
 export * from './distribution/index.js';
 export * from './writer/index.js';

package/dist/pipeline.d.ts

@@ -1,11 +1,28 @@
-import { Selector } from './selector.js';
-import { Step } from './step.js';
+import type { Selector } from './selector.js';
+import { Stage } from './stage.js';
+import type { Writer } from './writer/writer.js';
+import { type DistributionResolver } from './distribution/resolver.js';
+import type { StageOutputResolver } from './stageOutputResolver.js';
+import type { ProgressReporter } from './progressReporter.js';
+export interface PipelineOptions {
+    name: string;
+    selector: Selector;
+    stages: Stage[];
+    writer: Writer;
+    distributionResolver: DistributionResolver;
+    stageOutputResolver?: StageOutputResolver;
+    outputDir?: string;
+    outputFormat?: 'turtle' | 'n-triples' | 'n-quads';
+    reporter?: ProgressReporter;
+}
 export declare class Pipeline {
-    private readonly config;
-    constructor(config: {
-        selector: Selector;
-        steps: Step[];
-    });
+    private readonly options;
+    constructor(options: PipelineOptions);
     run(): Promise<void>;
+    private processDataset;
+    private runStage;
+    private runChain;
+    private runChainedStage;
+    private readFiles;
 }
 //# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAc,IAAI,EAAE,MAAM,WAAW,CAAC;AAI7C,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE;QAAE,QAAQ,EAAE,QAAQ,CAAC;QAAC,KAAK,EAAE,IAAI,EAAE,CAAA;KAAE;IAE7D,GAAG;CAiCjB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,oBAAoB,CAAC;IAC3C,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAClD,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,eAAe;IAe9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAcZ,cAAc;YA2Bd,QAAQ;YAgCR,QAAQ;YAoDR,eAAe;YAoCd,SAAS;CAUzB"}

package/dist/pipeline.js

@@ -1,42 +1,141 @@
-import { NotSupported } from './step.js';
-import { Readable } from 'node:stream';
+import { createReadStream } from 'node:fs';
+import { StreamParser } from 'n3';
+import { FileWriter } from './writer/fileWriter.js';
+import { NoDistributionAvailable, } from './distribution/resolver.js';
+import { NotSupported } from './sparql/executor.js';
 export class Pipeline {
-    config;
-    constructor(config) {
-        this.config = config;
+    options;
+    constructor(options) {
+        const hasSubStages = options.stages.some((stage) => stage.stages.length > 0);
+        if (hasSubStages && !options.stageOutputResolver) {
+            throw new Error('stageOutputResolver is required when any stage has sub-stages');
+        }
+        if (hasSubStages && !options.outputDir) {
+            throw new Error('outputDir is required when any stage has sub-stages');
+        }
+        this.options = options;
     }
     async run() {
-        const datasets = await this.config.selector.select();
+        const { selector, reporter, name } = this.options;
+        const start = Date.now();
+        reporter?.pipelineStart(name);
+        const datasets = await selector.select();
         for await (const dataset of datasets) {
-            const distribution = dataset.getSparqlDistribution() ?? undefined;
-            for (const step of this.config.steps) {
-                const result = await step.execute(dataset, distribution);
-                if (result instanceof NotSupported) {
-                    console.error(result);
+            await this.processDataset(dataset);
+        }
+        reporter?.pipelineComplete({ duration: Date.now() - start });
+    }
+    async processDataset(dataset) {
+        const { distributionResolver, reporter } = this.options;
+        const datasetIri = dataset.iri.toString();
+        reporter?.datasetStart(datasetIri);
+        const resolved = await distributionResolver.resolve(dataset);
+        if (resolved instanceof NoDistributionAvailable) {
+            reporter?.datasetSkipped(datasetIri, resolved.message);
+            return;
+        }
+        try {
+            for (const stage of this.options.stages) {
+                if (stage.stages.length > 0) {
+                    await this.runChain(dataset, resolved.distribution, stage);
                 }
-                else if (result instanceof Readable) {
-                    const promise = new Promise((resolve, reject) => {
-                        result.on('data', (data) => {
-                            // TODO: pipe to writers.
-                            console.log('Data:', data);
-                        });
-                        result.on('error', (error) => {
-                            console.error('rejecting');
-                            reject(error);
-                        });
-                        result.on('end', resolve);
-                    });
-                    await promise;
+                else {
+                    await this.runStage(dataset, resolved.distribution, stage);
                 }
             }
-            for (const step of this.config.steps) {
-                if (isFinishable(step)) {
-                    await step.finish();
+        }
+        catch {
+            // Stage error for this dataset; continue to next dataset.
+        }
+        reporter?.datasetComplete(datasetIri);
+    }
+    async runStage(dataset, distribution, stage) {
+        const { writer, reporter } = this.options;
+        reporter?.stageStart(stage.name);
+        const stageStart = Date.now();
+        let elementsProcessed = 0;
+        let quadsGenerated = 0;
+        const result = await stage.run(dataset, distribution, writer, {
+            onProgress: (elements, quads) => {
+                elementsProcessed = elements;
+                quadsGenerated = quads;
+                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+            },
+        });
+        if (result instanceof NotSupported) {
+            reporter?.stageSkipped(stage.name, result.message);
+        }
+        else {
+            reporter?.stageComplete(stage.name, {
+                elementsProcessed,
+                quadsGenerated,
+                duration: Date.now() - stageStart,
+            });
+        }
+    }
+    async runChain(dataset, distribution, stage) {
+        const { writer, stageOutputResolver, outputDir, outputFormat } = this.options;
+        const outputFiles = [];
+        try {
+            // 1. Run parent stage → FileWriter.
+            const parentWriter = new FileWriter({
+                outputDir: `${outputDir}/${stage.name}`,
+                format: outputFormat,
+            });
+            await this.runChainedStage(dataset, distribution, stage, parentWriter);
+            outputFiles.push(parentWriter.getOutputPath(dataset));
+            // 2. Chain through children.
+            let currentDistribution = await stageOutputResolver.resolve(parentWriter.getOutputPath(dataset));
+            for (let i = 0; i < stage.stages.length; i++) {
+                const child = stage.stages[i];
+                const childWriter = new FileWriter({
+                    outputDir: `${outputDir}/${child.name}`,
+                    format: outputFormat,
+                });
+                await this.runChainedStage(dataset, currentDistribution, child, childWriter);
+                outputFiles.push(childWriter.getOutputPath(dataset));
+                if (i < stage.stages.length - 1) {
+                    currentDistribution = await stageOutputResolver.resolve(childWriter.getOutputPath(dataset));
                 }
             }
+            // 3. Concatenate all output files → user writer.
+            await writer.write(dataset, this.readFiles(outputFiles));
+        }
+        finally {
+            await stageOutputResolver.cleanup();
+        }
+    }
+    async runChainedStage(dataset, distribution, stage, stageWriter) {
+        const { reporter } = this.options;
+        reporter?.stageStart(stage.name);
+        const stageStart = Date.now();
+        let elementsProcessed = 0;
+        let quadsGenerated = 0;
+        const result = await stage.run(dataset, distribution, stageWriter, {
+            onProgress: (elements, quads) => {
+                elementsProcessed = elements;
+                quadsGenerated = quads;
+                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+            },
+        });
+        if (result instanceof NotSupported) {
+            reporter?.stageSkipped(stage.name, result.message);
+            throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
+        }
+        reporter?.stageComplete(stage.name, {
+            elementsProcessed,
+            quadsGenerated,
+            duration: Date.now() - stageStart,
+        });
+    }
+    async *readFiles(paths) {
+        for (const path of paths) {
+            const stream = createReadStream(path);
+            const parser = new StreamParser();
+            stream.pipe(parser);
+            for await (const quad of parser) {
+                yield quad;
+            }
         }
     }
 }
-const isFinishable = (step) => {
-    return typeof step.finish === 'function';
-};

package/dist/progressReporter.d.ts

@@ -0,0 +1,21 @@
+export interface ProgressReporter {
+    pipelineStart(name: string): void;
+    datasetStart(dataset: string): void;
+    stageStart(stage: string): void;
+    stageProgress(update: {
+        elementsProcessed: number;
+        quadsGenerated: number;
+    }): void;
+    stageComplete(stage: string, result: {
+        elementsProcessed: number;
+        quadsGenerated: number;
+        duration: number;
+    }): void;
+    stageSkipped(stage: string, reason: string): void;
+    datasetComplete(dataset: string): void;
+    datasetSkipped(dataset: string, reason: string): void;
+    pipelineComplete(result: {
+        duration: number;
+    }): void;
+}
+//# sourceMappingURL=progressReporter.d.ts.map

package/dist/progressReporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,aAAa,CAAC,MAAM,EAAE;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC;IACT,aAAa,CACX,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAClD,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtD,gBAAgB,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACtD"}

package/dist/progressReporter.js

@@ -0,0 +1 @@
+export {};

package/dist/sparql/executor.d.ts

@@ -2,8 +2,13 @@ import { Dataset, Distribution } from '@lde/dataset';
 import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import type { NamedNode, Quad, Stream } from '@rdfjs/types';
 import type { Readable } from 'node:stream';
-import { NotSupported } from '../step.js';
-export { NotSupported } from '../step.js';
+/**
+ * An executor could not run because the dataset lacks a supported distribution.
+ */
+export declare class NotSupported {
+    readonly message: string;
+    constructor(message: string);
+}
 /** A single row of variable bindings (variable name → NamedNode). */
 export type VariableBindings = Record<string, NamedNode>;
 export interface ExecuteOptions {
@@ -40,12 +45,16 @@ export interface SparqlConstructExecutorOptions {
     fetcher?: SparqlEndpointFetcher;
 }
 /**
- * A streaming SPARQL CONSTRUCT executor that parses the query once (in the
- * constructor) and operates on the AST for graph and VALUES injection.
+ * A streaming SPARQL CONSTRUCT executor.
+ *
+ * Queries **without** `#subjectFilter#` are parsed once in the constructor
+ * (fast path). Queries that contain the template are stored as raw strings
+ * and parsed at {@link execute} time after substitution.
  *
  * Template substitution (applied in order):
- * 1. `FROM <graph>` — set via `withDefaultGraph` if the distribution has a named graph
- * 2. `?dataset` — replaced with the dataset IRI (string substitution on the serialised query)
+ * 1. `#subjectFilter#` — replaced with `distribution.subjectFilter` (deferred to execute)
+ * 2. `FROM <graph>` — set via `withDefaultGraph` if the distribution has a named graph
+ * 3. `?dataset` — replaced with the dataset IRI (string substitution on the serialised query)
  *
  * @example
  * ```typescript
@@ -63,7 +72,8 @@ export interface SparqlConstructExecutorOptions {
  * ```
  */
 export declare class SparqlConstructExecutor implements Executor {
-    private readonly query;
+    private readonly rawQuery;
+    private readonly preParsed?;
     private readonly fetcher;
     private readonly generator;
     constructor(options: SparqlConstructExecutorOptions);
@@ -84,14 +94,6 @@ export declare class SparqlConstructExecutor implements Executor {
      */
     static fromFile(filename: string, options?: Omit<SparqlConstructExecutorOptions, 'query'>): Promise<SparqlConstructExecutor>;
 }
-/**
- * Substitute template variables in a SPARQL query.
- *
- * - `#subjectFilter#` — replaced with the distribution's subject filter
- * - `#namedGraph#` — replaced with `FROM <graph>` clause if the distribution has a named graph
- * - `?dataset` — replaced with the dataset IRI
- */
-export declare function substituteQueryTemplates(query: string, distribution: Distribution | null, dataset: Dataset): string;
 /**
  * Read a SPARQL query from a file.
  */

package/dist/sparql/executor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAI5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAK1C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAcnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,UAAU,CAAC;IAoBtB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,OAAO,EAAE,OAAO,GACf,MAAM,CAWR;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAO5C;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAkBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,UAAU,CAAC;IAiCtB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}

package/dist/sparql/executor.js

@@ -4,15 +4,26 @@ import { resolve } from 'node:path';
 import { Generator, Parser } from 'sparqljs';
 import { withDefaultGraph } from './graph.js';
 import { injectValues } from './values.js';
-// Re-export for convenience
-export { NotSupported } from '../step.js';
 /**
- * A streaming SPARQL CONSTRUCT executor that parses the query once (in the
- * constructor) and operates on the AST for graph and VALUES injection.
+ * An executor could not run because the dataset lacks a supported distribution.
+ */
+export class NotSupported {
+    message;
+    constructor(message) {
+        this.message = message;
+    }
+}
+/**
+ * A streaming SPARQL CONSTRUCT executor.
+ *
+ * Queries **without** `#subjectFilter#` are parsed once in the constructor
+ * (fast path). Queries that contain the template are stored as raw strings
+ * and parsed at {@link execute} time after substitution.
  *
  * Template substitution (applied in order):
- * 1. `FROM <graph>` — set via `withDefaultGraph` if the distribution has a named graph
- * 2. `?dataset` — replaced with the dataset IRI (string substitution on the serialised query)
+ * 1. `#subjectFilter#` — replaced with `distribution.subjectFilter` (deferred to execute)
+ * 2. `FROM <graph>` — set via `withDefaultGraph` if the distribution has a named graph
+ * 3. `?dataset` — replaced with the dataset IRI (string substitution on the serialised query)
  *
  * @example
  * ```typescript
@@ -30,16 +41,19 @@ export { NotSupported } from '../step.js';
  * ```
  */
 export class SparqlConstructExecutor {
-    query;
+    rawQuery;
+    preParsed;
     fetcher;
     generator = new Generator();
     constructor(options) {
-        const parser = new Parser();
-        const parsed = parser.parse(options.query);
-        if (parsed.type !== 'query' || parsed.queryType !== 'CONSTRUCT') {
-            throw new Error('Query must be a CONSTRUCT query');
+        this.rawQuery = options.query;
+        if (!options.query.includes('#subjectFilter#')) {
+            const parsed = new Parser().parse(options.query);
+            if (parsed.type !== 'query' || parsed.queryType !== 'CONSTRUCT') {
+                throw new Error('Query must be a CONSTRUCT query');
+            }
+            this.preParsed = parsed;
         }
-        this.query = parsed;
         this.fetcher =
             options.fetcher ??
                 new SparqlEndpointFetcher({
@@ -56,7 +70,18 @@ export class SparqlConstructExecutor {
      */
     async execute(dataset, distribution, options) {
         const endpoint = distribution.accessUrl;
-        let ast = structuredClone(this.query);
+        let ast;
+        if (this.preParsed) {
+            ast = structuredClone(this.preParsed);
+        }
+        else {
+            const substituted = this.rawQuery.replace('#subjectFilter#', distribution.subjectFilter ?? '');
+            const parsed = new Parser().parse(substituted);
+            if (parsed.type !== 'query' || parsed.queryType !== 'CONSTRUCT') {
+                throw new Error('Query must be a CONSTRUCT query');
+            }
+            ast = parsed;
+        }
         if (distribution.namedGraph) {
             withDefaultGraph(ast, distribution.namedGraph);
         }
@@ -79,23 +104,6 @@ export class SparqlConstructExecutor {
         return new SparqlConstructExecutor({ ...options, query });
     }
 }
-/**
- * Substitute template variables in a SPARQL query.
- *
- * - `#subjectFilter#` — replaced with the distribution's subject filter
- * - `#namedGraph#` — replaced with `FROM <graph>` clause if the distribution has a named graph
- * - `?dataset` — replaced with the dataset IRI
- */
-export function substituteQueryTemplates(query, distribution, dataset) {
-    const subjectFilter = distribution?.subjectFilter ?? '';
-    const namedGraph = distribution?.namedGraph
-        ? `FROM <${distribution.namedGraph}>`
-        : '';
-    return query
-        .replace('#subjectFilter#', subjectFilter)
-        .replaceAll('?dataset', `<${dataset.iri}>`)
-        .replace('#namedGraph#', namedGraph);
-}
 /**
  * Read a SPARQL query from a file.
  */

package/dist/sparql/index.d.ts

@@ -1,4 +1,4 @@
-export { SparqlConstructExecutor, substituteQueryTemplates, NotSupported, readQueryFile, type ExecuteOptions, type Executor, type SparqlConstructExecutorOptions, type QuadStream, type VariableBindings, } from './executor.js';
+export { SparqlConstructExecutor, NotSupported, readQueryFile, type ExecuteOptions, type Executor, type SparqlConstructExecutorOptions, type QuadStream, type VariableBindings, } from './executor.js';
 export { SparqlSelector, type SparqlSelectorOptions } from './selector.js';
 export { injectValues } from './values.js';
 export { withDefaultGraph } from './graph.js';

package/dist/sparql/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,wBAAwB,EACxB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,UAAU,EACf,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,KAAK,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAE3E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,UAAU,EACf,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,KAAK,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAE3E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC"}

package/dist/sparql/index.js

@@ -1,4 +1,4 @@
-export { SparqlConstructExecutor, substituteQueryTemplates, NotSupported, readQueryFile, } from './executor.js';
+export { SparqlConstructExecutor, NotSupported, readQueryFile, } from './executor.js';
 export { SparqlSelector } from './selector.js';
 export { injectValues } from './values.js';
 export { withDefaultGraph } from './graph.js';

package/dist/stage.d.ts

@@ -2,22 +2,27 @@ import { Dataset, Distribution } from '@lde/dataset';
 import type { Executor, VariableBindings } from './sparql/executor.js';
 import { NotSupported } from './sparql/executor.js';
 import type { Writer } from './writer/writer.js';
+/** A selector, or a factory that receives the runtime distribution. */
+export type StageSelectorInput = StageSelector | ((distribution: Distribution) => StageSelector);
 export interface StageOptions {
     name: string;
     executors: Executor | Executor[];
-    selector?: StageSelector;
+    selector?: StageSelectorInput;
     /** Maximum number of bindings per executor call. @default 10 */
     batchSize?: number;
     /** Maximum concurrent in-flight executor batches. @default 10 */
     maxConcurrency?: number;
+    /** Child stages that chain off this stage's output. */
+    stages?: Stage[];
 }
 export interface RunOptions {
     onProgress?: (elementsProcessed: number, quadsGenerated: number) => void;
 }
 export declare class Stage {
     readonly name: string;
+    readonly stages: readonly Stage[];
     private readonly executors;
-    private readonly selector?;
+    private readonly selectorInput?;
     private readonly batchSize;
     private readonly maxConcurrency;
     constructor(options: StageOptions);

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAU3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAajB,eAAe;YA8Gf,UAAU;CAqBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,uEAAuE;AACvE,MAAM,MAAM,kBAAkB,GAC1B,aAAa,GACb,CAAC,CAAC,YAAY,EAAE,YAAY,KAAK,aAAa,CAAC,CAAC;AAEpD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAqB;IACpD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAW3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAuBjB,eAAe;YA+Gf,UAAU;CAqBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}

package/dist/stage.js

@@ -3,22 +3,27 @@ import { batch } from './batch.js';
 import { AsyncQueue } from './asyncQueue.js';
 export class Stage {
     name;
+    stages;
     executors;
-    selector;
+    selectorInput;
     batchSize;
     maxConcurrency;
     constructor(options) {
         this.name = options.name;
+        this.stages = options.stages ?? [];
         this.executors = Array.isArray(options.executors)
             ? options.executors
             : [options.executors];
-        this.selector = options.selector;
+        this.selectorInput = options.selector;
         this.batchSize = options.batchSize ?? 10;
         this.maxConcurrency = options.maxConcurrency ?? 10;
     }
     async run(dataset, distribution, writer, options) {
-        if (this.selector) {
-            return this.runWithSelector(dataset, distribution, writer, options);
+        if (this.selectorInput) {
+            const selector = typeof this.selectorInput === 'function'
+                ? this.selectorInput(distribution)
+                : this.selectorInput;
+            return this.runWithSelector(selector, dataset, distribution, writer, options);
         }
         const streams = await this.executeAll(dataset, distribution);
         if (streams instanceof NotSupported) {
@@ -26,10 +31,10 @@ export class Stage {
         }
         await writer.write(dataset, mergeStreams(streams));
     }
-    async runWithSelector(dataset, distribution, writer, options) {
+    async runWithSelector(selector, dataset, distribution, writer, options) {
         // Peek the first batch to detect an empty selector before starting the
         // writer (important because e.g. SparqlUpdateWriter does CLEAR GRAPH).
-        const batches = batch(this.selector, this.batchSize);
+        const batches = batch(selector, this.batchSize);
         const iter = batches[Symbol.asyncIterator]();
         const first = await iter.next();
         if (first.done) {

package/dist/stageOutputResolver.d.ts

@@ -0,0 +1,6 @@
+import { Distribution } from '@lde/dataset';
+export interface StageOutputResolver {
+    resolve(outputPath: string): Promise<Distribution>;
+    cleanup(): Promise<void>;
+}
+//# sourceMappingURL=stageOutputResolver.d.ts.map

package/dist/stageOutputResolver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stageOutputResolver.d.ts","sourceRoot":"","sources":["../src/stageOutputResolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACnD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B"}

package/dist/stageOutputResolver.js

@@ -0,0 +1 @@
+export {};

package/dist/writer/fileWriter.d.ts

@@ -14,10 +14,11 @@ export interface FileWriterOptions {
 }
 export declare class FileWriter implements Writer {
     private readonly outputDir;
-    private readonly format;
+    readonly format: 'turtle' | 'n-triples' | 'n-quads';
     constructor(options: FileWriterOptions);
     write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
-    private getFilename;
+    getOutputPath(dataset: Dataset): string;
+    getFilename(dataset: Dataset): string;
     private getExtension;
 }
 //# sourceMappingURL=fileWriter.d.ts.map

package/dist/writer/fileWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;CAC7C;AAaD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqC;gBAEhD,OAAO,EAAE,iBAAiB;IAKhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBxE,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,YAAY;CAUrB"}
+{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;CAC7C;AAaD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;gBAExC,OAAO,EAAE,iBAAiB;IAKhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBxE,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,YAAY;CAUrB"}

package/dist/writer/fileWriter.js

@@ -43,6 +43,9 @@ export class FileWriter {
             });
         });
     }
+    getOutputPath(dataset) {
+        return join(this.outputDir, this.getFilename(dataset));
+    }
     getFilename(dataset) {
         const extension = this.getExtension();
         const baseName = filenamifyUrl(dataset.iri.toString(), {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.24",
+  "version": "0.6.26",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"
@@ -23,12 +23,10 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "0.6.8",
-    "@lde/dataset-registry-client": "0.6.15",
-    "@lde/sparql-importer": "0.2.8",
-    "@lde/sparql-server": "0.4.8",
+    "@lde/dataset": "0.6.9",
+    "@lde/dataset-registry-client": "0.6.16",
+    "@lde/sparql-importer": "0.2.9",
     "@rdfjs/types": "^2.0.1",
-    "c12": "^3.0.2",
     "fetch-sparql-endpoint": "^7.1.0",
     "filenamify-url": "^3.0.0",
     "n3": "^1.17.0",