@lde/pipeline 0.6.23 → 0.6.25

package/dist/asyncQueue.d.ts

@@ -0,0 +1,29 @@
+/**
+ * A bounded async channel: producers `push()` items, a single consumer
+ * iterates with `for await...of`. Backpressure is applied when the buffer
+ * reaches `capacity` — `push()` will block until the consumer pulls.
+ */
+export declare class AsyncQueue<T> implements AsyncIterable<T> {
+    private buffer;
+    private readonly capacity;
+    private closed;
+    private error;
+    /** Resolvers for a blocked consumer waiting for data or close/abort. */
+    private consumerResolve?;
+    private consumerReject?;
+    /** Resolvers for blocked producers waiting for buffer space. */
+    private producerResolvers;
+    constructor(capacity?: number);
+    /**
+     * Push an item into the queue. Blocks (returns a Promise) when the buffer
+     * is full. Throws if the queue has been closed or aborted.
+     */
+    push(item: T): Promise<void>;
+    /** Signal that no more items will be pushed. */
+    close(): void;
+    /** Signal an error. Unblocks all waiting producers and the consumer. */
+    abort(error: unknown): void;
+    [Symbol.asyncIterator](): AsyncIterator<T, undefined>;
+    private pull;
+}
+//# sourceMappingURL=asyncQueue.d.ts.map

package/dist/asyncQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"asyncQueue.d.ts","sourceRoot":"","sources":["../src/asyncQueue.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,qBAAa,UAAU,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,CAAC;IACpD,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAsB;IAEnC,wEAAwE;IACxE,OAAO,CAAC,eAAe,CAAC,CAAgD;IACxE,OAAO,CAAC,cAAc,CAAC,CAA4B;IAEnD,gEAAgE;IAChE,OAAO,CAAC,iBAAiB,CAGjB;gBAEI,QAAQ,SAAM;IAI1B;;;OAGG;IACG,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BlC,gDAAgD;IAChD,KAAK,IAAI,IAAI;IAab,wEAAwE;IACxE,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAoB3B,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC;IAMrD,OAAO,CAAC,IAAI;CA2Bb"}

package/dist/asyncQueue.js

@@ -0,0 +1,106 @@
+/**
+ * A bounded async channel: producers `push()` items, a single consumer
+ * iterates with `for await...of`. Backpressure is applied when the buffer
+ * reaches `capacity` — `push()` will block until the consumer pulls.
+ */
+export class AsyncQueue {
+    buffer = [];
+    capacity;
+    closed = false;
+    error = undefined;
+    /** Resolvers for a blocked consumer waiting for data or close/abort. */
+    consumerResolve;
+    consumerReject;
+    /** Resolvers for blocked producers waiting for buffer space. */
+    producerResolvers = [];
+    constructor(capacity = 128) {
+        this.capacity = capacity;
+    }
+    /**
+     * Push an item into the queue. Blocks (returns a Promise) when the buffer
+     * is full. Throws if the queue has been closed or aborted.
+     */
+    async push(item) {
+        if (this.error !== undefined) {
+            throw this.error;
+        }
+        if (this.closed) {
+            throw new Error('Cannot push to a closed queue');
+        }
+        // If a consumer is already waiting, deliver directly.
+        if (this.consumerResolve) {
+            const resolve = this.consumerResolve;
+            this.consumerResolve = undefined;
+            this.consumerReject = undefined;
+            resolve({ value: item, done: false });
+            return;
+        }
+        // Wait for space if buffer is at capacity.
+        if (this.buffer.length >= this.capacity) {
+            await new Promise((resolve, reject) => {
+                this.producerResolvers.push({ resolve, reject });
+            });
+        }
+        this.buffer.push(item);
+    }
+    /** Signal that no more items will be pushed. */
+    close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        // Wake a waiting consumer with done signal if buffer is empty.
+        if (this.buffer.length === 0 && this.consumerResolve) {
+            const resolve = this.consumerResolve;
+            this.consumerResolve = undefined;
+            this.consumerReject = undefined;
+            resolve({ value: undefined, done: true });
+        }
+    }
+    /** Signal an error. Unblocks all waiting producers and the consumer. */
+    abort(error) {
+        if (this.error !== undefined)
+            return; // first error wins
+        this.error = error;
+        this.closed = true;
+        // Reject all blocked producers.
+        for (const { reject } of this.producerResolvers) {
+            reject(error);
+        }
+        this.producerResolvers = [];
+        // Reject or resolve the consumer depending on buffered items.
+        if (this.consumerReject) {
+            const reject = this.consumerReject;
+            this.consumerResolve = undefined;
+            this.consumerReject = undefined;
+            reject(error);
+        }
+    }
+    [Symbol.asyncIterator]() {
+        return {
+            next: () => this.pull(),
+        };
+    }
+    pull() {
+        // Drain buffer first.
+        if (this.buffer.length > 0) {
+            const item = this.buffer.shift();
+            // Unblock one waiting producer.
+            if (this.producerResolvers.length > 0) {
+                this.producerResolvers.shift().resolve();
+            }
+            return Promise.resolve({ value: item, done: false });
+        }
+        // Buffer empty — check for error or closed.
+        if (this.error !== undefined) {
+            return Promise.reject(this.error);
+        }
+        if (this.closed) {
+            return Promise.resolve({ value: undefined, done: true });
+        }
+        // Wait for a producer to push or for close/abort.
+        return new Promise((resolve, reject) => {
+            this.consumerResolve = resolve;
+            this.consumerReject = reject;
+        });
+    }
+}

package/dist/index.d.ts

@@ -1,11 +1,10 @@
+export * from './asyncQueue.js';
 export * from './batch.js';
 export * from './pipeline.js';
+export * from './progressReporter.js';
 export * from './selector.js';
 export * from './stage.js';
-export * from './step.js';
-export * from './step/sparqlQuery.js';
-export * from './builder.js';
-export * from './config.js';
+export * from './stageOutputResolver.js';
 export * from './sparql/index.js';
 export * from './distribution/index.js';
 export * from './writer/index.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC"}

package/dist/index.js

@@ -1,11 +1,10 @@
+export * from './asyncQueue.js';
 export * from './batch.js';
 export * from './pipeline.js';
+export * from './progressReporter.js';
 export * from './selector.js';
 export * from './stage.js';
-export * from './step.js';
-export * from './step/sparqlQuery.js';
-export * from './builder.js';
-export * from './config.js';
+export * from './stageOutputResolver.js';
 export * from './sparql/index.js';
 export * from './distribution/index.js';
 export * from './writer/index.js';

package/dist/pipeline.d.ts

@@ -1,11 +1,28 @@
-import { Selector } from './selector.js';
-import { Step } from './step.js';
+import type { Selector } from './selector.js';
+import { Stage } from './stage.js';
+import type { Writer } from './writer/writer.js';
+import { type DistributionResolver } from './distribution/resolver.js';
+import type { StageOutputResolver } from './stageOutputResolver.js';
+import type { ProgressReporter } from './progressReporter.js';
+export interface PipelineOptions {
+    name: string;
+    selector: Selector;
+    stages: Stage[];
+    writer: Writer;
+    distributionResolver: DistributionResolver;
+    stageOutputResolver?: StageOutputResolver;
+    outputDir?: string;
+    outputFormat?: 'turtle' | 'n-triples' | 'n-quads';
+    reporter?: ProgressReporter;
+}
 export declare class Pipeline {
-    private readonly config;
-    constructor(config: {
-        selector: Selector;
-        steps: Step[];
-    });
+    private readonly options;
+    constructor(options: PipelineOptions);
     run(): Promise<void>;
+    private processDataset;
+    private runStage;
+    private runChain;
+    private runChainedStage;
+    private readFiles;
 }
 //# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAc,IAAI,EAAE,MAAM,WAAW,CAAC;AAI7C,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE;QAAE,QAAQ,EAAE,QAAQ,CAAC;QAAC,KAAK,EAAE,IAAI,EAAE,CAAA;KAAE;IAE7D,GAAG;CAiCjB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,oBAAoB,CAAC;IAC3C,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAClD,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,eAAe;IAe9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAcZ,cAAc;YA2Bd,QAAQ;YAgCR,QAAQ;YAoDR,eAAe;YAoCd,SAAS;CAUzB"}

package/dist/pipeline.js

@@ -1,42 +1,141 @@
-import { NotSupported } from './step.js';
-import { Readable } from 'node:stream';
+import { createReadStream } from 'node:fs';
+import { StreamParser } from 'n3';
+import { FileWriter } from './writer/fileWriter.js';
+import { NoDistributionAvailable, } from './distribution/resolver.js';
+import { NotSupported } from './sparql/executor.js';
 export class Pipeline {
-    config;
-    constructor(config) {
-        this.config = config;
+    options;
+    constructor(options) {
+        const hasSubStages = options.stages.some((stage) => stage.stages.length > 0);
+        if (hasSubStages && !options.stageOutputResolver) {
+            throw new Error('stageOutputResolver is required when any stage has sub-stages');
+        }
+        if (hasSubStages && !options.outputDir) {
+            throw new Error('outputDir is required when any stage has sub-stages');
+        }
+        this.options = options;
     }
     async run() {
-        const datasets = await this.config.selector.select();
+        const { selector, reporter, name } = this.options;
+        const start = Date.now();
+        reporter?.pipelineStart(name);
+        const datasets = await selector.select();
         for await (const dataset of datasets) {
-            const distribution = dataset.getSparqlDistribution() ?? undefined;
-            for (const step of this.config.steps) {
-                const result = await step.execute(dataset, distribution);
-                if (result instanceof NotSupported) {
-                    console.error(result);
+            await this.processDataset(dataset);
+        }
+        reporter?.pipelineComplete({ duration: Date.now() - start });
+    }
+    async processDataset(dataset) {
+        const { distributionResolver, reporter } = this.options;
+        const datasetIri = dataset.iri.toString();
+        reporter?.datasetStart(datasetIri);
+        const resolved = await distributionResolver.resolve(dataset);
+        if (resolved instanceof NoDistributionAvailable) {
+            reporter?.datasetSkipped(datasetIri, resolved.message);
+            return;
+        }
+        try {
+            for (const stage of this.options.stages) {
+                if (stage.stages.length > 0) {
+                    await this.runChain(dataset, resolved.distribution, stage);
                 }
-                else if (result instanceof Readable) {
-                    const promise = new Promise((resolve, reject) => {
-                        result.on('data', (data) => {
-                            // TODO: pipe to writers.
-                            console.log('Data:', data);
-                        });
-                        result.on('error', (error) => {
-                            console.error('rejecting');
-                            reject(error);
-                        });
-                        result.on('end', resolve);
-                    });
-                    await promise;
+                else {
+                    await this.runStage(dataset, resolved.distribution, stage);
                 }
             }
-            for (const step of this.config.steps) {
-                if (isFinishable(step)) {
-                    await step.finish();
+        }
+        catch {
+            // Stage error for this dataset; continue to next dataset.
+        }
+        reporter?.datasetComplete(datasetIri);
+    }
+    async runStage(dataset, distribution, stage) {
+        const { writer, reporter } = this.options;
+        reporter?.stageStart(stage.name);
+        const stageStart = Date.now();
+        let elementsProcessed = 0;
+        let quadsGenerated = 0;
+        const result = await stage.run(dataset, distribution, writer, {
+            onProgress: (elements, quads) => {
+                elementsProcessed = elements;
+                quadsGenerated = quads;
+                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+            },
+        });
+        if (result instanceof NotSupported) {
+            reporter?.stageSkipped(stage.name, result.message);
+        }
+        else {
+            reporter?.stageComplete(stage.name, {
+                elementsProcessed,
+                quadsGenerated,
+                duration: Date.now() - stageStart,
+            });
+        }
+    }
+    async runChain(dataset, distribution, stage) {
+        const { writer, stageOutputResolver, outputDir, outputFormat } = this.options;
+        const outputFiles = [];
+        try {
+            // 1. Run parent stage → FileWriter.
+            const parentWriter = new FileWriter({
+                outputDir: `${outputDir}/${stage.name}`,
+                format: outputFormat,
+            });
+            await this.runChainedStage(dataset, distribution, stage, parentWriter);
+            outputFiles.push(parentWriter.getOutputPath(dataset));
+            // 2. Chain through children.
+            let currentDistribution = await stageOutputResolver.resolve(parentWriter.getOutputPath(dataset));
+            for (let i = 0; i < stage.stages.length; i++) {
+                const child = stage.stages[i];
+                const childWriter = new FileWriter({
+                    outputDir: `${outputDir}/${child.name}`,
+                    format: outputFormat,
+                });
+                await this.runChainedStage(dataset, currentDistribution, child, childWriter);
+                outputFiles.push(childWriter.getOutputPath(dataset));
+                if (i < stage.stages.length - 1) {
+                    currentDistribution = await stageOutputResolver.resolve(childWriter.getOutputPath(dataset));
                 }
             }
+            // 3. Concatenate all output files → user writer.
+            await writer.write(dataset, this.readFiles(outputFiles));
+        }
+        finally {
+            await stageOutputResolver.cleanup();
+        }
+    }
+    async runChainedStage(dataset, distribution, stage, stageWriter) {
+        const { reporter } = this.options;
+        reporter?.stageStart(stage.name);
+        const stageStart = Date.now();
+        let elementsProcessed = 0;
+        let quadsGenerated = 0;
+        const result = await stage.run(dataset, distribution, stageWriter, {
+            onProgress: (elements, quads) => {
+                elementsProcessed = elements;
+                quadsGenerated = quads;
+                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+            },
+        });
+        if (result instanceof NotSupported) {
+            reporter?.stageSkipped(stage.name, result.message);
+            throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
+        }
+        reporter?.stageComplete(stage.name, {
+            elementsProcessed,
+            quadsGenerated,
+            duration: Date.now() - stageStart,
+        });
+    }
+    async *readFiles(paths) {
+        for (const path of paths) {
+            const stream = createReadStream(path);
+            const parser = new StreamParser();
+            stream.pipe(parser);
+            for await (const quad of parser) {
+                yield quad;
+            }
         }
     }
 }
-const isFinishable = (step) => {
-    return typeof step.finish === 'function';
-};

package/dist/progressReporter.d.ts

@@ -0,0 +1,21 @@
+export interface ProgressReporter {
+    pipelineStart(name: string): void;
+    datasetStart(dataset: string): void;
+    stageStart(stage: string): void;
+    stageProgress(update: {
+        elementsProcessed: number;
+        quadsGenerated: number;
+    }): void;
+    stageComplete(stage: string, result: {
+        elementsProcessed: number;
+        quadsGenerated: number;
+        duration: number;
+    }): void;
+    stageSkipped(stage: string, reason: string): void;
+    datasetComplete(dataset: string): void;
+    datasetSkipped(dataset: string, reason: string): void;
+    pipelineComplete(result: {
+        duration: number;
+    }): void;
+}
+//# sourceMappingURL=progressReporter.d.ts.map

package/dist/progressReporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,aAAa,CAAC,MAAM,EAAE;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC;IACT,aAAa,CACX,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAClD,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtD,gBAAgB,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACtD"}

package/dist/progressReporter.js

@@ -0,0 +1 @@
+export {};

package/dist/sparql/executor.d.ts

@@ -2,8 +2,13 @@ import { Dataset, Distribution } from '@lde/dataset';
 import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import type { NamedNode, Quad, Stream } from '@rdfjs/types';
 import type { Readable } from 'node:stream';
-import { NotSupported } from '../step.js';
-export { NotSupported } from '../step.js';
+/**
+ * An executor could not run because the dataset lacks a supported distribution.
+ */
+export declare class NotSupported {
+    readonly message: string;
+    constructor(message: string);
+}
 /** A single row of variable bindings (variable name → NamedNode). */
 export type VariableBindings = Record<string, NamedNode>;
 export interface ExecuteOptions {

package/dist/sparql/executor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAI5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAK1C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAcnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,UAAU,CAAC;IAoBtB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,OAAO,EAAE,OAAO,GACf,MAAM,CAWR;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAO5C;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAcnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,UAAU,CAAC;IAoBtB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,OAAO,EAAE,OAAO,GACf,MAAM,CAWR;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}

package/dist/sparql/executor.js

@@ -4,8 +4,15 @@ import { resolve } from 'node:path';
 import { Generator, Parser } from 'sparqljs';
 import { withDefaultGraph } from './graph.js';
 import { injectValues } from './values.js';
-// Re-export for convenience
-export { NotSupported } from '../step.js';
+/**
+ * An executor could not run because the dataset lacks a supported distribution.
+ */
+export class NotSupported {
+    message;
+    constructor(message) {
+        this.message = message;
+    }
+}
 /**
  * A streaming SPARQL CONSTRUCT executor that parses the query once (in the
  * constructor) and operates on the AST for graph and VALUES injection.

package/dist/stage.d.ts

@@ -8,15 +8,24 @@ export interface StageOptions {
     selector?: StageSelector;
     /** Maximum number of bindings per executor call. @default 10 */
     batchSize?: number;
+    /** Maximum concurrent in-flight executor batches. @default 10 */
+    maxConcurrency?: number;
+    /** Child stages that chain off this stage's output. */
+    stages?: Stage[];
+}
+export interface RunOptions {
+    onProgress?: (elementsProcessed: number, quadsGenerated: number) => void;
 }
 export declare class Stage {
     readonly name: string;
+    readonly stages: readonly Stage[];
     private readonly executors;
     private readonly selector?;
     private readonly batchSize;
+    private readonly maxConcurrency;
     constructor(options: StageOptions);
-    run(dataset: Dataset, distribution: Distribution, writer: Writer): Promise<NotSupported | void>;
-    private executeWithSelector;
+    run(dataset: Dataset, distribution: Distribution, writer: Writer, options?: RunOptions): Promise<NotSupported | void>;
+    private runWithSelector;
     private executeAll;
 }
 /** Stage-level selector that yields variable bindings for use in executor queries. Pagination is an implementation detail. */

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,YAAY;IAS3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAYjB,mBAAmB;YAuBnB,UAAU;CAkBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAW3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAajB,eAAe;YA8Gf,UAAU;CAqBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}

package/dist/stage.js

@@ -1,48 +1,130 @@
 import { NotSupported } from './sparql/executor.js';
 import { batch } from './batch.js';
+import { AsyncQueue } from './asyncQueue.js';
 export class Stage {
     name;
+    stages;
     executors;
     selector;
     batchSize;
+    maxConcurrency;
     constructor(options) {
         this.name = options.name;
+        this.stages = options.stages ?? [];
         this.executors = Array.isArray(options.executors)
             ? options.executors
             : [options.executors];
         this.selector = options.selector;
         this.batchSize = options.batchSize ?? 10;
+        this.maxConcurrency = options.maxConcurrency ?? 10;
     }
-    async run(dataset, distribution, writer) {
-        const streams = this.selector
-            ? await this.executeWithSelector(dataset, distribution)
-            : await this.executeAll(dataset, distribution);
+    async run(dataset, distribution, writer, options) {
+        if (this.selector) {
+            return this.runWithSelector(dataset, distribution, writer, options);
+        }
+        const streams = await this.executeAll(dataset, distribution);
         if (streams instanceof NotSupported) {
             return streams;
         }
         await writer.write(dataset, mergeStreams(streams));
     }
-    async executeWithSelector(dataset, distribution) {
-        const streams = [];
-        for await (const bindings of batch(this.selector, this.batchSize)) {
-            for (const executor of this.executors) {
-                const result = await executor.execute(dataset, distribution, {
-                    bindings,
+    async runWithSelector(dataset, distribution, writer, options) {
+        // Peek the first batch to detect an empty selector before starting the
+        // writer (important because e.g. SparqlUpdateWriter does CLEAR GRAPH).
+        const batches = batch(this.selector, this.batchSize);
+        const iter = batches[Symbol.asyncIterator]();
+        const first = await iter.next();
+        if (first.done) {
+            return new NotSupported('All executors returned NotSupported');
+        }
+        // Reconstruct a full iterable including the peeked first batch.
+        const allBatches = (async function* () {
+            yield first.value;
+            // Continue yielding remaining batches from the same iterator.
+            for (;;) {
+                const next = await iter.next();
+                if (next.done)
+                    break;
+                yield next.value;
+            }
+        })();
+        const queue = new AsyncQueue();
+        let elementsProcessed = 0;
+        let quadsGenerated = 0;
+        let hasResults = false;
+        const dispatch = async () => {
+            const inFlight = new Set();
+            let firstError;
+            const track = (promise) => {
+                const p = promise.then(() => {
+                    inFlight.delete(p);
+                }, (err) => {
+                    inFlight.delete(p);
+                    firstError ??= err;
                 });
-                if (!(result instanceof NotSupported)) {
-                    streams.push(result);
+                inFlight.add(p);
+            };
+            try {
+                for await (const bindings of allBatches) {
+                    if (firstError)
+                        break;
+                    for (const executor of this.executors) {
+                        if (firstError)
+                            break;
+                        // Respect maxConcurrency: wait for a slot to open.
+                        if (inFlight.size >= this.maxConcurrency) {
+                            await Promise.race(inFlight);
+                            if (firstError)
+                                break;
+                        }
+                        track((async () => {
+                            const result = await executor.execute(dataset, distribution, {
+                                bindings,
+                            });
+                            if (!(result instanceof NotSupported)) {
+                                hasResults = true;
+                                for await (const quad of result) {
+                                    await queue.push(quad);
+                                    quadsGenerated++;
+                                }
+                            }
+                            elementsProcessed += bindings.length;
+                            options?.onProgress?.(elementsProcessed, quadsGenerated);
+                        })());
+                    }
                 }
             }
-        }
-        if (streams.length === 0) {
+            catch (err) {
+                firstError ??= err;
+            }
+            // Wait for all remaining in-flight tasks to settle.
+            await Promise.all(inFlight);
+            if (firstError) {
+                queue.abort(firstError);
+            }
+            else {
+                queue.close();
+            }
+        };
+        const dispatchPromise = dispatch();
+        const writePromise = (async () => {
+            try {
+                await writer.write(dataset, queue);
+            }
+            catch (err) {
+                queue.abort(err);
+                throw err;
+            }
+        })();
+        await Promise.all([dispatchPromise, writePromise]);
+        if (!hasResults) {
             return new NotSupported('All executors returned NotSupported');
         }
-        return streams;
     }
     async executeAll(dataset, distribution) {
+        const results = await Promise.all(this.executors.map((executor) => executor.execute(dataset, distribution)));
         const streams = [];
-        for (const executor of this.executors) {
-            const result = await executor.execute(dataset, distribution);
+        for (const result of results) {
             if (!(result instanceof NotSupported)) {
                 streams.push(result);
             }