@lde/pipeline 0.6.22 → 0.6.24

package/dist/asyncQueue.d.ts

@@ -0,0 +1,29 @@
+/**
+ * A bounded async channel: producers `push()` items, a single consumer
+ * iterates with `for await...of`. Backpressure is applied when the buffer
+ * reaches `capacity` — `push()` will block until the consumer pulls.
+ */
+export declare class AsyncQueue<T> implements AsyncIterable<T> {
+    private buffer;
+    private readonly capacity;
+    private closed;
+    private error;
+    /** Resolvers for a blocked consumer waiting for data or close/abort. */
+    private consumerResolve?;
+    private consumerReject?;
+    /** Resolvers for blocked producers waiting for buffer space. */
+    private producerResolvers;
+    constructor(capacity?: number);
+    /**
+     * Push an item into the queue. Blocks (returns a Promise) when the buffer
+     * is full. Throws if the queue has been closed or aborted.
+     */
+    push(item: T): Promise<void>;
+    /** Signal that no more items will be pushed. */
+    close(): void;
+    /** Signal an error. Unblocks all waiting producers and the consumer. */
+    abort(error: unknown): void;
+    [Symbol.asyncIterator](): AsyncIterator<T, undefined>;
+    private pull;
+}
+//# sourceMappingURL=asyncQueue.d.ts.map

package/dist/asyncQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"asyncQueue.d.ts","sourceRoot":"","sources":["../src/asyncQueue.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,qBAAa,UAAU,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,CAAC;IACpD,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAsB;IAEnC,wEAAwE;IACxE,OAAO,CAAC,eAAe,CAAC,CAAgD;IACxE,OAAO,CAAC,cAAc,CAAC,CAA4B;IAEnD,gEAAgE;IAChE,OAAO,CAAC,iBAAiB,CAGjB;gBAEI,QAAQ,SAAM;IAI1B;;;OAGG;IACG,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BlC,gDAAgD;IAChD,KAAK,IAAI,IAAI;IAab,wEAAwE;IACxE,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAoB3B,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,EAAE,SAAS,CAAC;IAMrD,OAAO,CAAC,IAAI;CA2Bb"}

package/dist/asyncQueue.js

@@ -0,0 +1,106 @@
+/**
+ * A bounded async channel: producers `push()` items, a single consumer
+ * iterates with `for await...of`. Backpressure is applied when the buffer
+ * reaches `capacity` — `push()` will block until the consumer pulls.
+ */
+export class AsyncQueue {
+    buffer = [];
+    capacity;
+    closed = false;
+    error = undefined;
+    /** Resolvers for a blocked consumer waiting for data or close/abort. */
+    consumerResolve;
+    consumerReject;
+    /** Resolvers for blocked producers waiting for buffer space. */
+    producerResolvers = [];
+    constructor(capacity = 128) {
+        this.capacity = capacity;
+    }
+    /**
+     * Push an item into the queue. Blocks (returns a Promise) when the buffer
+     * is full. Throws if the queue has been closed or aborted.
+     */
+    async push(item) {
+        if (this.error !== undefined) {
+            throw this.error;
+        }
+        if (this.closed) {
+            throw new Error('Cannot push to a closed queue');
+        }
+        // If a consumer is already waiting, deliver directly.
+        if (this.consumerResolve) {
+            const resolve = this.consumerResolve;
+            this.consumerResolve = undefined;
+            this.consumerReject = undefined;
+            resolve({ value: item, done: false });
+            return;
+        }
+        // Wait for space if buffer is at capacity.
+        if (this.buffer.length >= this.capacity) {
+            await new Promise((resolve, reject) => {
+                this.producerResolvers.push({ resolve, reject });
+            });
+        }
+        this.buffer.push(item);
+    }
+    /** Signal that no more items will be pushed. */
+    close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        // Wake a waiting consumer with done signal if buffer is empty.
+        if (this.buffer.length === 0 && this.consumerResolve) {
+            const resolve = this.consumerResolve;
+            this.consumerResolve = undefined;
+            this.consumerReject = undefined;
+            resolve({ value: undefined, done: true });
+        }
+    }
+    /** Signal an error. Unblocks all waiting producers and the consumer. */
+    abort(error) {
+        if (this.error !== undefined)
+            return; // first error wins
+        this.error = error;
+        this.closed = true;
+        // Reject all blocked producers.
+        for (const { reject } of this.producerResolvers) {
+            reject(error);
+        }
+        this.producerResolvers = [];
+        // Reject or resolve the consumer depending on buffered items.
+        if (this.consumerReject) {
+            const reject = this.consumerReject;
+            this.consumerResolve = undefined;
+            this.consumerReject = undefined;
+            reject(error);
+        }
+    }
+    [Symbol.asyncIterator]() {
+        return {
+            next: () => this.pull(),
+        };
+    }
+    pull() {
+        // Drain buffer first.
+        if (this.buffer.length > 0) {
+            const item = this.buffer.shift();
+            // Unblock one waiting producer.
+            if (this.producerResolvers.length > 0) {
+                this.producerResolvers.shift().resolve();
+            }
+            return Promise.resolve({ value: item, done: false });
+        }
+        // Buffer empty — check for error or closed.
+        if (this.error !== undefined) {
+            return Promise.reject(this.error);
+        }
+        if (this.closed) {
+            return Promise.resolve({ value: undefined, done: true });
+        }
+        // Wait for a producer to push or for close/abort.
+        return new Promise((resolve, reject) => {
+            this.consumerResolve = resolve;
+            this.consumerReject = reject;
+        });
+    }
+}

package/dist/builder.d.ts

@@ -30,6 +30,11 @@ export interface WriterConfig {
     type: 'file' | 'sparql';
     outputDir?: string;
     endpoint?: URL;
+    /**
+     * Value for the Authorization header sent with SPARQL UPDATE requests, e.g.
+     * `"Basic dXNlcjpwYXNz"`, `"Bearer my-token"`, or `"GDB eyJ…"`.
+     */
+    auth?: string;
 }
 /**
  * Complete pipeline configuration.
@@ -110,5 +115,6 @@ export declare function fileWriter(options: {
  */
 export declare function sparqlWriter(options: {
     endpoint: URL;
+    auth?: string;
 }): WriterConfig;
 //# sourceMappingURL=builder.d.ts.map

package/dist/builder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../src/builder.ts"],"names":[],"mappings":"AACA,OAAO,EACL,QAAQ,EACR,sBAAsB,EACtB,gBAAgB,EACjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAW;IAC5B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,YAAY,CAAC,CAAe;IAEpC;;OAEG;IACH,MAAM,CAAC,MAAM,IAAI,eAAe;IAIhC;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAKtC;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAKtC;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAKzB;;OAEG;IACH,QAAQ,CAAC,GAAG,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI;IAKhC;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAKrC;;;OAGG;IACH,KAAK,IAAI,cAAc;CAYxB;AAID;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,GAAG,gBAAgB,CAMjE;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,GAAG,WAAW,EAAE,GAAG,EAAE,GAAG,sBAAsB,CAKpE;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,YAAY,CAKvE;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE;IAAE,QAAQ,EAAE,GAAG,CAAA;CAAE,GAAG,YAAY,CAKrE"}
+{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../src/builder.ts"],"names":[],"mappings":"AACA,OAAO,EACL,QAAQ,EACR,sBAAsB,EACtB,gBAAgB,EACjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAW;IAC5B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,YAAY,CAAC,CAAe;IAEpC;;OAEG;IACH,MAAM,CAAC,MAAM,IAAI,eAAe;IAIhC;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAKtC;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAKtC;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAKzB;;OAEG;IACH,QAAQ,CAAC,GAAG,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI;IAKhC;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAKrC;;;OAGG;IACH,KAAK,IAAI,cAAc;CAYxB;AAID;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,GAAG,gBAAgB,CAMjE;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,GAAG,WAAW,EAAE,GAAG,EAAE,GAAG,sBAAsB,CAKpE;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,YAAY,CAKvE;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE;IACpC,QAAQ,EAAE,GAAG,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,YAAY,CAMf"}

package/dist/builder.js

@@ -111,5 +111,6 @@ export function sparqlWriter(options) {
     return {
         type: 'sparql',
         endpoint: options.endpoint,
+        auth: options.auth,
     };
 }

package/dist/config.d.ts

@@ -17,6 +17,7 @@ export interface RawPipelineConfig {
         type: 'file' | 'sparql';
         outputDir?: string;
         endpoint?: string;
+        auth?: string;
     }>;
 }
 /**

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,EACL,cAAc,EACd,YAAY,EAIb,MAAM,cAAc,CAAC;AAKtB;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE;QACT,IAAI,EAAE,UAAU,GAAG,QAAQ,CAAC;QAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,cAAc,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,iBAAiB,GAAG,iBAAiB,CAEzE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,cAAc,CAAC,CAWzB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,iBAAiB,GAAG,cAAc,CAOtE"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,EACL,cAAc,EACd,YAAY,EAIb,MAAM,cAAc,CAAC;AAKtB;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE;QACT,IAAI,EAAE,UAAU,GAAG,QAAQ,CAAC;QAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,cAAc,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,iBAAiB,GAAG,iBAAiB,CAEzE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,cAAc,CAAC,CAWzB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,iBAAiB,GAAG,cAAc,CAOtE"}

package/dist/config.js

@@ -102,7 +102,11 @@ function normalizeWriters(raw) {
                 if (!writer.endpoint) {
                     throw new Error('SPARQL writer requires endpoint');
                 }
-                return { type: 'sparql', endpoint: new URL(writer.endpoint) };
+                return {
+                    type: 'sparql',
+                    endpoint: new URL(writer.endpoint),
+                    auth: writer.auth,
+                };
             default:
                 throw new Error(`Unknown writer type: ${writer.type}`);
         }

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+export * from './asyncQueue.js';
 export * from './batch.js';
 export * from './pipeline.js';
 export * from './selector.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC"}

package/dist/index.js

@@ -1,3 +1,4 @@
+export * from './asyncQueue.js';
 export * from './batch.js';
 export * from './pipeline.js';
 export * from './selector.js';

package/dist/stage.d.ts

@@ -8,15 +8,21 @@ export interface StageOptions {
     selector?: StageSelector;
     /** Maximum number of bindings per executor call. @default 10 */
     batchSize?: number;
+    /** Maximum concurrent in-flight executor batches. @default 10 */
+    maxConcurrency?: number;
+}
+export interface RunOptions {
+    onProgress?: (elementsProcessed: number, quadsGenerated: number) => void;
 }
 export declare class Stage {
     readonly name: string;
     private readonly executors;
     private readonly selector?;
     private readonly batchSize;
+    private readonly maxConcurrency;
     constructor(options: StageOptions);
-    run(dataset: Dataset, distribution: Distribution, writer: Writer): Promise<NotSupported | void>;
-    private executeWithSelector;
+    run(dataset: Dataset, distribution: Distribution, writer: Writer, options?: RunOptions): Promise<NotSupported | void>;
+    private runWithSelector;
     private executeAll;
 }
 /** Stage-level selector that yields variable bindings for use in executor queries. Pagination is an implementation detail. */

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,YAAY;IAS3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAYjB,mBAAmB;YAuBnB,UAAU;CAkBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAU3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAajB,eAAe;YA8Gf,UAAU;CAqBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}

package/dist/stage.js

@@ -1,10 +1,12 @@
 import { NotSupported } from './sparql/executor.js';
 import { batch } from './batch.js';
+import { AsyncQueue } from './asyncQueue.js';
 export class Stage {
     name;
     executors;
     selector;
     batchSize;
+    maxConcurrency;
     constructor(options) {
         this.name = options.name;
         this.executors = Array.isArray(options.executors)
@@ -12,37 +14,115 @@ export class Stage {
             : [options.executors];
         this.selector = options.selector;
         this.batchSize = options.batchSize ?? 10;
+        this.maxConcurrency = options.maxConcurrency ?? 10;
     }
-    async run(dataset, distribution, writer) {
-        const streams = this.selector
-            ? await this.executeWithSelector(dataset, distribution)
-            : await this.executeAll(dataset, distribution);
+    async run(dataset, distribution, writer, options) {
+        if (this.selector) {
+            return this.runWithSelector(dataset, distribution, writer, options);
+        }
+        const streams = await this.executeAll(dataset, distribution);
         if (streams instanceof NotSupported) {
             return streams;
         }
         await writer.write(dataset, mergeStreams(streams));
     }
-    async executeWithSelector(dataset, distribution) {
-        const streams = [];
-        for await (const bindings of batch(this.selector, this.batchSize)) {
-            for (const executor of this.executors) {
-                const result = await executor.execute(dataset, distribution, {
-                    bindings,
+    async runWithSelector(dataset, distribution, writer, options) {
+        // Peek the first batch to detect an empty selector before starting the
+        // writer (important because e.g. SparqlUpdateWriter does CLEAR GRAPH).
+        const batches = batch(this.selector, this.batchSize);
+        const iter = batches[Symbol.asyncIterator]();
+        const first = await iter.next();
+        if (first.done) {
+            return new NotSupported('All executors returned NotSupported');
+        }
+        // Reconstruct a full iterable including the peeked first batch.
+        const allBatches = (async function* () {
+            yield first.value;
+            // Continue yielding remaining batches from the same iterator.
+            for (;;) {
+                const next = await iter.next();
+                if (next.done)
+                    break;
+                yield next.value;
+            }
+        })();
+        const queue = new AsyncQueue();
+        let elementsProcessed = 0;
+        let quadsGenerated = 0;
+        let hasResults = false;
+        const dispatch = async () => {
+            const inFlight = new Set();
+            let firstError;
+            const track = (promise) => {
+                const p = promise.then(() => {
+                    inFlight.delete(p);
+                }, (err) => {
+                    inFlight.delete(p);
+                    firstError ??= err;
                 });
-                if (!(result instanceof NotSupported)) {
-                    streams.push(result);
+                inFlight.add(p);
+            };
+            try {
+                for await (const bindings of allBatches) {
+                    if (firstError)
+                        break;
+                    for (const executor of this.executors) {
+                        if (firstError)
+                            break;
+                        // Respect maxConcurrency: wait for a slot to open.
+                        if (inFlight.size >= this.maxConcurrency) {
+                            await Promise.race(inFlight);
+                            if (firstError)
+                                break;
+                        }
+                        track((async () => {
+                            const result = await executor.execute(dataset, distribution, {
+                                bindings,
+                            });
+                            if (!(result instanceof NotSupported)) {
+                                hasResults = true;
+                                for await (const quad of result) {
+                                    await queue.push(quad);
+                                    quadsGenerated++;
+                                }
+                            }
+                            elementsProcessed += bindings.length;
+                            options?.onProgress?.(elementsProcessed, quadsGenerated);
+                        })());
+                    }
                 }
             }
-        }
-        if (streams.length === 0) {
+            catch (err) {
+                firstError ??= err;
+            }
+            // Wait for all remaining in-flight tasks to settle.
+            await Promise.all(inFlight);
+            if (firstError) {
+                queue.abort(firstError);
+            }
+            else {
+                queue.close();
+            }
+        };
+        const dispatchPromise = dispatch();
+        const writePromise = (async () => {
+            try {
+                await writer.write(dataset, queue);
+            }
+            catch (err) {
+                queue.abort(err);
+                throw err;
+            }
+        })();
+        await Promise.all([dispatchPromise, writePromise]);
+        if (!hasResults) {
             return new NotSupported('All executors returned NotSupported');
         }
-        return streams;
     }
     async executeAll(dataset, distribution) {
+        const results = await Promise.all(this.executors.map((executor) => executor.execute(dataset, distribution)));
         const streams = [];
-        for (const executor of this.executors) {
-            const result = await executor.execute(dataset, distribution);
+        for (const result of results) {
             if (!(result instanceof NotSupported)) {
                 streams.push(result);
             }

package/dist/writer/sparqlUpdateWriter.d.ts

@@ -6,6 +6,11 @@ export interface SparqlWriterOptions {
      * The SPARQL UPDATE endpoint URL.
      */
     endpoint: URL;
+    /**
+     * Value for the Authorization header, e.g.
+     * `"Basic dXNlcjpwYXNz"`, `"Bearer my-token"`, or `"GDB eyJ…"`.
+     */
+    auth?: string;
     /**
      * Optional fetch implementation for making HTTP requests.
      * @default globalThis.fetch
@@ -26,6 +31,7 @@ export interface SparqlWriterOptions {
  */
 export declare class SparqlUpdateWriter implements Writer {
     private readonly endpoint;
+    private readonly auth?;
     private readonly fetch;
     private readonly batchSize;
     constructor(options: SparqlWriterOptions);

package/dist/writer/sparqlUpdateWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sparqlUpdateWriter.d.ts","sourceRoot":"","sources":["../../src/writer/sparqlUpdateWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,mBAAmB;IAMlC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAS1D,UAAU;YAIV,WAAW;YAOX,aAAa;CAgB5B"}
+{"version":3,"file":"sparqlUpdateWriter.d.ts","sourceRoot":"","sources":["../../src/writer/sparqlUpdateWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,mBAAmB;IAOlC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAS1D,UAAU;YAIV,WAAW;YAOX,aAAa;CAqB5B"}

package/dist/writer/sparqlUpdateWriter.js

@@ -8,10 +8,12 @@ import { serializeQuads } from './serialize.js';
  */
 export class SparqlUpdateWriter {
     endpoint;
+    auth;
     fetch;
     batchSize;
     constructor(options) {
         this.endpoint = options.endpoint;
+        this.auth = options.auth;
         this.fetch = options.fetch ?? globalThis.fetch;
         this.batchSize = options.batchSize ?? 10000;
     }
@@ -30,11 +32,15 @@ export class SparqlUpdateWriter {
         await this.executeUpdate(`INSERT DATA { GRAPH <${graphUri}> { ${turtleData} } }`);
     }
     async executeUpdate(query) {
+        const headers = {
+            'Content-Type': 'application/sparql-update',
+        };
+        if (this.auth) {
+            headers['Authorization'] = this.auth;
+        }
         const response = await this.fetch(this.endpoint.toString(), {
             method: 'POST',
-            headers: {
-                'Content-Type': 'application/sparql-update',
-            },
+            headers,
             body: query,
         });
         if (!response.ok) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.22",
+  "version": "0.6.24",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"
@@ -24,12 +24,12 @@
   ],
   "dependencies": {
     "@lde/dataset": "0.6.8",
-    "@lde/dataset-registry-client": "0.6.14",
+    "@lde/dataset-registry-client": "0.6.15",
     "@lde/sparql-importer": "0.2.8",
     "@lde/sparql-server": "0.4.8",
     "@rdfjs/types": "^2.0.1",
     "c12": "^3.0.2",
-    "fetch-sparql-endpoint": "^6.0.0",
+    "fetch-sparql-endpoint": "^7.1.0",
     "filenamify-url": "^3.0.0",
     "n3": "^1.17.0",
     "sparqljs": "^3.7.3",