@lde/pipeline 0.28.1 → 0.28.3

package/README.md

@@ -50,20 +50,24 @@ new Stage({
 
 `batchSize` (default: 10) controls how many variable bindings are passed to each executor call as a `VALUES` clause. It also sets the page size for the item selector's SPARQL requests, so that each paginated request fills exactly one executor batch.
 
-A `LIMIT` clause in the selector query overrides `batchSize` as the page size — use this when the SPARQL endpoint enforces a hard result limit:
+Some SPARQL endpoints enforce different result limits for SELECT and CONSTRUCT queries. Since the selector uses SELECT and the executor uses CONSTRUCT, a `LIMIT` clause in the selector query overrides `batchSize` as the page size. Use this when the endpoint caps SELECT results below your desired batch size:
 
 ```typescript
-// Endpoint caps results at 1000, but process in batches of 100.
+// Endpoint caps SELECT results at 500, but each CONSTRUCT can handle 1000 bindings.
 new Stage({
   name: 'per-class',
   itemSelector: new SparqlItemSelector({
-    query: 'SELECT DISTINCT ?class WHERE { ?s a ?class } LIMIT 1000',
+    query: 'SELECT DISTINCT ?class WHERE { ?s a ?class } LIMIT 500',
   }),
   executors: executor,
-  batchSize: 100,
+  batchSize: 1000, // Two SELECT pages fill one CONSTRUCT batch.
 });
 ```
 
+#### Concurrency
+
+`maxConcurrency` (default: 10) controls how many batches run in parallel. Within each batch, all executors run in parallel too, so the peak number of concurrent SPARQL queries is `maxConcurrency × executorCount`. For example, with `maxConcurrency: 5` and two executors per stage, up to 10 SPARQL queries may be in flight at the same time. Lower `maxConcurrency` if the endpoint can't handle the load.
+
 ### Item Selector
 
 Selects resources from the distribution and fans out executor calls per batch of results. Implements the `ItemSelector` interface:
@@ -103,6 +107,15 @@ const executor = new SparqlConstructExecutor({
 });
 ```
 
+When querying endpoints that return line-oriented formats like N-Triples (e.g. QLever), enable `lineBuffer` to work around an [N3.js chunk-splitting bug](https://github.com/rdfjs/N3.js/issues/578) that causes intermittent parse errors on large responses:
+
+```typescript
+const executor = new SparqlConstructExecutor({
+  query: 'CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }',
+  lineBuffer: true,
+});
+```
+
 `Executor` is an interface, so you can implement your own for logic that's hard to express in pure SPARQL — for example, cleaning up messy date notations or converting locale-specific dates to ISO 8601. The decorator pattern lets you wrap a SPARQL executor and post-process its quad stream in TypeScript:
 
 ```typescript

package/dist/sparql/executor.d.ts

@@ -1,6 +1,7 @@
 import { Dataset, Distribution } from '@lde/dataset';
 import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import type { NamedNode, Quad } from '@rdfjs/types';
+import { Transform } from 'node:stream';
 /**
  * An executor could not run because the dataset lacks a supported distribution.
  */
@@ -42,6 +43,17 @@ export interface SparqlConstructExecutorOptions {
      * Optional custom SparqlEndpointFetcher instance.
      */
     fetcher?: SparqlEndpointFetcher;
+    /**
+     * Buffer complete lines before passing them to the N3 parser.
+     *
+     * Works around an [N3.js bug](https://github.com/rdfjs/N3.js/issues/578)
+     * where language tags (e.g. `@nl-nl`) split across HTTP chunk boundaries
+     * cause parse errors. Enable this when querying endpoints that return
+     * line-oriented formats such as N-Triples (e.g. QLever).
+     *
+     * @default false
+     */
+    lineBuffer?: boolean;
 }
 /**
  * A streaming SPARQL CONSTRUCT executor.
@@ -75,6 +87,7 @@ export declare class SparqlConstructExecutor implements Executor {
     private readonly preParsed?;
     private readonly fetcher;
     private readonly retries;
+    private readonly lineBuffer;
     private readonly generator;
     constructor(options: SparqlConstructExecutorOptions);
     /**
@@ -86,6 +99,12 @@ export declare class SparqlConstructExecutor implements Executor {
      * @returns AsyncIterable<Quad> stream of results.
      */
     execute(dataset: Dataset, distribution: Distribution, options?: ExecuteOptions): Promise<AsyncIterable<Quad>>;
+    /**
+     * Fetch quads from the endpoint, optionally line-buffering the response
+     * stream before it reaches the N3 parser to work around
+     * {@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}.
+     */
+    private fetchQuads;
     /**
      * Create an executor from a query file.
      *
@@ -98,4 +117,29 @@ export declare class SparqlConstructExecutor implements Executor {
  * Read a SPARQL query from a file.
  */
 export declare function readQueryFile(filename: string): Promise<string>;
+/**
+ * Buffers incoming data until complete lines (`\n`-terminated) are available,
+ * then pushes them downstream as a single chunk.
+ *
+ * **Why this exists:** `fetch-sparql-endpoint` pipes the raw HTTP response
+ * stream directly into N3.js's `StreamParser`. N3.js has a bug
+ * ({@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}) where
+ * tokens that straddle chunk boundaries — most commonly language tags like
+ * `@nl-nl` — cause spurious `Unexpected "-nl"` parse errors. The error is
+ * non-deterministic and typically surfaces only on responses larger than
+ * ~12 MB, because that is when HTTP chunking starts splitting mid-token.
+ *
+ * By ensuring each chunk passed to the parser ends on a line boundary, we
+ * prevent any N-Triples token from being split. Memory overhead is minimal:
+ * at most one partial line is buffered at a time.
+ *
+ * This transform can be removed once N3.js#578 is fixed upstream.
+ *
+ * @see https://github.com/rdfjs/N3.js/issues/578
+ */
+export declare class LineBufferTransform extends Transform {
+    private remainder;
+    _transform(chunk: Buffer, _encoding: string, callback: () => void): void;
+    _flush(callback: () => void): void;
+}
 //# sourceMappingURL=executor.d.ts.map

package/dist/sparql/executor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAWpD;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAmBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAwC/B;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAoBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAwC/B;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CACjB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,IAAI;IAWb,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC"}

package/dist/sparql/executor.js

@@ -2,6 +2,8 @@ import { assertSafeIri } from '@lde/dataset';
 import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
+import { Transform } from 'node:stream';
+import { StreamParser } from 'n3';
 import { Parser } from '@traqula/parser-sparql-1-1';
 import { Generator } from '@traqula/generator-sparql-1-1';
 import isNetworkError from 'is-network-error';
@@ -49,10 +51,12 @@ export class SparqlConstructExecutor {
     preParsed;
     fetcher;
     retries;
+    lineBuffer;
     generator = new Generator();
     constructor(options) {
         this.rawQuery = options.query;
         this.retries = options.retries ?? 3;
+        this.lineBuffer = options.lineBuffer ?? false;
         if (!options.query.includes('#subjectFilter#')) {
             const parsed = new Parser().parse(options.query);
             if (parsed.type !== 'query' || parsed.subType !== 'construct') {
@@ -98,11 +102,25 @@ export class SparqlConstructExecutor {
         let query = this.generator.generate(ast);
         assertSafeIri(dataset.iri.toString());
         query = query.replaceAll('?dataset', `<${dataset.iri}>`);
-        return await pRetry(() => this.fetcher.fetchTriples(endpoint.toString(), query), {
+        return await pRetry(() => this.fetchQuads(endpoint.toString(), query), {
             retries: this.retries,
             shouldRetry: ({ error }) => isTransientError(error),
         });
     }
+    /**
+     * Fetch quads from the endpoint, optionally line-buffering the response
+     * stream before it reaches the N3 parser to work around
+     * {@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}.
+     */
+    async fetchQuads(endpoint, query) {
+        if (!this.lineBuffer) {
+            return this.fetcher.fetchTriples(endpoint, query);
+        }
+        const [contentType, , responseStream] = await this.fetcher.fetchRawStream(endpoint, query, SparqlEndpointFetcher.CONTENTTYPE_TURTLE);
+        return responseStream
+            .pipe(new LineBufferTransform())
+            .pipe(new StreamParser({ format: contentType }));
+    }
     /**
      * Create an executor from a query file.
      *
@@ -120,6 +138,44 @@ export class SparqlConstructExecutor {
 export async function readQueryFile(filename) {
     return (await readFile(resolve(filename))).toString();
 }
+/**
+ * Buffers incoming data until complete lines (`\n`-terminated) are available,
+ * then pushes them downstream as a single chunk.
+ *
+ * **Why this exists:** `fetch-sparql-endpoint` pipes the raw HTTP response
+ * stream directly into N3.js's `StreamParser`. N3.js has a bug
+ * ({@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}) where
+ * tokens that straddle chunk boundaries — most commonly language tags like
+ * `@nl-nl` — cause spurious `Unexpected "-nl"` parse errors. The error is
+ * non-deterministic and typically surfaces only on responses larger than
+ * ~12 MB, because that is when HTTP chunking starts splitting mid-token.
+ *
+ * By ensuring each chunk passed to the parser ends on a line boundary, we
+ * prevent any N-Triples token from being split. Memory overhead is minimal:
+ * at most one partial line is buffered at a time.
+ *
+ * This transform can be removed once N3.js#578 is fixed upstream.
+ *
+ * @see https://github.com/rdfjs/N3.js/issues/578
+ */
+export class LineBufferTransform extends Transform {
+    remainder = '';
+    _transform(chunk, _encoding, callback) {
+        const data = this.remainder + chunk.toString();
+        const lines = data.split('\n');
+        this.remainder = lines.pop() ?? '';
+        if (lines.length > 0) {
+            this.push(lines.join('\n') + '\n');
+        }
+        callback();
+    }
+    _flush(callback) {
+        if (this.remainder.length > 0) {
+            this.push(this.remainder);
+        }
+        callback();
+    }
+}
 const transientStatusPattern = /HTTP status (\d+)/;
 function isTransientError(error) {
     if (isNetworkError(error))

package/dist/sparql/index.d.ts

@@ -1,4 +1,4 @@
-export { SparqlConstructExecutor, NotSupported, readQueryFile, type ExecuteOptions, type Executor, type SparqlConstructExecutorOptions, type VariableBindings, } from './executor.js';
+export { SparqlConstructExecutor, LineBufferTransform, NotSupported, readQueryFile, type ExecuteOptions, type Executor, type SparqlConstructExecutorOptions, type VariableBindings, } from './executor.js';
 export { SparqlItemSelector, type SparqlItemSelectorOptions, } from './selector.js';
 export { injectValues } from './values.js';
 export { withDefaultGraph } from './graph.js';

package/dist/sparql/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC"}

package/dist/sparql/index.js

@@ -1,4 +1,4 @@
-export { SparqlConstructExecutor, NotSupported, readQueryFile, } from './executor.js';
+export { SparqlConstructExecutor, LineBufferTransform, NotSupported, readQueryFile, } from './executor.js';
 export { SparqlItemSelector, } from './selector.js';
 export { injectValues } from './values.js';
 export { withDefaultGraph } from './graph.js';

package/dist/stage.d.ts

@@ -20,7 +20,13 @@ export interface StageOptions {
      * @default 10
      */
     batchSize?: number;
-    /** Maximum concurrent in-flight executor batches. @default 10 */
+    /**
+     * Maximum concurrent in-flight batches. Within each batch, all executors
+     * run in parallel, so the peak number of concurrent SPARQL queries is
+     * `maxConcurrency × executorCount`.
+     *
+     * @default 10
+     */
     maxConcurrency?: number;
     /** Child stages that chain off this stage's output. */
     stages?: Stage[];

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAYjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAkDjB,eAAe;IA2I7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAYjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAkDjB,eAAe;IAiJ7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}

package/dist/stage.js

@@ -110,18 +110,19 @@ export class Stage {
                             break;
                     }
                     track((async () => {
-                        const batchQuads = [];
-                        for (const executor of this.executors) {
-                            const result = await executor.execute(dataset, distribution, {
-                                bindings,
-                            });
-                            if (!(result instanceof NotSupported)) {
-                                hasResults = true;
-                                for await (const quad of result) {
-                                    batchQuads.push(quad);
-                                }
+                        // Run all executors for this batch in parallel.
+                        const executorOutputs = await Promise.all(this.executors.map(async (executor) => {
+                            const result = await executor.execute(dataset, distribution, { bindings });
+                            if (result instanceof NotSupported)
+                                return [];
+                            hasResults = true;
+                            const quads = [];
+                            for await (const quad of result) {
+                                quads.push(quad);
                             }
-                        }
+                            return quads;
+                        }));
+                        const batchQuads = executorOutputs.flat();
                         if (this.validation &&
                             batchQuads.length > 0 &&
                             onInvalid !== 'write') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.28.1",
+  "version": "0.28.3",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"