@lde/pipeline 0.28.0 → 0.28.2

package/README.md

@@ -30,17 +30,51 @@ const selector = new RegistrySelector({
 const selector = new ManualDatasetSelection([dataset]);
 ```
 
+### Stage
+
+A stage groups an item selector, one or more executors, and configuration:
+
+```typescript
+new Stage({
+  name: 'per-class',
+  itemSelector: new SparqlItemSelector({
+    query: 'SELECT DISTINCT ?class WHERE { ?s a ?class }',
+  }),
+  executors: executor,
+  batchSize: 100,
+  maxConcurrency: 5,
+});
+```
+
+#### Batch size
+
+`batchSize` (default: 10) controls how many variable bindings are passed to each executor call as a `VALUES` clause. It also sets the page size for the item selector's SPARQL requests, so that each paginated request fills exactly one executor batch.
+
+A `LIMIT` clause in the selector query overrides `batchSize` as the page size — use this when the SPARQL endpoint enforces a hard result limit:
+
+```typescript
+// Endpoint caps results at 1000, but process in batches of 100.
+new Stage({
+  name: 'per-class',
+  itemSelector: new SparqlItemSelector({
+    query: 'SELECT DISTINCT ?class WHERE { ?s a ?class } LIMIT 1000',
+  }),
+  executors: executor,
+  batchSize: 100,
+});
+```
+
 ### Item Selector
 
 Selects resources from the distribution and fans out executor calls per batch of results. Implements the `ItemSelector` interface:
 
 ```typescript
 interface ItemSelector {
-  select(distribution: Distribution): AsyncIterable<VariableBindings>;
+  select(distribution: Distribution, batchSize?: number): AsyncIterable<VariableBindings>;
 }
 ```
 
-The distribution is received at run time, so selectors don't need the endpoint URL at construction time. Use `SparqlItemSelector` for SPARQL-based selection with automatic pagination:
+The distribution is received at run time, so selectors don't need the endpoint URL at construction time. The `batchSize` parameter is set by the stage. Use `SparqlItemSelector` for SPARQL-based selection with automatic pagination:
 
 ```typescript
 new SparqlItemSelector({
@@ -52,9 +86,9 @@ For dynamic queries that depend on the distribution, implement `ItemSelector` di
 
 ```typescript
 const itemSelector: ItemSelector = {
-  select: (distribution) => {
+  select: (distribution, batchSize) => {
     const query = buildQuery(distribution);
-    return new SparqlItemSelector({ query }).select(distribution);
+    return new SparqlItemSelector({ query }).select(distribution, batchSize);
   },
 };
 ```
@@ -69,6 +103,15 @@ const executor = new SparqlConstructExecutor({
 });
 ```
 
+When querying endpoints that return line-oriented formats like N-Triples (e.g. QLever), enable `lineBuffer` to work around an [N3.js chunk-splitting bug](https://github.com/rdfjs/N3.js/issues/578) that causes intermittent parse errors on large responses:
+
+```typescript
+const executor = new SparqlConstructExecutor({
+  query: 'CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }',
+  lineBuffer: true,
+});
+```
+
 `Executor` is an interface, so you can implement your own for logic that's hard to express in pure SPARQL — for example, cleaning up messy date notations or converting locale-specific dates to ISO 8601. The decorator pattern lets you wrap a SPARQL executor and post-process its quad stream in TypeScript:
 
 ```typescript

package/dist/sparql/executor.d.ts

@@ -1,6 +1,7 @@
 import { Dataset, Distribution } from '@lde/dataset';
 import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import type { NamedNode, Quad } from '@rdfjs/types';
+import { Transform } from 'node:stream';
 /**
  * An executor could not run because the dataset lacks a supported distribution.
  */
@@ -42,6 +43,17 @@ export interface SparqlConstructExecutorOptions {
      * Optional custom SparqlEndpointFetcher instance.
      */
     fetcher?: SparqlEndpointFetcher;
+    /**
+     * Buffer complete lines before passing them to the N3 parser.
+     *
+     * Works around an [N3.js bug](https://github.com/rdfjs/N3.js/issues/578)
+     * where language tags (e.g. `@nl-nl`) split across HTTP chunk boundaries
+     * cause parse errors. Enable this when querying endpoints that return
+     * line-oriented formats such as N-Triples (e.g. QLever).
+     *
+     * @default false
+     */
+    lineBuffer?: boolean;
 }
 /**
  * A streaming SPARQL CONSTRUCT executor.
@@ -75,6 +87,7 @@ export declare class SparqlConstructExecutor implements Executor {
     private readonly preParsed?;
     private readonly fetcher;
     private readonly retries;
+    private readonly lineBuffer;
     private readonly generator;
     constructor(options: SparqlConstructExecutorOptions);
     /**
@@ -86,6 +99,12 @@ export declare class SparqlConstructExecutor implements Executor {
      * @returns AsyncIterable<Quad> stream of results.
      */
     execute(dataset: Dataset, distribution: Distribution, options?: ExecuteOptions): Promise<AsyncIterable<Quad>>;
+    /**
+     * Fetch quads from the endpoint, optionally line-buffering the response
+     * stream before it reaches the N3 parser to work around
+     * {@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}.
+     */
+    private fetchQuads;
     /**
      * Create an executor from a query file.
      *
@@ -98,4 +117,29 @@ export declare class SparqlConstructExecutor implements Executor {
  * Read a SPARQL query from a file.
  */
 export declare function readQueryFile(filename: string): Promise<string>;
+/**
+ * Buffers incoming data until complete lines (`\n`-terminated) are available,
+ * then pushes them downstream as a single chunk.
+ *
+ * **Why this exists:** `fetch-sparql-endpoint` pipes the raw HTTP response
+ * stream directly into N3.js's `StreamParser`. N3.js has a bug
+ * ({@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}) where
+ * tokens that straddle chunk boundaries — most commonly language tags like
+ * `@nl-nl` — cause spurious `Unexpected "-nl"` parse errors. The error is
+ * non-deterministic and typically surfaces only on responses larger than
+ * ~12 MB, because that is when HTTP chunking starts splitting mid-token.
+ *
+ * By ensuring each chunk passed to the parser ends on a line boundary, we
+ * prevent any N-Triples token from being split. Memory overhead is minimal:
+ * at most one partial line is buffered at a time.
+ *
+ * This transform can be removed once N3.js#578 is fixed upstream.
+ *
+ * @see https://github.com/rdfjs/N3.js/issues/578
+ */
+export declare class LineBufferTransform extends Transform {
+    private remainder;
+    _transform(chunk: Buffer, _encoding: string, callback: () => void): void;
+    _flush(callback: () => void): void;
+}
 //# sourceMappingURL=executor.d.ts.map

package/dist/sparql/executor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAWpD;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAmBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAwC/B;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAoBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAwC/B;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CACjB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,IAAI;IAWb,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC"}

package/dist/sparql/executor.js

@@ -2,6 +2,8 @@ import { assertSafeIri } from '@lde/dataset';
 import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
+import { Transform } from 'node:stream';
+import { StreamParser } from 'n3';
 import { Parser } from '@traqula/parser-sparql-1-1';
 import { Generator } from '@traqula/generator-sparql-1-1';
 import isNetworkError from 'is-network-error';
@@ -49,10 +51,12 @@ export class SparqlConstructExecutor {
     preParsed;
     fetcher;
     retries;
+    lineBuffer;
     generator = new Generator();
     constructor(options) {
         this.rawQuery = options.query;
         this.retries = options.retries ?? 3;
+        this.lineBuffer = options.lineBuffer ?? false;
         if (!options.query.includes('#subjectFilter#')) {
             const parsed = new Parser().parse(options.query);
             if (parsed.type !== 'query' || parsed.subType !== 'construct') {
@@ -98,11 +102,25 @@ export class SparqlConstructExecutor {
         let query = this.generator.generate(ast);
         assertSafeIri(dataset.iri.toString());
         query = query.replaceAll('?dataset', `<${dataset.iri}>`);
-        return await pRetry(() => this.fetcher.fetchTriples(endpoint.toString(), query), {
+        return await pRetry(() => this.fetchQuads(endpoint.toString(), query), {
             retries: this.retries,
             shouldRetry: ({ error }) => isTransientError(error),
         });
     }
+    /**
+     * Fetch quads from the endpoint, optionally line-buffering the response
+     * stream before it reaches the N3 parser to work around
+     * {@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}.
+     */
+    async fetchQuads(endpoint, query) {
+        if (!this.lineBuffer) {
+            return this.fetcher.fetchTriples(endpoint, query);
+        }
+        const [contentType, , responseStream] = await this.fetcher.fetchRawStream(endpoint, query, SparqlEndpointFetcher.CONTENTTYPE_TURTLE);
+        return responseStream
+            .pipe(new LineBufferTransform())
+            .pipe(new StreamParser({ format: contentType }));
+    }
     /**
      * Create an executor from a query file.
      *
@@ -120,6 +138,44 @@ export class SparqlConstructExecutor {
 export async function readQueryFile(filename) {
     return (await readFile(resolve(filename))).toString();
 }
+/**
+ * Buffers incoming data until complete lines (`\n`-terminated) are available,
+ * then pushes them downstream as a single chunk.
+ *
+ * **Why this exists:** `fetch-sparql-endpoint` pipes the raw HTTP response
+ * stream directly into N3.js's `StreamParser`. N3.js has a bug
+ * ({@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}) where
+ * tokens that straddle chunk boundaries — most commonly language tags like
+ * `@nl-nl` — cause spurious `Unexpected "-nl"` parse errors. The error is
+ * non-deterministic and typically surfaces only on responses larger than
+ * ~12 MB, because that is when HTTP chunking starts splitting mid-token.
+ *
+ * By ensuring each chunk passed to the parser ends on a line boundary, we
+ * prevent any N-Triples token from being split. Memory overhead is minimal:
+ * at most one partial line is buffered at a time.
+ *
+ * This transform can be removed once N3.js#578 is fixed upstream.
+ *
+ * @see https://github.com/rdfjs/N3.js/issues/578
+ */
+export class LineBufferTransform extends Transform {
+    remainder = '';
+    _transform(chunk, _encoding, callback) {
+        const data = this.remainder + chunk.toString();
+        const lines = data.split('\n');
+        this.remainder = lines.pop() ?? '';
+        if (lines.length > 0) {
+            this.push(lines.join('\n') + '\n');
+        }
+        callback();
+    }
+    _flush(callback) {
+        if (this.remainder.length > 0) {
+            this.push(this.remainder);
+        }
+        callback();
+    }
+}
 const transientStatusPattern = /HTTP status (\d+)/;
 function isTransientError(error) {
     if (isNetworkError(error))

package/dist/sparql/index.d.ts

@@ -1,4 +1,4 @@
-export { SparqlConstructExecutor, NotSupported, readQueryFile, type ExecuteOptions, type Executor, type SparqlConstructExecutorOptions, type VariableBindings, } from './executor.js';
+export { SparqlConstructExecutor, LineBufferTransform, NotSupported, readQueryFile, type ExecuteOptions, type Executor, type SparqlConstructExecutorOptions, type VariableBindings, } from './executor.js';
 export { SparqlItemSelector, type SparqlItemSelectorOptions, } from './selector.js';
 export { injectValues } from './values.js';
 export { withDefaultGraph } from './graph.js';

package/dist/sparql/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC"}

package/dist/sparql/index.js

@@ -1,4 +1,4 @@
-export { SparqlConstructExecutor, NotSupported, readQueryFile, } from './executor.js';
+export { SparqlConstructExecutor, LineBufferTransform, NotSupported, readQueryFile, } from './executor.js';
 export { SparqlItemSelector, } from './selector.js';
 export { injectValues } from './values.js';
 export { withDefaultGraph } from './graph.js';

package/dist/sparql/selector.d.ts

@@ -3,10 +3,13 @@ import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import type { ItemSelector } from '../stage.js';
 import type { VariableBindings } from './executor.js';
 export interface SparqlItemSelectorOptions {
-    /** SELECT query projecting at least one named variable. A LIMIT in the query sets the default page size. */
+    /**
+     * SELECT query projecting at least one named variable.
+     *
+     * A `LIMIT` clause in the query overrides the stage's `batchSize` as the
+     * page size — use this when the SPARQL endpoint enforces a result limit.
+     */
     query: string;
-    /** Results per page. Overrides any LIMIT in the query. @default 10 */
-    pageSize?: number;
     /** Custom fetcher instance. */
     fetcher?: SparqlEndpointFetcher;
 }
@@ -16,15 +19,17 @@ export interface SparqlItemSelectorOptions {
  *
  * The endpoint URL comes from the {@link Distribution} passed to {@link select}.
  * Pagination is an internal detail — consumers iterate binding rows directly.
- * If the query contains a LIMIT, it is used as the default page size
- * (can be overridden by the `pageSize` option). Pagination continues
- * until a page returns fewer results than the page size.
+ *
+ * The page size (results per SPARQL request) is determined by, in order:
+ * 1. A `LIMIT` clause in the selector query (for endpoints with hard result limits)
+ * 2. The stage's {@link StageOptions.batchSize} (passed via {@link select})
+ * 3. A default of 10
  */
 export declare class SparqlItemSelector implements ItemSelector {
     private readonly parsed;
-    private readonly pageSize;
+    private readonly queryLimit?;
     private readonly fetcher;
     constructor(options: SparqlItemSelectorOptions);
-    select(distribution: Distribution): AsyncIterableIterator<VariableBindings>;
+    select(distribution: Distribution, batchSize?: number): AsyncIterableIterator<VariableBindings>;
 }
 //# sourceMappingURL=selector.d.ts.map

package/dist/sparql/selector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtD,MAAM,WAAW,yBAAyB;IACxC,4GAA4G;IAC5G,KAAK,EAAE,MAAM,CAAC;IACd,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;GASG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;gBAEpC,OAAO,EAAE,yBAAyB;IAqBvC,MAAM,CACX,YAAY,EAAE,YAAY,GACzB,qBAAqB,CAAC,gBAAgB,CAAC;CAsC3C"}
+{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtD,MAAM,WAAW,yBAAyB;IACxC;;;;;OAKG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;gBAEpC,OAAO,EAAE,yBAAyB;IAkBvC,MAAM,CACX,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,qBAAqB,CAAC,gBAAgB,CAAC;CAuC3C"}

package/dist/sparql/selector.js

@@ -11,13 +11,15 @@ const F = new AstFactory();
  *
  * The endpoint URL comes from the {@link Distribution} passed to {@link select}.
  * Pagination is an internal detail — consumers iterate binding rows directly.
- * If the query contains a LIMIT, it is used as the default page size
- * (can be overridden by the `pageSize` option). Pagination continues
- * until a page returns fewer results than the page size.
+ *
+ * The page size (results per SPARQL request) is determined by, in order:
+ * 1. A `LIMIT` clause in the selector query (for endpoints with hard result limits)
+ * 2. The stage's {@link StageOptions.batchSize} (passed via {@link select})
+ * 3. A default of 10
  */
 export class SparqlItemSelector {
     parsed;
-    pageSize;
+    queryLimit;
     fetcher;
     constructor(options) {
         const parsed = parser.parse(options.query);
@@ -29,31 +31,29 @@ export class SparqlItemSelector {
             throw new Error('Query must project at least one named variable (SELECT * is not supported)');
         }
         this.parsed = parsed;
-        this.pageSize =
-            options.pageSize ??
-                this.parsed.solutionModifiers.limitOffset?.limit ??
-                10;
+        this.queryLimit = this.parsed.solutionModifiers.limitOffset?.limit;
         this.fetcher = options.fetcher ?? new SparqlEndpointFetcher();
     }
-    async *select(distribution) {
+    async *select(distribution, batchSize) {
+        const effectivePageSize = this.queryLimit ?? batchSize ?? 10;
         const endpoint = distribution.accessUrl;
         let offset = 0;
         while (true) {
-            this.parsed.solutionModifiers.limitOffset = F.solutionModifierLimitOffset(this.pageSize, offset, F.gen());
+            this.parsed.solutionModifiers.limitOffset = F.solutionModifierLimitOffset(effectivePageSize, offset, F.gen());
             const paginatedQuery = generator.generate(this.parsed);
             const stream = (await this.fetcher.fetchBindings(endpoint.toString(), paginatedQuery));
-            let pageSize = 0;
+            let count = 0;
             for await (const record of stream) {
                 const row = Object.fromEntries(Object.entries(record).filter(([, term]) => term.termType === 'NamedNode'));
                 if (Object.keys(row).length > 0) {
                     yield row;
-                    pageSize++;
+                    count++;
                 }
             }
-            if (pageSize === 0 || pageSize < this.pageSize) {
+            if (count === 0 || count < effectivePageSize) {
                 return;
             }
-            offset += pageSize;
+            offset += count;
         }
     }
 }

package/dist/stage.d.ts

@@ -10,7 +10,15 @@ export interface StageOptions {
     name: string;
     executors: Executor | Executor[];
     itemSelector?: ItemSelector;
-    /** Maximum number of bindings per executor call. @default 10 */
+    /**
+     * Maximum number of bindings per executor call.
+     *
+     * Also used as the selector's page size so that each paginated request
+     * fills exactly one batch. A `LIMIT` clause in the selector query
+     * overrides this for endpoints with hard result limits.
+     *
+     * @default 10
+     */
     batchSize?: number;
     /** Maximum concurrent in-flight executor batches. @default 10 */
     maxConcurrency?: number;
@@ -48,6 +56,6 @@ export declare class Stage {
 }
 /** Selects items (as variable bindings) for executors to process. Pagination is an implementation detail. */
 export interface ItemSelector {
-    select(distribution: Distribution): AsyncIterable<VariableBindings>;
+    select(distribution: Distribution, batchSize?: number): AsyncIterable<VariableBindings>;
 }
 //# sourceMappingURL=stage.d.ts.map

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAYjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAkDjB,eAAe;IA2I7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAYjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAkDjB,eAAe;IA2I7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}

package/dist/stage.js

@@ -26,7 +26,7 @@ export class Stage {
     }
     async run(dataset, distribution, writer, options) {
         if (this.itemSelector) {
-            return this.runWithSelector(this.itemSelector.select(distribution), dataset, distribution, writer, options);
+            return this.runWithSelector(this.itemSelector.select(distribution, this.batchSize), dataset, distribution, writer, options);
         }
         const streams = await this.executeAll(dataset, distribution);
         if (streams instanceof NotSupported) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.28.0",
+  "version": "0.28.2",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"