@lde/pipeline 0.35.1 → 0.35.2

package/README.md

@@ -113,6 +113,8 @@ new SparqlItemSelector({
 });
 ```
 
+A row is yielded only when **every** projected variable binds a `NamedNode` – binding values double as stable item identities downstream, which a blank-node label or literal cannot provide. Other rows are dropped client-side but still count toward pagination, so they are fetched first; to skip them at the endpoint, filter in the query itself (e.g. `FILTER(isIRI(?s))`, as `selectByClass` in `@lde/search-pipeline` does with `FILTER(!isBlank(?root))`).
+
 #### Capping total results with `maxResults`
 
 By default, `SparqlItemSelector` paginates through **all** matching rows: any `LIMIT` clause in the query is interpreted as the page size, then it walks pages with `OFFSET` until the source is exhausted. To cap the total bindings yielded across all pages – for sampling, testing, prototyping, or just safety – set `maxResults`:
@@ -127,7 +129,7 @@ new SparqlItemSelector({
 When `maxResults` is set:
 
 - Pagination stops as soon as `maxResults` bindings have been yielded – no wasted page request after the cap is hit.
-- The last (partial) page's `LIMIT` is shrunk to the remaining cap so the endpoint doesn't over-fetch on the remainder (e.g. with `maxResults: 85` and `pageSize: 10`, the 9th page request is `LIMIT 5`, not `LIMIT 10`).
+- As long as no row has been dropped, the last (partial) page's `LIMIT` is shrunk to the remaining cap so the endpoint doesn't over-fetch on the remainder (e.g. with `maxResults: 85` and `pageSize: 10`, the 9th page request is `LIMIT 5`, not `LIMIT 10`). Once rows have been dropped, pages stay at full size – shrinking to the yielded remainder would crawl a dropped-row region one row per request.
 - The first page uses the configured page size as-is; `maxResults` and page size stay orthogonal. If `maxResults < pageSize`, the first page may return a few rows that aren't yielded.
 - `maxResults: 0` is a valid no-op; the selector yields nothing without issuing any SPARQL request.
 - `maxResults` is independent of any `LIMIT` clause in the query, which still controls page size when the cap is larger than one page.

package/dist/sparql/selector.d.ts

@@ -18,15 +18,23 @@ export interface SparqlItemSelectorOptions {
      * Use this for sampling — “give me at most N items, don’t walk the full
      * source”. Independent of {@link query}’s `LIMIT`, which controls page
      * size. Pagination stops as soon as `maxResults` bindings have been
-     * yielded.
+     * yielded. Must not be negative.
      */
     maxResults?: number;
     /** Custom fetcher instance. */
     fetcher?: SparqlEndpointFetcher;
 }
 /**
- * {@link ItemSelector} that pages through SPARQL SELECT results,
- * yielding all projected variable bindings (NamedNode values only) per row.
+ * {@link ItemSelector} that pages through SPARQL SELECT results, yielding one
+ * bindings row per result row. A row is yielded only when **every** projected
+ * variable binds a NamedNode: binding values double as stable item identities
+ * downstream and are re-injected into reader queries as a `VALUES` block,
+ * which needs uniform rows – a blank-node label or literal provides no stable
+ * identity, and a partially-bound row would silently weaken the join. Any
+ * other row is dropped, but still counts toward pagination (it occupied a
+ * result slot at the endpoint), so a partly-dropped page never ends
+ * pagination early. Dropped rows are still fetched; to skip them at the
+ * endpoint, filter in the query itself (e.g. `FILTER(isIRI(?s))`).
  *
  * The endpoint URL comes from the {@link Distribution} passed to {@link select}.
  * Pagination is an internal detail — consumers iterate binding rows directly.
@@ -36,15 +44,21 @@ export interface SparqlItemSelectorOptions {
  * 2. The stage's {@link StageOptions.batchSize} (passed via {@link select})
  * 3. A default of 10
  *
+ * The page size must be positive – a page size of 0 could never terminate,
+ * so it is rejected as a configuration error.
+ *
  * {@link SparqlItemSelectorOptions.maxResults} is independent of page size:
  * it caps the *total* bindings yielded across pages without changing how
- * the first page is requested. The last (partial) page’s `LIMIT` is
- * shrunk to whatever’s left of the cap so the endpoint doesn’t over-fetch
- * on the remainder.
+ * the first page is requested. As long as no row has been dropped, the last
+ * (partial) page’s `LIMIT` is shrunk to whatever’s left of the cap so the
+ * endpoint doesn’t over-fetch on the remainder; once rows have been dropped,
+ * pages stay at full size – shrinking them to the yielded remainder would
+ * crawl a dropped-row region one row per request.
  */
 export declare class SparqlItemSelector implements ItemSelector {
     private readonly parsed;
     private readonly queryLimit?;
+    private readonly variableNames;
     private readonly maxResults?;
     private readonly userFetcher?;
     constructor(options: SparqlItemSelectorOptions);

package/dist/sparql/selector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAoBpD,MAAM,WAAW,yBAAyB;IACxC;;;;;;;;OAQG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;gBAEzC,OAAO,EAAE,yBAAyB;IAmBvC,MAAM,CACX,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,qBAAqB,CAAC,gBAAgB,CAAC;IA4D1C;;;;OAIG;YACW,uBAAuB;CA8BtC"}
+{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAoBpD,MAAM,WAAW,yBAAyB;IACxC;;;;;;;;OAQG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoB;IAClD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;gBAEzC,OAAO,EAAE,yBAAyB;IA0BvC,MAAM,CACX,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,qBAAqB,CAAC,gBAAgB,CAAC;IAwE1C;;;;OAIG;YACW,uBAAuB;CA8BtC"}

package/dist/sparql/selector.js

@@ -14,8 +14,16 @@ const parser = new Parser();
 const generator = new Generator();
 const F = new AstFactory();
 /**
- * {@link ItemSelector} that pages through SPARQL SELECT results,
- * yielding all projected variable bindings (NamedNode values only) per row.
+ * {@link ItemSelector} that pages through SPARQL SELECT results, yielding one
+ * bindings row per result row. A row is yielded only when **every** projected
+ * variable binds a NamedNode: binding values double as stable item identities
+ * downstream and are re-injected into reader queries as a `VALUES` block,
+ * which needs uniform rows – a blank-node label or literal provides no stable
+ * identity, and a partially-bound row would silently weaken the join. Any
+ * other row is dropped, but still counts toward pagination (it occupied a
+ * result slot at the endpoint), so a partly-dropped page never ends
+ * pagination early. Dropped rows are still fetched; to skip them at the
+ * endpoint, filter in the query itself (e.g. `FILTER(isIRI(?s))`).
  *
  * The endpoint URL comes from the {@link Distribution} passed to {@link select}.
  * Pagination is an internal detail — consumers iterate binding rows directly.
@@ -25,15 +33,21 @@ const F = new AstFactory();
  * 2. The stage's {@link StageOptions.batchSize} (passed via {@link select})
  * 3. A default of 10
  *
+ * The page size must be positive – a page size of 0 could never terminate,
+ * so it is rejected as a configuration error.
+ *
  * {@link SparqlItemSelectorOptions.maxResults} is independent of page size:
  * it caps the *total* bindings yielded across pages without changing how
- * the first page is requested. The last (partial) page’s `LIMIT` is
- * shrunk to whatever’s left of the cap so the endpoint doesn’t over-fetch
- * on the remainder.
+ * the first page is requested. As long as no row has been dropped, the last
+ * (partial) page’s `LIMIT` is shrunk to whatever’s left of the cap so the
+ * endpoint doesn’t over-fetch on the remainder; once rows have been dropped,
+ * pages stay at full size – shrinking them to the yielded remainder would
+ * crawl a dropped-row region one row per request.
  */
 export class SparqlItemSelector {
     parsed;
     queryLimit;
+    variableNames;
     maxResults;
     userFetcher;
     constructor(options) {
@@ -45,8 +59,12 @@ export class SparqlItemSelector {
         if (variables.length === 0) {
             throw new Error('Query must project at least one named variable (SELECT * is not supported)');
         }
+        if (options.maxResults !== undefined && options.maxResults < 0) {
+            throw new Error(`maxResults must not be negative; got ${options.maxResults}`);
+        }
         this.parsed = parsed;
         this.queryLimit = this.parsed.solutionModifiers.limitOffset?.limit;
+        this.variableNames = variables.map((variable) => variable.value);
         this.maxResults = options.maxResults;
         this.userFetcher = options.fetcher;
     }
@@ -54,27 +72,37 @@ export class SparqlItemSelector {
         if (this.maxResults === 0)
             return;
         const basePageSize = this.queryLimit ?? batchSize ?? 10;
+        if (basePageSize <= 0) {
+            throw new Error(`Page size must be positive; got ${basePageSize} (from the query’s LIMIT or the stage’s batchSize)`);
+        }
         const endpoint = distribution.accessUrl;
         const policy = options?.timeout ?? defaultTimeoutPolicy;
         let offset = 0;
+        let totalFetched = 0;
         let totalYielded = 0;
         while (true) {
             const remaining = this.maxResults !== undefined
                 ? this.maxResults - totalYielded
                 : Infinity;
-            // The first page uses the configured page size as-is — keeps page-size
-            // and total-cap orthogonal. Subsequent pages clamp to `remaining` so
-            // the last (partial) page doesn’t over-fetch.
-            const effectivePageSize = offset === 0 ? basePageSize : Math.min(basePageSize, remaining);
+            // The first page uses the configured page size as-is – keeps page-size
+            // and total-cap orthogonal. Subsequent pages clamp to `remaining` so the
+            // last (partial) page doesn’t over-fetch – but only while nothing has
+            // been dropped: once yields lag fetches, clamping would crawl a
+            // dropped-row region at down to one row per request (see class JSDoc).
+            const effectivePageSize = offset === 0 || totalFetched > totalYielded
+                ? basePageSize
+                : Math.min(basePageSize, remaining);
             this.parsed.solutionModifiers.limitOffset = F.solutionModifierLimitOffset(effectivePageSize, offset, F.gen());
             const paginatedQuery = generator.generate(this.parsed);
             const stream = await this.fetchBindingsWithPolicy(endpoint, paginatedQuery, policy);
-            let count = 0;
+            // Fetched rows drive pagination, yielded rows only `maxResults` – see
+            // the class JSDoc for why dropped rows must keep their page slot.
+            let fetched = 0;
             for await (const record of stream) {
-                const row = Object.fromEntries(Object.entries(record).filter(([, term]) => term.termType === 'NamedNode'));
-                if (Object.keys(row).length > 0) {
-                    yield row;
-                    count++;
+                fetched++;
+                totalFetched++;
+                if (this.variableNames.every((name) => record[name]?.termType === 'NamedNode')) {
+                    yield record;
                     totalYielded++;
                     if (this.maxResults !== undefined &&
                         totalYielded >= this.maxResults) {
@@ -82,10 +110,10 @@ export class SparqlItemSelector {
                     }
                 }
             }
-            if (count === 0 || count < effectivePageSize) {
+            if (fetched < effectivePageSize) {
                 return;
             }
-            offset += count;
+            offset += fetched;
         }
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.35.1",
+  "version": "0.35.2",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"