@lde/pipeline 0.35.1 → 0.35.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/README.md +3 -1
  2. package/dist/sparql/selector.d.ts +20 -6
  3. package/dist/sparql/selector.d.ts.map +1 -1
  4. package/dist/sparql/selector.js +44 -16
  5. package/package.json +7 -7
package/README.md CHANGED
@@ -113,6 +113,8 @@ new SparqlItemSelector({
113
113
  });
114
114
  ```
115
115
 
116
+ A row is yielded only when **every** projected variable binds a `NamedNode` – binding values double as stable item identities downstream, which a blank-node label or literal cannot provide. Other rows are dropped client-side but still count toward pagination, so they are fetched first; to skip them at the endpoint, filter in the query itself (e.g. `FILTER(isIRI(?s))`, as `selectByClass` in `@lde/search-pipeline` does with `FILTER(!isBlank(?root))`).
117
+
116
118
  #### Capping total results with `maxResults`
117
119
 
118
120
  By default, `SparqlItemSelector` paginates through **all** matching rows: any `LIMIT` clause in the query is interpreted as the page size, then it walks pages with `OFFSET` until the source is exhausted. To cap the total bindings yielded across all pages – for sampling, testing, prototyping, or just safety – set `maxResults`:
@@ -127,7 +129,7 @@ new SparqlItemSelector({
127
129
  When `maxResults` is set:
128
130
 
129
131
  - Pagination stops as soon as `maxResults` bindings have been yielded – no wasted page request after the cap is hit.
130
- - The last (partial) page's `LIMIT` is shrunk to the remaining cap so the endpoint doesn't over-fetch on the remainder (e.g. with `maxResults: 85` and `pageSize: 10`, the 9th page request is `LIMIT 5`, not `LIMIT 10`).
132
+ - As long as no row has been dropped, the last (partial) page's `LIMIT` is shrunk to the remaining cap so the endpoint doesn't over-fetch on the remainder (e.g. with `maxResults: 85` and `pageSize: 10`, the 9th page request is `LIMIT 5`, not `LIMIT 10`). Once rows have been dropped, pages stay at full size – shrinking to the yielded remainder would crawl a dropped-row region one row per request.
131
133
  - The first page uses the configured page size as-is; `maxResults` and page size stay orthogonal. If `maxResults < pageSize`, the first page may return a few rows that aren't yielded.
132
134
  - `maxResults: 0` is a valid no-op; the selector yields nothing without issuing any SPARQL request.
133
135
  - `maxResults` is independent of any `LIMIT` clause in the query, which still controls page size when the cap is larger than one page.
package/dist/sparql/selector.d.ts CHANGED
@@ -18,15 +18,23 @@ export interface SparqlItemSelectorOptions {
18
18
  * Use this for sampling — “give me at most N items, don’t walk the full
19
19
  * source”. Independent of {@link query}’s `LIMIT`, which controls page
20
20
  * size. Pagination stops as soon as `maxResults` bindings have been
21
- * yielded.
21
+ * yielded. Must not be negative.
22
22
  */
23
23
  maxResults?: number;
24
24
  /** Custom fetcher instance. */
25
25
  fetcher?: SparqlEndpointFetcher;
26
26
  }
27
27
  /**
28
- * {@link ItemSelector} that pages through SPARQL SELECT results,
29
- * yielding all projected variable bindings (NamedNode values only) per row.
28
+ * {@link ItemSelector} that pages through SPARQL SELECT results, yielding one
29
+ * bindings row per result row. A row is yielded only when **every** projected
30
+ * variable binds a NamedNode: binding values double as stable item identities
31
+ * downstream and are re-injected into reader queries as a `VALUES` block,
32
+ * which needs uniform rows – a blank-node label or literal provides no stable
33
+ * identity, and a partially-bound row would silently weaken the join. Any
34
+ * other row is dropped, but still counts toward pagination (it occupied a
35
+ * result slot at the endpoint), so a partly-dropped page never ends
36
+ * pagination early. Dropped rows are still fetched; to skip them at the
37
+ * endpoint, filter in the query itself (e.g. `FILTER(isIRI(?s))`).
30
38
  *
31
39
  * The endpoint URL comes from the {@link Distribution} passed to {@link select}.
32
40
  * Pagination is an internal detail — consumers iterate binding rows directly.
@@ -36,15 +44,21 @@ export interface SparqlItemSelectorOptions {
36
44
  * 2. The stage's {@link StageOptions.batchSize} (passed via {@link select})
37
45
  * 3. A default of 10
38
46
  *
47
+ * The page size must be positive – a page size of 0 could never terminate,
48
+ * so it is rejected as a configuration error.
49
+ *
39
50
  * {@link SparqlItemSelectorOptions.maxResults} is independent of page size:
40
51
  * it caps the *total* bindings yielded across pages without changing how
41
- * the first page is requested. The last (partial) page’s `LIMIT` is
42
- * shrunk to whatever’s left of the cap so the endpoint doesn’t over-fetch
43
- * on the remainder.
52
+ * the first page is requested. As long as no row has been dropped, the last
53
+ * (partial) page’s `LIMIT` is shrunk to whatever’s left of the cap so the
54
+ * endpoint doesn’t over-fetch on the remainder; once rows have been dropped,
55
+ * pages stay at full size – shrinking them to the yielded remainder would
56
+ * crawl a dropped-row region one row per request.
44
57
  */
45
58
  export declare class SparqlItemSelector implements ItemSelector {
46
59
  private readonly parsed;
47
60
  private readonly queryLimit?;
61
+ private readonly variableNames;
48
62
  private readonly maxResults?;
49
63
  private readonly userFetcher?;
50
64
  constructor(options: SparqlItemSelectorOptions);
package/dist/sparql/selector.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAoBpD,MAAM,WAAW,yBAAyB;IACxC;;;;;;;;OAQG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;gBAEzC,OAAO,EAAE,yBAAyB;IAmBvC,MAAM,CACX,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,qBAAqB,CAAC,gBAAgB,CAAC;IA4D1C;;;;OAIG;YACW,uBAAuB;CA8BtC"}
1
+ {"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAoBpD,MAAM,WAAW,yBAAyB;IACxC;;;;;;;;OAQG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoB;IAClD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;gBAEzC,OAAO,EAAE,yBAAyB;IA0BvC,MAAM,CACX,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,qBAAqB,CAAC,gBAAgB,CAAC;IAwE1C;;;;OAIG;YACW,uBAAuB;CA8BtC"}
package/dist/sparql/selector.js CHANGED
@@ -14,8 +14,16 @@ const parser = new Parser();
14
14
  const generator = new Generator();
15
15
  const F = new AstFactory();
16
16
  /**
17
- * {@link ItemSelector} that pages through SPARQL SELECT results,
18
- * yielding all projected variable bindings (NamedNode values only) per row.
17
+ * {@link ItemSelector} that pages through SPARQL SELECT results, yielding one
18
+ * bindings row per result row. A row is yielded only when **every** projected
19
+ * variable binds a NamedNode: binding values double as stable item identities
20
+ * downstream and are re-injected into reader queries as a `VALUES` block,
21
+ * which needs uniform rows – a blank-node label or literal provides no stable
22
+ * identity, and a partially-bound row would silently weaken the join. Any
23
+ * other row is dropped, but still counts toward pagination (it occupied a
24
+ * result slot at the endpoint), so a partly-dropped page never ends
25
+ * pagination early. Dropped rows are still fetched; to skip them at the
26
+ * endpoint, filter in the query itself (e.g. `FILTER(isIRI(?s))`).
19
27
  *
20
28
  * The endpoint URL comes from the {@link Distribution} passed to {@link select}.
21
29
  * Pagination is an internal detail — consumers iterate binding rows directly.
@@ -25,15 +33,21 @@ const F = new AstFactory();
25
33
  * 2. The stage's {@link StageOptions.batchSize} (passed via {@link select})
26
34
  * 3. A default of 10
27
35
  *
36
+ * The page size must be positive – a page size of 0 could never terminate,
37
+ * so it is rejected as a configuration error.
38
+ *
28
39
  * {@link SparqlItemSelectorOptions.maxResults} is independent of page size:
29
40
  * it caps the *total* bindings yielded across pages without changing how
30
- * the first page is requested. The last (partial) page’s `LIMIT` is
31
- * shrunk to whatever’s left of the cap so the endpoint doesn’t over-fetch
32
- * on the remainder.
41
+ * the first page is requested. As long as no row has been dropped, the last
42
+ * (partial) page’s `LIMIT` is shrunk to whatever’s left of the cap so the
43
+ * endpoint doesn’t over-fetch on the remainder; once rows have been dropped,
44
+ * pages stay at full size – shrinking them to the yielded remainder would
45
+ * crawl a dropped-row region one row per request.
33
46
  */
34
47
  export class SparqlItemSelector {
35
48
  parsed;
36
49
  queryLimit;
50
+ variableNames;
37
51
  maxResults;
38
52
  userFetcher;
39
53
  constructor(options) {
@@ -45,8 +59,12 @@ export class SparqlItemSelector {
45
59
  if (variables.length === 0) {
46
60
  throw new Error('Query must project at least one named variable (SELECT * is not supported)');
47
61
  }
62
+ if (options.maxResults !== undefined && options.maxResults < 0) {
63
+ throw new Error(`maxResults must not be negative; got ${options.maxResults}`);
64
+ }
48
65
  this.parsed = parsed;
49
66
  this.queryLimit = this.parsed.solutionModifiers.limitOffset?.limit;
67
+ this.variableNames = variables.map((variable) => variable.value);
50
68
  this.maxResults = options.maxResults;
51
69
  this.userFetcher = options.fetcher;
52
70
  }
@@ -54,27 +72,37 @@ export class SparqlItemSelector {
54
72
  if (this.maxResults === 0)
55
73
  return;
56
74
  const basePageSize = this.queryLimit ?? batchSize ?? 10;
75
+ if (basePageSize <= 0) {
76
+ throw new Error(`Page size must be positive; got ${basePageSize} (from the query’s LIMIT or the stage’s batchSize)`);
77
+ }
57
78
  const endpoint = distribution.accessUrl;
58
79
  const policy = options?.timeout ?? defaultTimeoutPolicy;
59
80
  let offset = 0;
81
+ let totalFetched = 0;
60
82
  let totalYielded = 0;
61
83
  while (true) {
62
84
  const remaining = this.maxResults !== undefined
63
85
  ? this.maxResults - totalYielded
64
86
  : Infinity;
65
- // The first page uses the configured page size as-is — keeps page-size
66
- // and total-cap orthogonal. Subsequent pages clamp to `remaining` so
67
- // the last (partial) page doesn’t over-fetch.
68
- const effectivePageSize = offset === 0 ? basePageSize : Math.min(basePageSize, remaining);
87
+ // The first page uses the configured page size as-is – keeps page-size
88
+ // and total-cap orthogonal. Subsequent pages clamp to `remaining` so the
89
+ // last (partial) page doesn’t over-fetch – but only while nothing has
90
+ // been dropped: once yields lag fetches, clamping would crawl a
91
+ // dropped-row region at down to one row per request (see class JSDoc).
92
+ const effectivePageSize = offset === 0 || totalFetched > totalYielded
93
+ ? basePageSize
94
+ : Math.min(basePageSize, remaining);
69
95
  this.parsed.solutionModifiers.limitOffset = F.solutionModifierLimitOffset(effectivePageSize, offset, F.gen());
70
96
  const paginatedQuery = generator.generate(this.parsed);
71
97
  const stream = await this.fetchBindingsWithPolicy(endpoint, paginatedQuery, policy);
72
- let count = 0;
98
+ // Fetched rows drive pagination, yielded rows only `maxResults` – see
99
+ // the class JSDoc for why dropped rows must keep their page slot.
100
+ let fetched = 0;
73
101
  for await (const record of stream) {
74
- const row = Object.fromEntries(Object.entries(record).filter(([, term]) => term.termType === 'NamedNode'));
75
- if (Object.keys(row).length > 0) {
76
- yield row;
77
- count++;
102
+ fetched++;
103
+ totalFetched++;
104
+ if (this.variableNames.every((name) => record[name]?.termType === 'NamedNode')) {
105
+ yield record;
78
106
  totalYielded++;
79
107
  if (this.maxResults !== undefined &&
80
108
  totalYielded >= this.maxResults) {
@@ -82,10 +110,10 @@ export class SparqlItemSelector {
82
110
  }
83
111
  }
84
112
  }
85
- if (count === 0 || count < effectivePageSize) {
113
+ if (fetched < effectivePageSize) {
86
114
  return;
87
115
  }
88
- offset += count;
116
+ offset += fetched;
89
117
  }
90
118
  }
91
119
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lde/pipeline",
3
- "version": "0.35.1",
3
+ "version": "0.35.3",
4
4
  "repository": {
5
5
  "url": "git+https://github.com/ldelements/lde.git",
6
6
  "directory": "packages/pipeline"
@@ -24,12 +24,12 @@
24
24
  "!**/*.tsbuildinfo"
25
25
  ],
26
26
  "dependencies": {
27
- "@lde/dataset": "^0.7.8",
28
- "@lde/dataset-registry-client": "^0.9.0",
29
- "@lde/distribution-health": "^0.2.8",
30
- "@lde/distribution-probe": "^0.2.6",
31
- "@lde/sparql-importer": "^0.6.8",
32
- "@lde/sparql-server": "^0.4.13",
27
+ "@lde/dataset": "^0.7.9",
28
+ "@lde/dataset-registry-client": "^0.9.1",
29
+ "@lde/distribution-health": "^0.2.9",
30
+ "@lde/distribution-probe": "^0.2.7",
31
+ "@lde/sparql-importer": "^0.6.9",
32
+ "@lde/sparql-server": "^0.4.14",
33
33
  "@rdfjs/namespace": "^2.0.1",
34
34
  "@rdfjs/types": "^2.0.1",
35
35
  "@tpluscode/rdf-ns-builders": "^5.0.0",