@lde/pipeline 0.28.0 → 0.28.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/README.md +38 -4
  2. package/dist/sparql/selector.d.ts +13 -8
  3. package/dist/sparql/selector.d.ts.map +1 -1
  4. package/dist/sparql/selector.js +14 -14
  5. package/dist/stage.d.ts +10 -2
  6. package/dist/stage.d.ts.map +1 -1
  7. package/dist/stage.js +1 -1
  8. package/package.json +1 -1
package/README.md CHANGED
@@ -30,17 +30,51 @@ const selector = new RegistrySelector({
30
30
  const selector = new ManualDatasetSelection([dataset]);
31
31
  ```
32
32
 
33
+ ### Stage
34
+
35
+ A stage groups an item selector, one or more executors, and configuration:
36
+
37
+ ```typescript
38
+ new Stage({
39
+ name: 'per-class',
40
+ itemSelector: new SparqlItemSelector({
41
+ query: 'SELECT DISTINCT ?class WHERE { ?s a ?class }',
42
+ }),
43
+ executors: executor,
44
+ batchSize: 100,
45
+ maxConcurrency: 5,
46
+ });
47
+ ```
48
+
49
+ #### Batch size
50
+
51
+ `batchSize` (default: 10) controls how many variable bindings are passed to each executor call as a `VALUES` clause. It also sets the page size for the item selector's SPARQL requests, so that each paginated request fills exactly one executor batch.
52
+
53
+ A `LIMIT` clause in the selector query overrides `batchSize` as the page size — use this when the SPARQL endpoint enforces a hard result limit:
54
+
55
+ ```typescript
56
+ // Endpoint caps results at 1000, but process in batches of 100.
57
+ new Stage({
58
+ name: 'per-class',
59
+ itemSelector: new SparqlItemSelector({
60
+ query: 'SELECT DISTINCT ?class WHERE { ?s a ?class } LIMIT 1000',
61
+ }),
62
+ executors: executor,
63
+ batchSize: 100,
64
+ });
65
+ ```
66
+
33
67
  ### Item Selector
34
68
 
35
69
  Selects resources from the distribution and fans out executor calls per batch of results. Implements the `ItemSelector` interface:
36
70
 
37
71
  ```typescript
38
72
  interface ItemSelector {
39
- select(distribution: Distribution): AsyncIterable<VariableBindings>;
73
+ select(distribution: Distribution, batchSize?: number): AsyncIterable<VariableBindings>;
40
74
  }
41
75
  ```
42
76
 
43
- The distribution is received at run time, so selectors don't need the endpoint URL at construction time. Use `SparqlItemSelector` for SPARQL-based selection with automatic pagination:
77
+ The distribution is received at run time, so selectors don't need the endpoint URL at construction time. The `batchSize` parameter is set by the stage. Use `SparqlItemSelector` for SPARQL-based selection with automatic pagination:
44
78
 
45
79
  ```typescript
46
80
  new SparqlItemSelector({
@@ -52,9 +86,9 @@ For dynamic queries that depend on the distribution, implement `ItemSelector` di
52
86
 
53
87
  ```typescript
54
88
  const itemSelector: ItemSelector = {
55
- select: (distribution) => {
89
+ select: (distribution, batchSize) => {
56
90
  const query = buildQuery(distribution);
57
- return new SparqlItemSelector({ query }).select(distribution);
91
+ return new SparqlItemSelector({ query }).select(distribution, batchSize);
58
92
  },
59
93
  };
60
94
  ```
package/dist/sparql/selector.d.ts CHANGED
@@ -3,10 +3,13 @@ import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
3
3
  import type { ItemSelector } from '../stage.js';
4
4
  import type { VariableBindings } from './executor.js';
5
5
  export interface SparqlItemSelectorOptions {
6
- /** SELECT query projecting at least one named variable. A LIMIT in the query sets the default page size. */
6
+ /**
7
+ * SELECT query projecting at least one named variable.
8
+ *
9
+ * A `LIMIT` clause in the query overrides the stage's `batchSize` as the
10
+ * page size — use this when the SPARQL endpoint enforces a result limit.
11
+ */
7
12
  query: string;
8
- /** Results per page. Overrides any LIMIT in the query. @default 10 */
9
- pageSize?: number;
10
13
  /** Custom fetcher instance. */
11
14
  fetcher?: SparqlEndpointFetcher;
12
15
  }
@@ -16,15 +19,17 @@ export interface SparqlItemSelectorOptions {
16
19
  *
17
20
  * The endpoint URL comes from the {@link Distribution} passed to {@link select}.
18
21
  * Pagination is an internal detail — consumers iterate binding rows directly.
19
- * If the query contains a LIMIT, it is used as the default page size
20
- * (can be overridden by the `pageSize` option). Pagination continues
21
- * until a page returns fewer results than the page size.
22
+ *
23
+ * The page size (results per SPARQL request) is determined by, in order:
24
+ * 1. A `LIMIT` clause in the selector query (for endpoints with hard result limits)
25
+ * 2. The stage's {@link StageOptions.batchSize} (passed via {@link select})
26
+ * 3. A default of 10
22
27
  */
23
28
  export declare class SparqlItemSelector implements ItemSelector {
24
29
  private readonly parsed;
25
- private readonly pageSize;
30
+ private readonly queryLimit?;
26
31
  private readonly fetcher;
27
32
  constructor(options: SparqlItemSelectorOptions);
28
- select(distribution: Distribution): AsyncIterableIterator<VariableBindings>;
33
+ select(distribution: Distribution, batchSize?: number): AsyncIterableIterator<VariableBindings>;
29
34
  }
30
35
  //# sourceMappingURL=selector.d.ts.map
package/dist/sparql/selector.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtD,MAAM,WAAW,yBAAyB;IACxC,4GAA4G;IAC5G,KAAK,EAAE,MAAM,CAAC;IACd,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;GASG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;gBAEpC,OAAO,EAAE,yBAAyB;IAqBvC,MAAM,CACX,YAAY,EAAE,YAAY,GACzB,qBAAqB,CAAC,gBAAgB,CAAC;CAsC3C"}
1
+ {"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtD,MAAM,WAAW,yBAAyB;IACxC;;;;;OAKG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;gBAEpC,OAAO,EAAE,yBAAyB;IAkBvC,MAAM,CACX,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,qBAAqB,CAAC,gBAAgB,CAAC;CAuC3C"}
package/dist/sparql/selector.js CHANGED
@@ -11,13 +11,15 @@ const F = new AstFactory();
11
11
  *
12
12
  * The endpoint URL comes from the {@link Distribution} passed to {@link select}.
13
13
  * Pagination is an internal detail — consumers iterate binding rows directly.
14
- * If the query contains a LIMIT, it is used as the default page size
15
- * (can be overridden by the `pageSize` option). Pagination continues
16
- * until a page returns fewer results than the page size.
14
+ *
15
+ * The page size (results per SPARQL request) is determined by, in order:
16
+ * 1. A `LIMIT` clause in the selector query (for endpoints with hard result limits)
17
+ * 2. The stage's {@link StageOptions.batchSize} (passed via {@link select})
18
+ * 3. A default of 10
17
19
  */
18
20
  export class SparqlItemSelector {
19
21
  parsed;
20
- pageSize;
22
+ queryLimit;
21
23
  fetcher;
22
24
  constructor(options) {
23
25
  const parsed = parser.parse(options.query);
@@ -29,31 +31,29 @@ export class SparqlItemSelector {
29
31
  throw new Error('Query must project at least one named variable (SELECT * is not supported)');
30
32
  }
31
33
  this.parsed = parsed;
32
- this.pageSize =
33
- options.pageSize ??
34
- this.parsed.solutionModifiers.limitOffset?.limit ??
35
- 10;
34
+ this.queryLimit = this.parsed.solutionModifiers.limitOffset?.limit;
36
35
  this.fetcher = options.fetcher ?? new SparqlEndpointFetcher();
37
36
  }
38
- async *select(distribution) {
37
+ async *select(distribution, batchSize) {
38
+ const effectivePageSize = this.queryLimit ?? batchSize ?? 10;
39
39
  const endpoint = distribution.accessUrl;
40
40
  let offset = 0;
41
41
  while (true) {
42
- this.parsed.solutionModifiers.limitOffset = F.solutionModifierLimitOffset(this.pageSize, offset, F.gen());
42
+ this.parsed.solutionModifiers.limitOffset = F.solutionModifierLimitOffset(effectivePageSize, offset, F.gen());
43
43
  const paginatedQuery = generator.generate(this.parsed);
44
44
  const stream = (await this.fetcher.fetchBindings(endpoint.toString(), paginatedQuery));
45
- let pageSize = 0;
45
+ let count = 0;
46
46
  for await (const record of stream) {
47
47
  const row = Object.fromEntries(Object.entries(record).filter(([, term]) => term.termType === 'NamedNode'));
48
48
  if (Object.keys(row).length > 0) {
49
49
  yield row;
50
- pageSize++;
50
+ count++;
51
51
  }
52
52
  }
53
- if (pageSize === 0 || pageSize < this.pageSize) {
53
+ if (count === 0 || count < effectivePageSize) {
54
54
  return;
55
55
  }
56
- offset += pageSize;
56
+ offset += count;
57
57
  }
58
58
  }
59
59
  }
package/dist/stage.d.ts CHANGED
@@ -10,7 +10,15 @@ export interface StageOptions {
10
10
  name: string;
11
11
  executors: Executor | Executor[];
12
12
  itemSelector?: ItemSelector;
13
- /** Maximum number of bindings per executor call. @default 10 */
13
+ /**
14
+ * Maximum number of bindings per executor call.
15
+ *
16
+ * Also used as the selector's page size so that each paginated request
17
+ * fills exactly one batch. A `LIMIT` clause in the selector query
18
+ * overrides this for endpoints with hard result limits.
19
+ *
20
+ * @default 10
21
+ */
14
22
  batchSize?: number;
15
23
  /** Maximum concurrent in-flight executor batches. @default 10 */
16
24
  maxConcurrency?: number;
@@ -48,6 +56,6 @@ export declare class Stage {
48
56
  }
49
57
  /** Selects items (as variable bindings) for executors to process. Pagination is an implementation detail. */
50
58
  export interface ItemSelector {
51
- select(distribution: Distribution): AsyncIterable<VariableBindings>;
59
+ select(distribution: Distribution, batchSize?: number): AsyncIterable<VariableBindings>;
52
60
  }
53
61
  //# sourceMappingURL=stage.d.ts.map
package/dist/stage.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAYjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAkDjB,eAAe;IA2I7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}
1
+ {"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAYjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAkDjB,eAAe;IA2I7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
package/dist/stage.js CHANGED
@@ -26,7 +26,7 @@ export class Stage {
26
26
  }
27
27
  async run(dataset, distribution, writer, options) {
28
28
  if (this.itemSelector) {
29
- return this.runWithSelector(this.itemSelector.select(distribution), dataset, distribution, writer, options);
29
+ return this.runWithSelector(this.itemSelector.select(distribution, this.batchSize), dataset, distribution, writer, options);
30
30
  }
31
31
  const streams = await this.executeAll(dataset, distribution);
32
32
  if (streams instanceof NotSupported) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lde/pipeline",
3
- "version": "0.28.0",
3
+ "version": "0.28.1",
4
4
  "repository": {
5
5
  "url": "git+https://github.com/ldelements/lde.git",
6
6
  "directory": "packages/pipeline"