@lde/pipeline 0.6.27 → 0.6.29

package/README.md

@@ -2,63 +2,109 @@
 
 Framework for building RDF data processing pipelines with SPARQL.
 
-## Features
-
-- **Pipeline** — orchestrates steps that process DCAT datasets
-- **PipelineBuilder** — fluent API for constructing pipelines from steps and selectors
-- **PipelineConfig** — load pipeline configuration from YAML/JSON files
-- **SparqlConstructExecutor** — streaming SPARQL CONSTRUCT with template substitution and variable bindings
-- **Distribution analysis** — probe and analyze dataset distributions
-
 ## Components
 
 A **Pipeline** consists of:
 
-- one **[Dataset Selector](#dataset-selector)**
-- one **[Distribution Resolver](#distribution-resolver)** that resolves the input dataset to a usable SPARQL distribution
+- a **Dataset Selector** that selects which datasets to process
+- a **Distribution Resolver** that resolves each dataset to a usable SPARQL endpoint
 - one or more **Stages**, each consisting of:
-  - an optional **Selector** that filters resources
-  - one or more **Executors** that generate triples for each selected resource
+  - an optional **Item Selector** that selects resources (as variable bindings) for fan-out
+  - one or more **Executors** that generate triples
 
 ### Dataset Selector
 
-Selects datasets, either manually by the user or dynamically by querying a DCAT Dataset Registry.
+Selects datasets, either manually or by querying a DCAT Dataset Registry:
 
-### Distribution Resolver
+```typescript
+// From a registry
+const selector = new RegistrySelector({
+  registry: new Client(new URL('https://example.com/sparql')),
+});
 
-Resolves each selected dataset to a usable distribution.
+// Manual
+const selector = new ManualDatasetSelection([dataset]);
+```
 
-#### SPARQL Distribution Resolver
+### Item Selector
 
-If a working SPARQL endpoint is already available for the dataset, that is used.
-If not, and a valid RDF datadump is available, that is imported to a local SPARQL server.
+Selects resources from the distribution and fans out executor calls per batch of results. Implements the `ItemSelector` interface:
 
-#### Other Distribution Resolvers
+```typescript
+interface ItemSelector {
+  select(distribution: Distribution): AsyncIterable<VariableBindings>;
+}
+```
 
-### Bindings Selector
+The distribution is received at run time, so selectors don't need the endpoint URL at construction time. Use `SparqlItemSelector` for SPARQL-based selection with automatic pagination:
 
-Selects resources from the dataset and to fan out queries per result in the executor.
-Bindings are free, and replaced with `VALUES { ... }`.
+```typescript
+new SparqlItemSelector({
+  query: 'SELECT DISTINCT ?class WHERE { ?s a ?class }',
+});
+```
+
+For dynamic queries that depend on the distribution, implement `ItemSelector` directly:
+
+```typescript
+const itemSelector: ItemSelector = {
+  select: (distribution) => {
+    const query = buildQuery(distribution);
+    return new SparqlItemSelector({ query }).select(distribution);
+  },
+};
+```
 
 ### Executor
 
+Generates RDF triples. `SparqlConstructExecutor` runs a SPARQL CONSTRUCT query with template substitution and variable bindings:
+
+```typescript
+const executor = new SparqlConstructExecutor({
+  query: 'CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }',
+});
+```
+
+### Writer
+
+Writes generated quads to a destination:
+
+- `SparqlUpdateWriter` — writes to a SPARQL endpoint via UPDATE queries
+- `FileWriter` — writes to local files
+
 ## Usage
 
 ```typescript
 import {
-  PipelineBuilder,
+  Pipeline,
+  Stage,
   SparqlConstructExecutor,
-  collect,
+  SparqlItemSelector,
+  SparqlUpdateWriter,
+  ManualDatasetSelection,
+  SparqlDistributionResolver,
 } from '@lde/pipeline';
 
-// Build a pipeline from steps
-const pipeline = new PipelineBuilder().addStep(myStep).build();
-
-// Or use the SPARQL executor directly
-const executor = new SparqlConstructExecutor({
-  query: 'CONSTRUCT { ?dataset ?p ?o } WHERE { ?s ?p ?o }',
+const pipeline = new Pipeline({
+  datasetSelector: new ManualDatasetSelection([dataset]),
+  stages: [
+    new Stage({
+      name: 'per-class',
+      itemSelector: new SparqlItemSelector({
+        query: 'SELECT DISTINCT ?class WHERE { ?s a ?class }',
+      }),
+      executors: new SparqlConstructExecutor({
+        query:
+          'CONSTRUCT { ?class a <http://example.org/Class> } WHERE { ?s a ?class }',
+      }),
+    }),
+  ],
+  writers: new SparqlUpdateWriter({
+    endpoint: new URL('http://localhost:7200/repositories/lde/statements'),
+  }),
 });
-const result = await executor.execute(dataset);
+
+await pipeline.run();
 ```
 
 ## Validation

package/dist/pipeline.d.ts

@@ -5,18 +5,26 @@ import { type DistributionResolver } from './distribution/resolver.js';
 import type { StageOutputResolver } from './stageOutputResolver.js';
 import type { ProgressReporter } from './progressReporter.js';
 export interface PipelineOptions {
-    name: string;
     datasetSelector: DatasetSelector;
     stages: Stage[];
-    writer: Writer;
-    distributionResolver: DistributionResolver;
-    stageOutputResolver?: StageOutputResolver;
-    outputDir?: string;
-    outputFormat?: 'turtle' | 'n-triples' | 'n-quads';
+    writers: Writer | Writer[];
+    name?: string;
+    distributionResolver?: DistributionResolver;
+    chaining?: {
+        stageOutputResolver: StageOutputResolver;
+        outputDir: string;
+        outputFormat?: 'turtle' | 'n-triples' | 'n-quads';
+    };
     reporter?: ProgressReporter;
 }
 export declare class Pipeline {
-    private readonly options;
+    private readonly name;
+    private readonly datasetSelector;
+    private readonly stages;
+    private readonly writer;
+    private readonly distributionResolver;
+    private readonly chaining?;
+    private readonly reporter?;
     constructor(options: PipelineOptions);
     run(): Promise<void>;
     private processDataset;

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,oBAAoB,CAAC;IAC3C,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAClD,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,eAAe;IAe9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAcZ,cAAc;YA2Bd,QAAQ;YAgCR,QAAQ;YAoDR,eAAe;YAoCd,SAAS;CAUzB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAGpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;KACnD,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAmBD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAoB9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;YA0Bd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}

package/dist/pipeline.js

@@ -2,40 +2,67 @@ import { createReadStream } from 'node:fs';
 import { StreamParser } from 'n3';
 import { FileWriter } from './writer/fileWriter.js';
 import { NoDistributionAvailable, } from './distribution/resolver.js';
+import { SparqlDistributionResolver } from './distribution/index.js';
 import { NotSupported } from './sparql/executor.js';
+class FanOutWriter {
+    writers;
+    constructor(writers) {
+        this.writers = writers;
+    }
+    async write(dataset, quads) {
+        const collected = [];
+        for await (const quad of quads)
+            collected.push(quad);
+        for (const w of this.writers) {
+            await w.write(dataset, (async function* () {
+                yield* collected;
+            })());
+        }
+    }
+}
 export class Pipeline {
-    options;
+    name;
+    datasetSelector;
+    stages;
+    writer;
+    distributionResolver;
+    chaining;
+    reporter;
     constructor(options) {
         const hasSubStages = options.stages.some((stage) => stage.stages.length > 0);
-        if (hasSubStages && !options.stageOutputResolver) {
-            throw new Error('stageOutputResolver is required when any stage has sub-stages');
-        }
-        if (hasSubStages && !options.outputDir) {
-            throw new Error('outputDir is required when any stage has sub-stages');
+        if (hasSubStages && !options.chaining) {
+            throw new Error('chaining is required when any stage has sub-stages');
         }
-        this.options = options;
+        this.name = options.name ?? '';
+        this.datasetSelector = options.datasetSelector;
+        this.stages = options.stages;
+        this.writer = Array.isArray(options.writers)
+            ? new FanOutWriter(options.writers)
+            : options.writers;
+        this.distributionResolver =
+            options.distributionResolver ?? new SparqlDistributionResolver();
+        this.chaining = options.chaining;
+        this.reporter = options.reporter;
     }
     async run() {
-        const { datasetSelector, reporter, name } = this.options;
         const start = Date.now();
-        reporter?.pipelineStart(name);
-        const datasets = await datasetSelector.select();
+        this.reporter?.pipelineStart(this.name);
+        const datasets = await this.datasetSelector.select();
         for await (const dataset of datasets) {
             await this.processDataset(dataset);
         }
-        reporter?.pipelineComplete({ duration: Date.now() - start });
+        this.reporter?.pipelineComplete({ duration: Date.now() - start });
     }
     async processDataset(dataset) {
-        const { distributionResolver, reporter } = this.options;
         const datasetIri = dataset.iri.toString();
-        reporter?.datasetStart(datasetIri);
-        const resolved = await distributionResolver.resolve(dataset);
+        this.reporter?.datasetStart(datasetIri);
+        const resolved = await this.distributionResolver.resolve(dataset);
         if (resolved instanceof NoDistributionAvailable) {
-            reporter?.datasetSkipped(datasetIri, resolved.message);
+            this.reporter?.datasetSkipped(datasetIri, resolved.message);
             return;
         }
         try {
-            for (const stage of this.options.stages) {
+            for (const stage of this.stages) {
                 if (stage.stages.length > 0) {
                     await this.runChain(dataset, resolved.distribution, stage);
                 }
@@ -47,26 +74,25 @@ export class Pipeline {
         catch {
             // Stage error for this dataset; continue to next dataset.
         }
-        reporter?.datasetComplete(datasetIri);
+        this.reporter?.datasetComplete(datasetIri);
     }
     async runStage(dataset, distribution, stage) {
-        const { writer, reporter } = this.options;
-        reporter?.stageStart(stage.name);
+        this.reporter?.stageStart(stage.name);
         const stageStart = Date.now();
         let elementsProcessed = 0;
         let quadsGenerated = 0;
-        const result = await stage.run(dataset, distribution, writer, {
+        const result = await stage.run(dataset, distribution, this.writer, {
             onProgress: (elements, quads) => {
                 elementsProcessed = elements;
                 quadsGenerated = quads;
-                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+                this.reporter?.stageProgress({ elementsProcessed, quadsGenerated });
             },
         });
         if (result instanceof NotSupported) {
-            reporter?.stageSkipped(stage.name, result.message);
+            this.reporter?.stageSkipped(stage.name, result.message);
         }
         else {
-            reporter?.stageComplete(stage.name, {
+            this.reporter?.stageComplete(stage.name, {
                 elementsProcessed,
                 quadsGenerated,
                 duration: Date.now() - stageStart,
@@ -74,7 +100,7 @@ export class Pipeline {
         }
     }
     async runChain(dataset, distribution, stage) {
-        const { writer, stageOutputResolver, outputDir, outputFormat } = this.options;
+        const { stageOutputResolver, outputDir, outputFormat } = this.chaining;
         const outputFiles = [];
         try {
             // 1. Run parent stage → FileWriter.
@@ -99,15 +125,14 @@ export class Pipeline {
                 }
             }
             // 3. Concatenate all output files → user writer.
-            await writer.write(dataset, this.readFiles(outputFiles));
+            await this.writer.write(dataset, this.readFiles(outputFiles));
         }
         finally {
             await stageOutputResolver.cleanup();
         }
     }
     async runChainedStage(dataset, distribution, stage, stageWriter) {
-        const { reporter } = this.options;
-        reporter?.stageStart(stage.name);
+        this.reporter?.stageStart(stage.name);
         const stageStart = Date.now();
         let elementsProcessed = 0;
         let quadsGenerated = 0;
@@ -115,14 +140,14 @@ export class Pipeline {
             onProgress: (elements, quads) => {
                 elementsProcessed = elements;
                 quadsGenerated = quads;
-                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+                this.reporter?.stageProgress({ elementsProcessed, quadsGenerated });
             },
         });
         if (result instanceof NotSupported) {
-            reporter?.stageSkipped(stage.name, result.message);
+            this.reporter?.stageSkipped(stage.name, result.message);
             throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
         }
-        reporter?.stageComplete(stage.name, {
+        this.reporter?.stageComplete(stage.name, {
             elementsProcessed,
             quadsGenerated,
             duration: Date.now() - stageStart,

package/dist/sparql/selector.d.ts

@@ -1,11 +1,10 @@
+import type { Distribution } from '@lde/dataset';
 import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import type { ItemSelector } from '../stage.js';
 import type { VariableBindings } from './executor.js';
 export interface SparqlItemSelectorOptions {
     /** SELECT query projecting at least one named variable. A LIMIT in the query sets the default page size. */
     query: string;
-    /** SPARQL endpoint URL. */
-    endpoint: URL;
     /** Results per page. Overrides any LIMIT in the query. @default 10 */
     pageSize?: number;
     /** Custom fetcher instance. */
@@ -15,6 +14,7 @@ export interface SparqlItemSelectorOptions {
  * {@link ItemSelector} that pages through SPARQL SELECT results,
  * yielding all projected variable bindings (NamedNode values only) per row.
  *
+ * The endpoint URL comes from the {@link Distribution} passed to {@link select}.
  * Pagination is an internal detail — consumers iterate binding rows directly.
  * If the query contains a LIMIT, it is used as the default page size
  * (can be overridden by the `pageSize` option). Pagination continues
@@ -22,10 +22,9 @@ export interface SparqlItemSelectorOptions {
  */
 export declare class SparqlItemSelector implements ItemSelector {
     private readonly parsed;
-    private readonly endpoint;
     private readonly pageSize;
     private readonly fetcher;
     constructor(options: SparqlItemSelectorOptions);
-    [Symbol.asyncIterator](): AsyncIterableIterator<VariableBindings>;
+    select(distribution: Distribution): AsyncIterableIterator<VariableBindings>;
 }
 //# sourceMappingURL=selector.d.ts.map

package/dist/sparql/selector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAKtD,MAAM,WAAW,yBAAyB;IACxC,4GAA4G;IAC5G,KAAK,EAAE,MAAM,CAAC;IACd,2BAA2B;IAC3B,QAAQ,EAAE,GAAG,CAAC;IACd,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;GAQG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;gBAEpC,OAAO,EAAE,yBAAyB;IAmBvC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,gBAAgB,CAAC;CAkCzE"}
+{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAKtD,MAAM,WAAW,yBAAyB;IACxC,4GAA4G;IAC5G,KAAK,EAAE,MAAM,CAAC;IACd,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;GASG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;gBAEpC,OAAO,EAAE,yBAAyB;IAkBvC,MAAM,CACX,YAAY,EAAE,YAAY,GACzB,qBAAqB,CAAC,gBAAgB,CAAC;CAmC3C"}

package/dist/sparql/selector.js

@@ -6,6 +6,7 @@ const generator = new Generator();
  * {@link ItemSelector} that pages through SPARQL SELECT results,
  * yielding all projected variable bindings (NamedNode values only) per row.
  *
+ * The endpoint URL comes from the {@link Distribution} passed to {@link select}.
  * Pagination is an internal detail — consumers iterate binding rows directly.
  * If the query contains a LIMIT, it is used as the default page size
  * (can be overridden by the `pageSize` option). Pagination continues
@@ -13,7 +14,6 @@ const generator = new Generator();
  */
 export class SparqlItemSelector {
     parsed;
-    endpoint;
     pageSize;
     fetcher;
     constructor(options) {
@@ -26,17 +26,17 @@ export class SparqlItemSelector {
             throw new Error('Query must project at least one named variable (SELECT * is not supported)');
         }
         this.parsed = parsed;
-        this.endpoint = options.endpoint;
         this.pageSize = options.pageSize ?? parsed.limit ?? 10;
         this.fetcher = options.fetcher ?? new SparqlEndpointFetcher();
     }
-    async *[Symbol.asyncIterator]() {
+    async *select(distribution) {
+        const endpoint = distribution.accessUrl;
         let offset = 0;
         while (true) {
             this.parsed.limit = this.pageSize;
             this.parsed.offset = offset;
             const paginatedQuery = generator.stringify(this.parsed);
-            const stream = (await this.fetcher.fetchBindings(this.endpoint.toString(), paginatedQuery));
+            const stream = (await this.fetcher.fetchBindings(endpoint.toString(), paginatedQuery));
             let pageSize = 0;
             for await (const record of stream) {
                 const row = Object.fromEntries(Object.entries(record).filter(([, term]) => term.termType === 'NamedNode'));

package/dist/stage.d.ts

@@ -2,12 +2,10 @@ import { Dataset, Distribution } from '@lde/dataset';
 import type { Executor, VariableBindings } from './sparql/executor.js';
 import { NotSupported } from './sparql/executor.js';
 import type { Writer } from './writer/writer.js';
-/** An item selector, or a factory that receives the runtime distribution. */
-export type ItemSelectorInput = ItemSelector | ((distribution: Distribution) => ItemSelector);
 export interface StageOptions {
     name: string;
     executors: Executor | Executor[];
-    itemSelector?: ItemSelectorInput;
+    itemSelector?: ItemSelector;
     /** Maximum number of bindings per executor call. @default 10 */
     batchSize?: number;
     /** Maximum concurrent in-flight executor batches. @default 10 */
@@ -22,7 +20,7 @@ export declare class Stage {
     readonly name: string;
     readonly stages: readonly Stage[];
     private readonly executors;
-    private readonly itemSelectorInput?;
+    private readonly itemSelector?;
     private readonly batchSize;
     private readonly maxConcurrency;
     constructor(options: StageOptions);
@@ -31,6 +29,7 @@ export declare class Stage {
     private executeAll;
 }
 /** Selects items (as variable bindings) for executors to process. Pagination is an implementation detail. */
-export interface ItemSelector extends AsyncIterable<VariableBindings> {
+export interface ItemSelector {
+    select(distribution: Distribution): AsyncIterable<VariableBindings>;
 }
 //# sourceMappingURL=stage.d.ts.map

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,6EAA6E;AAC7E,MAAM,MAAM,iBAAiB,GACzB,YAAY,GACZ,CAAC,CAAC,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC,CAAC;AAEnD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,iBAAiB,CAAC;IACjC,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAoB;IACvD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAW3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAuBjB,eAAe;YA+Gf,UAAU;CAqBzB;AAUD,6GAA6G;AAE7G,MAAM,WAAW,YAAa,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAW3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAmBjB,eAAe;YA+Gf,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}

package/dist/stage.js

@@ -5,7 +5,7 @@ export class Stage {
     name;
     stages;
     executors;
-    itemSelectorInput;
+    itemSelector;
     batchSize;
     maxConcurrency;
     constructor(options) {
@@ -14,16 +14,13 @@ export class Stage {
         this.executors = Array.isArray(options.executors)
             ? options.executors
             : [options.executors];
-        this.itemSelectorInput = options.itemSelector;
+        this.itemSelector = options.itemSelector;
         this.batchSize = options.batchSize ?? 10;
         this.maxConcurrency = options.maxConcurrency ?? 10;
     }
     async run(dataset, distribution, writer, options) {
-        if (this.itemSelectorInput) {
-            const selector = typeof this.itemSelectorInput === 'function'
-                ? this.itemSelectorInput(distribution)
-                : this.itemSelectorInput;
-            return this.runWithSelector(selector, dataset, distribution, writer, options);
+        if (this.itemSelector) {
+            return this.runWithSelector(this.itemSelector.select(distribution), dataset, distribution, writer, options);
         }
         const streams = await this.executeAll(dataset, distribution);
         if (streams instanceof NotSupported) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.27",
+  "version": "0.6.29",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"