@lde/pipeline 0.48.0 → 0.50.0

package/dist/pipeline.d.ts

@@ -1,5 +1,4 @@
 import { Dataset } from '@lde/dataset';
-import type { HostLimiter } from '@lde/host-limiter';
 import type { Quad } from '@rdfjs/types';
 import type { DatasetSelector } from './selector.js';
 import { Stage } from './stage.js';
@@ -120,18 +119,6 @@ export interface PipelineOptions<Out = Quad> {
      * once per dataset so state resets between datasets.
      */
     timeout?: () => TimeoutPolicy;
-    /**
-     * The limiter every stage’s readers run their queries on, keyed on the
-     * endpoint ({@link endpointKeyOf}). Each endpoint’s limit adapts to how it
-     * responds, up to the limiter’s per-host ceiling, and carries over to the
-     * next dataset it serves; see {@link SparqlConstructReader}.
-     *
-     * Pass the limiter you also give a resolver that loads the same backend –
-     * keyed on the endpoint too – so the two share one limit.
-     *
-     * Left out, the readers share a default: `createHostLimiter(10, 10)`.
-     */
-    limiter?: HostLimiter;
 }
 export declare class Pipeline<Out = Quad> {
     private readonly name;
@@ -144,7 +131,13 @@ export declare class Pipeline<Out = Quad> {
     private readonly chaining?;
     private readonly reporters;
     private readonly timeoutFactory;
-    private readonly limiter?;
+    /**
+     * The limiter every stage’s readers run their queries on, keyed on the
+     * endpoint, so each endpoint’s limit adapts to how it responds and carries
+     * over to the next dataset it serves; see {@link SparqlConstructReader}.
+     * Its ceiling is the highest `maxConcurrency` of any stage.
+     */
+    private readonly limiter;
     private readonly provenanceStore?;
     private readonly pipelineVersion?;
     constructor(options: PipelineOptions<Out>);
@@ -155,7 +148,6 @@ export declare class Pipeline<Out = Quad> {
     /** Persist the processing record for a dataset, when a store is configured. */
     private recordOutcome;
     private reportValidators;
-    private collectStages;
     /**
      * The writer a stage's merged output is written through: the open run
      * wrapped with the plugins' {@link PipelinePlugin.beforeStageWrite}

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAW7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AASpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAG9C,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,QAAQ,EAAE,CAAC;IAC1C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;IAC9B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsB;IAChD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAc;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA0DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAyFZ,cAAc;IA8R5B,OAAO,CAAC,MAAM;IAyBd,OAAO,CAAC,iBAAiB;IAczB,+EAA+E;YACjE,aAAa;YAuBb,gBAAgB;IAgB9B,OAAO,CAAE,aAAa;IAStB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA+BlC;;;;;;OAMG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA6EtB,2EAA2E;YAC7D,eAAe;YAuBf,QAAQ;YA+EP,SAAS;CAczB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAMrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAE5D,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAW7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AASpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAG9C,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,QAAQ,EAAE,CAAC;IAC1C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsB;IAChD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IAgEnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAyFZ,cAAc;IAgS5B,OAAO,CAAC,MAAM;IAyBd,OAAO,CAAC,iBAAiB;IAczB,+EAA+E;YACjE,aAAa;YAuBb,gBAAgB;IAiB9B;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA+BlC;;;;;;OAMG;YACW,SAAS;IA0CvB;;;OAGG;YACW,QAAQ;IA+EtB,2EAA2E;YAC7D,eAAe;YAyBf,QAAQ;YAsFP,SAAS;CAczB"}

package/dist/pipeline.js

@@ -1,5 +1,6 @@
 import { createReadStream } from 'node:fs';
 import { randomUUID } from 'node:crypto';
+import { createHostLimiter, positiveIntOrDefault, } from '@lde/host-limiter';
 import { StreamParser } from 'n3';
 import { AsyncQueue } from './asyncQueue.js';
 import { FileWriter } from './writer/fileWriter.js';
@@ -240,6 +241,12 @@ export class Pipeline {
     chaining;
     reporters;
     timeoutFactory;
+    /**
+     * The limiter every stage’s readers run their queries on, keyed on the
+     * endpoint, so each endpoint’s limit adapts to how it responds and carries
+     * over to the next dataset it serves; see {@link SparqlConstructReader}.
+     * Its ceiling is the highest `maxConcurrency` of any stage.
+     */
     limiter;
     provenanceStore;
     pipelineVersion;
@@ -282,7 +289,8 @@ export class Pipeline {
         this.reporters = [options.reporter ?? []].flat();
         this.timeoutFactory =
             options.timeout ?? (() => new ConstantTimeoutPolicy(300_000));
-        this.limiter = options.limiter;
+        const ceiling = positiveIntOrDefault(Math.max(...[...allStages(options.stages)].map((stage) => stage.maxConcurrency)), 10);
+        this.limiter = createHostLimiter(ceiling, ceiling);
         this.provenanceStore = options.provenanceStore;
         this.pipelineVersion = options.pipelineVersion;
     }
@@ -491,7 +499,7 @@ export class Pipeline {
         // completes.
         let unread;
         try {
-            ({ stageFailed, unread } = await this.runStages(dataset, resolved.distribution, timeout, runWriter, context));
+            ({ stageFailed, unread } = await this.runStages(dataset, resolved.distribution, sourceKindOf(resolved), timeout, runWriter, context));
             // Reactive fallback: an endpoint that passed probing but could not serve
             // the analysis stages is empirically incapable. Switch to the dataset’s
             // data dump and re-run all stages locally, discarding the
@@ -524,7 +532,7 @@ export class Pipeline {
                         // Discard the endpoint-sourced partial output before the re-run so
                         // the dump-sourced stats replace it rather than appending to it.
                         await runWriter.reset?.(dataset);
-                        ({ stageFailed, unread } = await this.runStages(dataset, fallback.distribution, timeout, runWriter, context));
+                        ({ stageFailed, unread } = await this.runStages(dataset, fallback.distribution, sourceKindOf(fallback), timeout, runWriter, context));
                         // Adopt the dump’s fingerprint only once the re-run has succeeded.
                         // If the reset or the re-run throws, or a stage fails against the
                         // dump, the dataset is recorded ‘failed’ under the endpoint’s
@@ -637,7 +645,7 @@ export class Pipeline {
     }
     async reportValidators(runId, dataset) {
         const validators = new Set();
-        for (const stage of this.collectStages(this.stages)) {
+        for (const stage of allStages(this.stages)) {
             if (stage.validator)
                 validators.add(stage.validator);
         }
@@ -648,13 +656,6 @@ export class Pipeline {
     }
     // Walks every stage and sub-stage to collect validators; it reads only
     // `validator`/`stages`, never output, so it is agnostic to the item type.
-    *collectStages(stages) {
-        for (const stage of stages) {
-            yield stage;
-            if (stage.stages.length > 0)
-                yield* this.collectStages(stage.stages);
-        }
-    }
     /**
      * The writer a stage's merged output is written through: the open run
      * wrapped with the plugins' {@link PipelinePlugin.beforeStageWrite}
@@ -706,16 +707,16 @@ export class Pipeline {
      * reacts to – and what the stages could not read, which belongs to this
      * attempt alone.
      */
-    async runStages(dataset, distribution, timeout, runWriter, context) {
+    async runStages(dataset, distribution, source, timeout, runWriter, context) {
         const unread = new UnreadValueTally();
         let stageFailed = false;
         for (const stage of this.stages) {
             try {
                 if (stage.stages.length > 0) {
-                    await this.runChain(dataset, distribution, stage, runWriter, context, timeout);
+                    await this.runChain(dataset, distribution, source, stage, runWriter, context, timeout);
                 }
                 else {
-                    await this.runStage(context.runId, dataset, distribution, stage, this.stageWriter(runWriter, stage.name), timeout, unread);
+                    await this.runStage(context.runId, dataset, distribution, stage, this.stageWriter(runWriter, stage.name), source, timeout, unread);
                 }
             }
             catch (error) {
@@ -729,13 +730,14 @@ export class Pipeline {
      * Run a stage with reporting and return whether it was supported.
      * Returns `true` if the stage produced results, `false` if NotSupported.
      */
-    async runStage(runId, dataset, distribution, stage, writer, timeout, unread) {
+    async runStage(runId, dataset, distribution, stage, writer, source, timeout, unread) {
         this.report(runId, { type: 'stageStart', dataset, stage: stage.name });
         const stageStart = Date.now();
         let itemsProcessed = 0;
         let quadsGenerated = 0;
         let summary;
         const result = await stage.run(dataset, distribution, writer, {
+            source,
             onProgress: (items, quads) => {
                 itemsProcessed = items;
                 quadsGenerated = quads;
@@ -790,13 +792,13 @@ export class Pipeline {
         return true;
     }
     /** Run a stage in chained mode, throwing if the stage is not supported. */
-    async runChainedStage(runId, dataset, distribution, stage, writer, timeout) {
-        const supported = await this.runStage(runId, dataset, distribution, stage, writer, timeout);
+    async runChainedStage(runId, dataset, distribution, stage, writer, source, timeout) {
+        const supported = await this.runStage(runId, dataset, distribution, stage, writer, source, timeout);
         if (!supported) {
             throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
         }
     }
-    async runChain(dataset, distribution, stage, runWriter, context, timeout) {
+    async runChain(dataset, distribution, source, stage, runWriter, context, timeout) {
         const { stageOutputResolver, outputDir } = this.chaining;
         const outputFiles = [];
         // Chaining is a quad-only concern: every stage in a chain writes N-Triples
@@ -805,14 +807,14 @@ export class Pipeline {
         // requirement "sub-stages imply `Out` is `Quad`" is not expressible – `stages`
         // and `chaining` are independent options – so it is a runtime contract, and
         // the scratch machinery treats the parent stage as `Stage<Quad>`.
-        const runScratchStage = async (chainedStage, stageDistribution) => {
+        const runScratchStage = async (chainedStage, stageDistribution, stageSource) => {
             const scratchWriter = new FileWriter({
                 outputDir: `${outputDir}/${chainedStage.name}`,
                 format: 'n-triples',
             });
             const scratchRun = await scratchWriter.openRun(context);
             try {
-                await this.runChainedStage(context.runId, dataset, stageDistribution, chainedStage, scratchRun, timeout);
+                await this.runChainedStage(context.runId, dataset, stageDistribution, chainedStage, scratchRun, stageSource, timeout);
                 await scratchRun.flush(dataset, 'success');
                 await scratchRun.commit();
             }
@@ -824,12 +826,14 @@ export class Pipeline {
         };
         try {
             // 1. Run parent stage → scratch FileWriter (quad-only, per above).
-            const parentOutput = await runScratchStage(stage, distribution);
+            const parentOutput = await runScratchStage(stage, distribution, source);
             outputFiles.push(parentOutput);
             // 2. Chain through children.
             let currentDistribution = await stageOutputResolver.resolve(parentOutput);
             for (let i = 0; i < stage.stages.length; i++) {
-                const childOutput = await runScratchStage(stage.stages[i], currentDistribution);
+                // A child reads its parent’s output, which the pipeline serves from
+                // a store of its own, as it does an imported dump.
+                const childOutput = await runScratchStage(stage.stages[i], currentDistribution, 'imported');
                 outputFiles.push(childOutput);
                 if (i < stage.stages.length - 1) {
                     currentDistribution = await stageOutputResolver.resolve(childOutput);
@@ -861,6 +865,13 @@ export class Pipeline {
         }
     }
 }
+/** Every stage, with the stages chained under it. */
+function* allStages(stages) {
+    for (const stage of stages) {
+        yield stage;
+        yield* allStages(stage.stages);
+    }
+}
 function mapProbeResult(distribution, result) {
     const fingerprint = sourceFingerprint(distribution, result);
     if (result instanceof NetworkError) {
@@ -883,3 +894,7 @@ function mapProbeResult(distribution, result) {
         fingerprint,
     };
 }
+/** What kind of source a resolved distribution serves the stages. */
+function sourceKindOf(resolved) {
+    return resolved.imported === undefined ? 'endpoint' : 'imported';
+}

package/dist/sparql/reader.d.ts

@@ -40,12 +40,12 @@ export interface ReadOptions {
     timeout?: TimeoutPolicy;
     /**
      * The limiter every query of a read goes through, keyed on the endpoint
-     * ({@link endpointKeyOf}), so every dataset an endpoint serves – and
-     * anything else given that key, such as a resolver fetching from the same
-     * backend – shares one adaptive limit and one pause. See
+     * ({@link endpointKeyOf}), so every read on one endpoint shares one adaptive
+     * limit and one pause. A pipeline passes its own. See
      * {@link SparqlConstructReader}.
      *
-     * Reads not given one share a default: `createHostLimiter(10, 10)`.
+     * A read not given one gets a limiter for that read alone:
+     * `createHostLimiter(10, 10)`. A {@link Stage} passes one for all its batches.
      */
     limiter?: HostLimiter;
     /**

package/dist/sparql/reader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxC,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EAAyB,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAa/E;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,YAAY;;CAIhD;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE9B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IAExB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;IAEtB;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAChE;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CACF,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,4BAA4B;IAgBjD;;;;;;;;OAQG;IACG,IAAI,CACR,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA+C/B;;;OAGG;YACW,SAAS;IAiCvB;;;OAGG;YACW,gBAAgB;IAwC9B;;;;;OAKG;YACW,oBAAoB;IAgClC;;;;;;;;OAQG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GACpD,OAAO,CAAC,qBAAqB,CAAC;CAIlC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI;IAUjE,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GACzB,aAAa,CAAC,IAAI,CAAC,CASrB"}
+{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxC,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EAAyB,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAU/E;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,YAAY;;CAIhD;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE9B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IAExB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;IAEtB;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAChE;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CACF,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,4BAA4B;IAgBjD;;;;;;;;OAQG;IACG,IAAI,CACR,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA+C/B;;;OAGG;YACW,SAAS;IAiCvB;;;OAGG;YACW,gBAAgB;IAwC9B;;;;;OAKG;YACW,oBAAoB;IAgClC;;;;;;;;OAQG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GACpD,OAAO,CAAC,qBAAqB,CAAC;CAIlC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI;IAUjE,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GACzB,aAAa,CAAC,IAAI,CAAC,CASrB"}

package/dist/sparql/reader.js

@@ -18,8 +18,6 @@ import { classifyOutcome, isTransientError, statusOf } from './failure.js';
  * matters when the reader is driven directly (without a Pipeline).
  */
 const defaultTimeoutPolicy = new ConstantTimeoutPolicy(300_000);
-/** The limiter of every read not given one, so they share one limit. */
-const defaultLimiter = createHostLimiter(10, 10);
 /**
  * A reader could not run because the dataset lacks a supported distribution.
  */
@@ -129,7 +127,7 @@ export class SparqlConstructReader {
         const quads = await this.readBatch(bindings, query, {
             endpoint,
             key: endpointKeyOf(endpoint),
-            limiter: options?.limiter ?? defaultLimiter,
+            limiter: options?.limiter ?? createHostLimiter(10, 10),
             policy: options?.timeout ?? defaultTimeoutPolicy,
             onRootSkipped: (root, error) => {
                 skipped++;

package/dist/stage.d.ts

@@ -105,9 +105,19 @@ export interface StageOptions<Out = Quad> {
      * fills exactly one batch. A `LIMIT` clause in the selector query
      * overrides this for endpoints with hard result limits.
      *
+     * Give a value per {@link SourceKind} where the two kinds of source want
+     * different sizes: an engine that imported a dump for this run alone
+     * answers a large batch easily, while a live endpoint shared with others
+     * may fail on one. The pipeline tells the stage which kind it reads
+     * ({@link RunOptions.source}).
+     *
+     * ```ts
+     * batchSize: { endpoint: 10, imported: 100 }
+     * ```
+     *
      * @default 10
      */
-    batchSize?: number;
+    batchSize?: number | Readonly<Record<SourceKind, number>>;
     /**
      * Maximum concurrent in-flight SPARQL queries. Within each batch, all
      * readers run in parallel; the number of concurrent batches is
@@ -172,8 +182,23 @@ export interface StageOptions<Out = Quad> {
         onInvalid?: 'write' | 'skip' | 'halt';
     };
 }
+/**
+ * What a stage reads: a live SPARQL `endpoint` the dataset publishes, or a
+ * data dump the pipeline `imported` into a store of its own for the run.
+ */
+export type SourceKind = 'endpoint' | 'imported';
 export interface RunOptions extends Pick<ReadOptions, 'limiter' | 'onRootSkipped'> {
     onProgress?: (itemsProcessed: number, quadsGenerated: number) => void;
+    /**
+     * What the distribution handed to {@link Stage.run} serves, which picks the
+     * {@link StageOptions.batchSize} of a stage that gives one per kind. A stage
+     * whose {@link StageOptions.sourceFor} substitutes another distribution reads
+     * a source the pipeline did not resolve, so it reads that as an `endpoint`
+     * whatever this says.
+     *
+     * @default 'endpoint'
+     */
+    source?: SourceKind;
     /**
      * Per-dataset {@link TimeoutPolicy} threaded through to readers and
      * item selectors. The Pipeline owns lifecycle (factory invocation per
@@ -201,10 +226,13 @@ export declare class Stage<Out = Quad> {
     /** Whether this stage reads a source of its own rather than the dataset’s
      *  resolved distribution. @see {@link StageOptions.sourceFor} */
     readonly sourcesOwnData: boolean;
+    /** Maximum concurrent in-flight SPARQL queries. @see {@link StageOptions.maxConcurrency} */
+    readonly maxConcurrency: number;
     private readonly readers;
     private readonly itemSelector?;
     private readonly batchSize;
-    private readonly maxConcurrency;
+    /** The limiter of a run not given one, so its batches back off together. */
+    private readonly limiter;
     private readonly validation?;
     private readonly project?;
     private readonly queueCapacity?;

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGpE;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;CACvE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACtB,MAAM,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAUxD,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC;IAC3E,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAW,SAAQ,IAAI,CACtC,WAAW,EACX,SAAS,GAAG,eAAe,CAC5B;IACC,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB;iEAC6D;IAC7D,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C;oFACgF;IAChF,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC9C;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK,CAAC,GAAG,GAAG,IAAI;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC;qEACiE;IACjE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAkC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiC;gBAEhD,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;IAmCtC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IA4G/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IA0L7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA0BrB;;;;;OAKG;YACW,IAAI;IA2BlB,8EAA8E;IAC9E,OAAO,KAAK,cAAc,GAOzB;CACF;AA0DD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAMrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGpE;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;CACvE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACtB,MAAM,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAUxD,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;;;;;;;OAkBG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1D;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC;IAC3E,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAEjD,MAAM,WAAW,UAAW,SAAQ,IAAI,CACtC,WAAW,EACX,SAAS,GAAG,eAAe,CAC5B;IACC,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;;;;OAQG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB;iEAC6D;IAC7D,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C;oFACgF;IAChF,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC9C;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK,CAAC,GAAG,GAAG,IAAI;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC;qEACiE;IACjE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,4FAA4F;IAC5F,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgD;IAC1E,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAkC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiC;gBAEhD,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;IAqCtC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAuH/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IA2L7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA0BrB;;;;;OAKG;YACW,IAAI;IA2BlB,8EAA8E;IAC9E,OAAO,KAAK,cAAc,GAOzB;CACF;AA0DD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}

package/dist/stage.js

@@ -1,3 +1,4 @@
+import { createHostLimiter, positiveIntOrDefault, } from '@lde/host-limiter';
 import { NoItemsSelected, NotSupported } from './sparql/reader.js';
 import { batch } from './batch.js';
 import { AsyncQueue } from './asyncQueue.js';
@@ -10,10 +11,13 @@ export class Stage {
     /** Whether this stage reads a source of its own rather than the dataset’s
      *  resolved distribution. @see {@link StageOptions.sourceFor} */
     sourcesOwnData;
+    /** Maximum concurrent in-flight SPARQL queries. @see {@link StageOptions.maxConcurrency} */
+    maxConcurrency;
     readers;
     itemSelector;
     batchSize;
-    maxConcurrency;
+    /** The limiter of a run not given one, so its batches back off together. */
+    limiter;
     validation;
     project;
     queueCapacity;
@@ -39,6 +43,8 @@ export class Stage {
         this.itemSelector = options.itemSelector;
         this.batchSize = options.batchSize ?? 10;
         this.maxConcurrency = options.maxConcurrency ?? 10;
+        const ceiling = positiveIntOrDefault(this.maxConcurrency, 10);
+        this.limiter = createHostLimiter(ceiling, ceiling);
         this.validation = options.validation;
         this.expectsOutput = options.expectsOutput ?? false;
         this.project = options.project;
@@ -54,7 +60,7 @@ export class Stage {
         const timeout = options?.timeout;
         const readOptions = {
             timeout,
-            limiter: options?.limiter,
+            limiter: options?.limiter ?? this.limiter,
             onRootSkipped: options?.onRootSkipped,
         };
         // Substituted once, before anything reads: the selector and the readers
@@ -64,9 +70,17 @@ export class Stage {
             : resolved;
         const tally = new StageTally(this.transformNames);
         if (this.itemSelector) {
-            const result = await this.runWithSelector(tally.selected(this.itemSelector.select(distribution, this.batchSize, {
+            // A substituted source is one the pipeline did not resolve; one handed
+            // back unchanged is still the source the pipeline resolved.
+            const source = distribution === resolved
+                ? (options?.source ?? 'endpoint')
+                : 'endpoint';
+            const batchSize = typeof this.batchSize === 'number'
+                ? this.batchSize
+                : this.batchSize[source];
+            const result = await this.runWithSelector(tally.selected(this.itemSelector.select(distribution, batchSize, {
                 timeout,
-            })), dataset, distribution, writer, tally, readOptions, options);
+            })), batchSize, dataset, distribution, writer, tally, readOptions, options);
             if (!(result instanceof NotSupported)) {
                 options?.onComplete?.(tally.summary());
             }
@@ -138,10 +152,10 @@ export class Stage {
             throw new Error(`Stage '${this.name}' expected output but produced none`);
         }
     }
-    async runWithSelector(selector, dataset, distribution, writer, tally, readOptions, options) {
+    async runWithSelector(selector, batchSize, dataset, distribution, writer, tally, readOptions, options) {
         // Peek the first batch to detect an empty selector before starting the
         // writer (important because e.g. SparqlUpdateWriter does CLEAR GRAPH).
-        const batches = batch(selector, this.batchSize);
+        const batches = batch(selector, batchSize);
         const iter = batches[Symbol.asyncIterator]();
         const first = await iter.next();
         if (first.done) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.48.0",
+  "version": "0.50.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"