@lde/pipeline 0.48.0 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pipeline.d.ts +7 -15
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +17 -9
- package/dist/sparql/reader.d.ts +4 -4
- package/dist/sparql/reader.d.ts.map +1 -1
- package/dist/sparql/reader.js +1 -3
- package/dist/stage.d.ts +4 -1
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +8 -2
- package/package.json +1 -1
package/dist/pipeline.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { Dataset } from '@lde/dataset';
|
|
2
|
-
import type { HostLimiter } from '@lde/host-limiter';
|
|
3
2
|
import type { Quad } from '@rdfjs/types';
|
|
4
3
|
import type { DatasetSelector } from './selector.js';
|
|
5
4
|
import { Stage } from './stage.js';
|
|
@@ -120,18 +119,6 @@ export interface PipelineOptions<Out = Quad> {
|
|
|
120
119
|
* once per dataset so state resets between datasets.
|
|
121
120
|
*/
|
|
122
121
|
timeout?: () => TimeoutPolicy;
|
|
123
|
-
/**
|
|
124
|
-
* The limiter every stage’s readers run their queries on, keyed on the
|
|
125
|
-
* endpoint ({@link endpointKeyOf}). Each endpoint’s limit adapts to how it
|
|
126
|
-
* responds, up to the limiter’s per-host ceiling, and carries over to the
|
|
127
|
-
* next dataset it serves; see {@link SparqlConstructReader}.
|
|
128
|
-
*
|
|
129
|
-
* Pass the limiter you also give a resolver that loads the same backend –
|
|
130
|
-
* keyed on the endpoint too – so the two share one limit.
|
|
131
|
-
*
|
|
132
|
-
* Left out, the readers share a default: `createHostLimiter(10, 10)`.
|
|
133
|
-
*/
|
|
134
|
-
limiter?: HostLimiter;
|
|
135
122
|
}
|
|
136
123
|
export declare class Pipeline<Out = Quad> {
|
|
137
124
|
private readonly name;
|
|
@@ -144,7 +131,13 @@ export declare class Pipeline<Out = Quad> {
|
|
|
144
131
|
private readonly chaining?;
|
|
145
132
|
private readonly reporters;
|
|
146
133
|
private readonly timeoutFactory;
|
|
147
|
-
|
|
134
|
+
/**
|
|
135
|
+
* The limiter every stage’s readers run their queries on, keyed on the
|
|
136
|
+
* endpoint, so each endpoint’s limit adapts to how it responds and carries
|
|
137
|
+
* over to the next dataset it serves; see {@link SparqlConstructReader}.
|
|
138
|
+
* Its ceiling is the highest `maxConcurrency` of any stage.
|
|
139
|
+
*/
|
|
140
|
+
private readonly limiter;
|
|
148
141
|
private readonly provenanceStore?;
|
|
149
142
|
private readonly pipelineVersion?;
|
|
150
143
|
constructor(options: PipelineOptions<Out>);
|
|
@@ -155,7 +148,6 @@ export declare class Pipeline<Out = Quad> {
|
|
|
155
148
|
/** Persist the processing record for a dataset, when a store is configured. */
|
|
156
149
|
private recordOutcome;
|
|
157
150
|
private reportValidators;
|
|
158
|
-
private collectStages;
|
|
159
151
|
/**
|
|
160
152
|
* The writer a stage's merged output is written through: the open run
|
|
161
153
|
* wrapped with the plugins' {@link PipelinePlugin.beforeStageWrite}
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAMrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAW7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AASpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAG9C,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,QAAQ,EAAE,CAAC;IAC1C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsB;IAChD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IAgEnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAyFZ,cAAc;IA8R5B,OAAO,CAAC,MAAM;IAyBd,OAAO,CAAC,iBAAiB;IAczB,+EAA+E;YACjE,aAAa;YAuBb,gBAAgB;IAiB9B;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA+BlC;;;;;;OAMG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA6EtB,2EAA2E;YAC7D,eAAe;YAuBf,QAAQ;YA+EP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createReadStream } from 'node:fs';
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { createHostLimiter, positiveIntOrDefault, } from '@lde/host-limiter';
|
|
3
4
|
import { StreamParser } from 'n3';
|
|
4
5
|
import { AsyncQueue } from './asyncQueue.js';
|
|
5
6
|
import { FileWriter } from './writer/fileWriter.js';
|
|
@@ -240,6 +241,12 @@ export class Pipeline {
|
|
|
240
241
|
chaining;
|
|
241
242
|
reporters;
|
|
242
243
|
timeoutFactory;
|
|
244
|
+
/**
|
|
245
|
+
* The limiter every stage’s readers run their queries on, keyed on the
|
|
246
|
+
* endpoint, so each endpoint’s limit adapts to how it responds and carries
|
|
247
|
+
* over to the next dataset it serves; see {@link SparqlConstructReader}.
|
|
248
|
+
* Its ceiling is the highest `maxConcurrency` of any stage.
|
|
249
|
+
*/
|
|
243
250
|
limiter;
|
|
244
251
|
provenanceStore;
|
|
245
252
|
pipelineVersion;
|
|
@@ -282,7 +289,8 @@ export class Pipeline {
|
|
|
282
289
|
this.reporters = [options.reporter ?? []].flat();
|
|
283
290
|
this.timeoutFactory =
|
|
284
291
|
options.timeout ?? (() => new ConstantTimeoutPolicy(300_000));
|
|
285
|
-
|
|
292
|
+
const ceiling = positiveIntOrDefault(Math.max(...[...allStages(options.stages)].map((stage) => stage.maxConcurrency)), 10);
|
|
293
|
+
this.limiter = createHostLimiter(ceiling, ceiling);
|
|
286
294
|
this.provenanceStore = options.provenanceStore;
|
|
287
295
|
this.pipelineVersion = options.pipelineVersion;
|
|
288
296
|
}
|
|
@@ -637,7 +645,7 @@ export class Pipeline {
|
|
|
637
645
|
}
|
|
638
646
|
async reportValidators(runId, dataset) {
|
|
639
647
|
const validators = new Set();
|
|
640
|
-
for (const stage of
|
|
648
|
+
for (const stage of allStages(this.stages)) {
|
|
641
649
|
if (stage.validator)
|
|
642
650
|
validators.add(stage.validator);
|
|
643
651
|
}
|
|
@@ -648,13 +656,6 @@ export class Pipeline {
|
|
|
648
656
|
}
|
|
649
657
|
// Walks every stage and sub-stage to collect validators; it reads only
|
|
650
658
|
// `validator`/`stages`, never output, so it is agnostic to the item type.
|
|
651
|
-
*collectStages(stages) {
|
|
652
|
-
for (const stage of stages) {
|
|
653
|
-
yield stage;
|
|
654
|
-
if (stage.stages.length > 0)
|
|
655
|
-
yield* this.collectStages(stage.stages);
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
659
|
/**
|
|
659
660
|
* The writer a stage's merged output is written through: the open run
|
|
660
661
|
* wrapped with the plugins' {@link PipelinePlugin.beforeStageWrite}
|
|
@@ -861,6 +862,13 @@ export class Pipeline {
|
|
|
861
862
|
}
|
|
862
863
|
}
|
|
863
864
|
}
|
|
865
|
+
/** Every stage, with the stages chained under it. */
|
|
866
|
+
function* allStages(stages) {
|
|
867
|
+
for (const stage of stages) {
|
|
868
|
+
yield stage;
|
|
869
|
+
yield* allStages(stage.stages);
|
|
870
|
+
}
|
|
871
|
+
}
|
|
864
872
|
function mapProbeResult(distribution, result) {
|
|
865
873
|
const fingerprint = sourceFingerprint(distribution, result);
|
|
866
874
|
if (result instanceof NetworkError) {
|
package/dist/sparql/reader.d.ts
CHANGED
|
@@ -40,12 +40,12 @@ export interface ReadOptions {
|
|
|
40
40
|
timeout?: TimeoutPolicy;
|
|
41
41
|
/**
|
|
42
42
|
* The limiter every query of a read goes through, keyed on the endpoint
|
|
43
|
-
* ({@link endpointKeyOf}), so every
|
|
44
|
-
*
|
|
45
|
-
* backend – shares one adaptive limit and one pause. See
|
|
43
|
+
* ({@link endpointKeyOf}), so every read on one endpoint shares one adaptive
|
|
44
|
+
* limit and one pause. A pipeline passes its own. See
|
|
46
45
|
* {@link SparqlConstructReader}.
|
|
47
46
|
*
|
|
48
|
-
*
|
|
47
|
+
* A read not given one gets a limiter for that read alone:
|
|
48
|
+
* `createHostLimiter(10, 10)`. A {@link Stage} passes one for all its batches.
|
|
49
49
|
*/
|
|
50
50
|
limiter?: HostLimiter;
|
|
51
51
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxC,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EAAyB,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxC,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EAAyB,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAU/E;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,YAAY;;CAIhD;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE9B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IAExB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;IAEtB;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAChE;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CACF,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,4BAA4B;IAgBjD;;;;;;;;OAQG;IACG,IAAI,CACR,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA+C/B;;;OAGG;YACW,SAAS;IAiCvB;;;OAGG;YACW,gBAAgB;IAwC9B;;;;;OAKG;YACW,oBAAoB;IAgClC;;;;;;;;OAQG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GACpD,OAAO,CAAC,qBAAqB,CAAC;CAIlC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI;IAUjE,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GACzB,aAAa,CAAC,IAAI,CAAC,CASrB"}
|
package/dist/sparql/reader.js
CHANGED
|
@@ -18,8 +18,6 @@ import { classifyOutcome, isTransientError, statusOf } from './failure.js';
|
|
|
18
18
|
* matters when the reader is driven directly (without a Pipeline).
|
|
19
19
|
*/
|
|
20
20
|
const defaultTimeoutPolicy = new ConstantTimeoutPolicy(300_000);
|
|
21
|
-
/** The limiter of every read not given one, so they share one limit. */
|
|
22
|
-
const defaultLimiter = createHostLimiter(10, 10);
|
|
23
21
|
/**
|
|
24
22
|
* A reader could not run because the dataset lacks a supported distribution.
|
|
25
23
|
*/
|
|
@@ -129,7 +127,7 @@ export class SparqlConstructReader {
|
|
|
129
127
|
const quads = await this.readBatch(bindings, query, {
|
|
130
128
|
endpoint,
|
|
131
129
|
key: endpointKeyOf(endpoint),
|
|
132
|
-
limiter: options?.limiter ??
|
|
130
|
+
limiter: options?.limiter ?? createHostLimiter(10, 10),
|
|
133
131
|
policy: options?.timeout ?? defaultTimeoutPolicy,
|
|
134
132
|
onRootSkipped: (root, error) => {
|
|
135
133
|
skipped++;
|
package/dist/stage.d.ts
CHANGED
|
@@ -201,10 +201,13 @@ export declare class Stage<Out = Quad> {
|
|
|
201
201
|
/** Whether this stage reads a source of its own rather than the dataset’s
|
|
202
202
|
* resolved distribution. @see {@link StageOptions.sourceFor} */
|
|
203
203
|
readonly sourcesOwnData: boolean;
|
|
204
|
+
/** Maximum concurrent in-flight SPARQL queries. @see {@link StageOptions.maxConcurrency} */
|
|
205
|
+
readonly maxConcurrency: number;
|
|
204
206
|
private readonly readers;
|
|
205
207
|
private readonly itemSelector?;
|
|
206
208
|
private readonly batchSize;
|
|
207
|
-
|
|
209
|
+
/** The limiter of a run not given one, so its batches back off together. */
|
|
210
|
+
private readonly limiter;
|
|
208
211
|
private readonly validation?;
|
|
209
212
|
private readonly project?;
|
|
210
213
|
private readonly queueCapacity?;
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAMrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGpE;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;CACvE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACtB,MAAM,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAUxD,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC;IAC3E,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAW,SAAQ,IAAI,CACtC,WAAW,EACX,SAAS,GAAG,eAAe,CAC5B;IACC,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB;iEAC6D;IAC7D,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C;oFACgF;IAChF,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC9C;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK,CAAC,GAAG,GAAG,IAAI;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC;qEACiE;IACjE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,4FAA4F;IAC5F,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAkC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiC;gBAEhD,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;IAqCtC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IA4G/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IA0L7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA0BrB;;;;;OAKG;YACW,IAAI;IA2BlB,8EAA8E;IAC9E,OAAO,KAAK,cAAc,GAOzB;CACF;AA0DD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
|
package/dist/stage.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHostLimiter, positiveIntOrDefault, } from '@lde/host-limiter';
|
|
1
2
|
import { NoItemsSelected, NotSupported } from './sparql/reader.js';
|
|
2
3
|
import { batch } from './batch.js';
|
|
3
4
|
import { AsyncQueue } from './asyncQueue.js';
|
|
@@ -10,10 +11,13 @@ export class Stage {
|
|
|
10
11
|
/** Whether this stage reads a source of its own rather than the dataset’s
|
|
11
12
|
* resolved distribution. @see {@link StageOptions.sourceFor} */
|
|
12
13
|
sourcesOwnData;
|
|
14
|
+
/** Maximum concurrent in-flight SPARQL queries. @see {@link StageOptions.maxConcurrency} */
|
|
15
|
+
maxConcurrency;
|
|
13
16
|
readers;
|
|
14
17
|
itemSelector;
|
|
15
18
|
batchSize;
|
|
16
|
-
|
|
19
|
+
/** The limiter of a run not given one, so its batches back off together. */
|
|
20
|
+
limiter;
|
|
17
21
|
validation;
|
|
18
22
|
project;
|
|
19
23
|
queueCapacity;
|
|
@@ -39,6 +43,8 @@ export class Stage {
|
|
|
39
43
|
this.itemSelector = options.itemSelector;
|
|
40
44
|
this.batchSize = options.batchSize ?? 10;
|
|
41
45
|
this.maxConcurrency = options.maxConcurrency ?? 10;
|
|
46
|
+
const ceiling = positiveIntOrDefault(this.maxConcurrency, 10);
|
|
47
|
+
this.limiter = createHostLimiter(ceiling, ceiling);
|
|
42
48
|
this.validation = options.validation;
|
|
43
49
|
this.expectsOutput = options.expectsOutput ?? false;
|
|
44
50
|
this.project = options.project;
|
|
@@ -54,7 +60,7 @@ export class Stage {
|
|
|
54
60
|
const timeout = options?.timeout;
|
|
55
61
|
const readOptions = {
|
|
56
62
|
timeout,
|
|
57
|
-
limiter: options?.limiter,
|
|
63
|
+
limiter: options?.limiter ?? this.limiter,
|
|
58
64
|
onRootSkipped: options?.onRootSkipped,
|
|
59
65
|
};
|
|
60
66
|
// Substituted once, before anything reads: the selector and the readers
|