@lde/pipeline 0.47.0 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pipeline.d.ts +7 -14
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +17 -9
- package/dist/sparql/reader.d.ts +11 -10
- package/dist/sparql/reader.d.ts.map +1 -1
- package/dist/sparql/reader.js +9 -11
- package/dist/stage.d.ts +4 -1
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +8 -2
- package/package.json +5 -5
package/dist/pipeline.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { Dataset } from '@lde/dataset';
|
|
2
|
-
import type { HostLimiter } from '@lde/host-limiter';
|
|
3
2
|
import type { Quad } from '@rdfjs/types';
|
|
4
3
|
import type { DatasetSelector } from './selector.js';
|
|
5
4
|
import { Stage } from './stage.js';
|
|
@@ -120,17 +119,6 @@ export interface PipelineOptions<Out = Quad> {
|
|
|
120
119
|
* once per dataset so state resets between datasets.
|
|
121
120
|
*/
|
|
122
121
|
timeout?: () => TimeoutPolicy;
|
|
123
|
-
/**
|
|
124
|
-
* The limiter every stage’s readers run their queries on, keyed on the
|
|
125
|
-
* dataset’s IRI. Each dataset’s limit adapts to how its endpoint responds,
|
|
126
|
-
* up to the limiter’s per-host ceiling; see {@link SparqlConstructReader}.
|
|
127
|
-
*
|
|
128
|
-
* Pass the limiter you also give a resolver that loads the same backend –
|
|
129
|
-
* keyed on the dataset too – so the two share one limit.
|
|
130
|
-
*
|
|
131
|
-
* Left out, the readers share a default: `createHostLimiter(10, 10)`.
|
|
132
|
-
*/
|
|
133
|
-
limiter?: HostLimiter;
|
|
134
122
|
}
|
|
135
123
|
export declare class Pipeline<Out = Quad> {
|
|
136
124
|
private readonly name;
|
|
@@ -143,7 +131,13 @@ export declare class Pipeline<Out = Quad> {
|
|
|
143
131
|
private readonly chaining?;
|
|
144
132
|
private readonly reporters;
|
|
145
133
|
private readonly timeoutFactory;
|
|
146
|
-
|
|
134
|
+
/**
|
|
135
|
+
* The limiter every stage’s readers run their queries on, keyed on the
|
|
136
|
+
* endpoint, so each endpoint’s limit adapts to how it responds and carries
|
|
137
|
+
* over to the next dataset it serves; see {@link SparqlConstructReader}.
|
|
138
|
+
* Its ceiling is the highest `maxConcurrency` of any stage.
|
|
139
|
+
*/
|
|
140
|
+
private readonly limiter;
|
|
147
141
|
private readonly provenanceStore?;
|
|
148
142
|
private readonly pipelineVersion?;
|
|
149
143
|
constructor(options: PipelineOptions<Out>);
|
|
@@ -154,7 +148,6 @@ export declare class Pipeline<Out = Quad> {
|
|
|
154
148
|
/** Persist the processing record for a dataset, when a store is configured. */
|
|
155
149
|
private recordOutcome;
|
|
156
150
|
private reportValidators;
|
|
157
|
-
private collectStages;
|
|
158
151
|
/**
|
|
159
152
|
* The writer a stage's merged output is written through: the open run
|
|
160
153
|
* wrapped with the plugins' {@link PipelinePlugin.beforeStageWrite}
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAMrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAW7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AASpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAG9C,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,QAAQ,EAAE,CAAC;IAC1C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsB;IAChD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IAgEnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAyFZ,cAAc;IA8R5B,OAAO,CAAC,MAAM;IAyBd,OAAO,CAAC,iBAAiB;IAczB,+EAA+E;YACjE,aAAa;YAuBb,gBAAgB;IAiB9B;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA+BlC;;;;;;OAMG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA6EtB,2EAA2E;YAC7D,eAAe;YAuBf,QAAQ;YA+EP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createReadStream } from 'node:fs';
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { createHostLimiter, positiveIntOrDefault, } from '@lde/host-limiter';
|
|
3
4
|
import { StreamParser } from 'n3';
|
|
4
5
|
import { AsyncQueue } from './asyncQueue.js';
|
|
5
6
|
import { FileWriter } from './writer/fileWriter.js';
|
|
@@ -240,6 +241,12 @@ export class Pipeline {
|
|
|
240
241
|
chaining;
|
|
241
242
|
reporters;
|
|
242
243
|
timeoutFactory;
|
|
244
|
+
/**
|
|
245
|
+
* The limiter every stage’s readers run their queries on, keyed on the
|
|
246
|
+
* endpoint, so each endpoint’s limit adapts to how it responds and carries
|
|
247
|
+
* over to the next dataset it serves; see {@link SparqlConstructReader}.
|
|
248
|
+
* Its ceiling is the highest `maxConcurrency` of any stage.
|
|
249
|
+
*/
|
|
243
250
|
limiter;
|
|
244
251
|
provenanceStore;
|
|
245
252
|
pipelineVersion;
|
|
@@ -282,7 +289,8 @@ export class Pipeline {
|
|
|
282
289
|
this.reporters = [options.reporter ?? []].flat();
|
|
283
290
|
this.timeoutFactory =
|
|
284
291
|
options.timeout ?? (() => new ConstantTimeoutPolicy(300_000));
|
|
285
|
-
|
|
292
|
+
const ceiling = positiveIntOrDefault(Math.max(...[...allStages(options.stages)].map((stage) => stage.maxConcurrency)), 10);
|
|
293
|
+
this.limiter = createHostLimiter(ceiling, ceiling);
|
|
286
294
|
this.provenanceStore = options.provenanceStore;
|
|
287
295
|
this.pipelineVersion = options.pipelineVersion;
|
|
288
296
|
}
|
|
@@ -637,7 +645,7 @@ export class Pipeline {
|
|
|
637
645
|
}
|
|
638
646
|
async reportValidators(runId, dataset) {
|
|
639
647
|
const validators = new Set();
|
|
640
|
-
for (const stage of
|
|
648
|
+
for (const stage of allStages(this.stages)) {
|
|
641
649
|
if (stage.validator)
|
|
642
650
|
validators.add(stage.validator);
|
|
643
651
|
}
|
|
@@ -648,13 +656,6 @@ export class Pipeline {
|
|
|
648
656
|
}
|
|
649
657
|
// Walks every stage and sub-stage to collect validators; it reads only
|
|
650
658
|
// `validator`/`stages`, never output, so it is agnostic to the item type.
|
|
651
|
-
*collectStages(stages) {
|
|
652
|
-
for (const stage of stages) {
|
|
653
|
-
yield stage;
|
|
654
|
-
if (stage.stages.length > 0)
|
|
655
|
-
yield* this.collectStages(stage.stages);
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
659
|
/**
|
|
659
660
|
* The writer a stage's merged output is written through: the open run
|
|
660
661
|
* wrapped with the plugins' {@link PipelinePlugin.beforeStageWrite}
|
|
@@ -861,6 +862,13 @@ export class Pipeline {
|
|
|
861
862
|
}
|
|
862
863
|
}
|
|
863
864
|
}
|
|
865
|
+
/** Every stage, with the stages chained under it. */
|
|
866
|
+
function* allStages(stages) {
|
|
867
|
+
for (const stage of stages) {
|
|
868
|
+
yield stage;
|
|
869
|
+
yield* allStages(stage.stages);
|
|
870
|
+
}
|
|
871
|
+
}
|
|
864
872
|
function mapProbeResult(distribution, result) {
|
|
865
873
|
const fingerprint = sourceFingerprint(distribution, result);
|
|
866
874
|
if (result instanceof NetworkError) {
|
package/dist/sparql/reader.d.ts
CHANGED
|
@@ -39,12 +39,13 @@ export interface ReadOptions {
|
|
|
39
39
|
*/
|
|
40
40
|
timeout?: TimeoutPolicy;
|
|
41
41
|
/**
|
|
42
|
-
* The limiter every query of a read goes through, keyed on the
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
42
|
+
* The limiter every query of a read goes through, keyed on the endpoint
|
|
43
|
+
* ({@link endpointKeyOf}), so every read on one endpoint shares one adaptive
|
|
44
|
+
* limit and one pause. A pipeline passes its own. See
|
|
45
|
+
* {@link SparqlConstructReader}.
|
|
46
46
|
*
|
|
47
|
-
*
|
|
47
|
+
* A read not given one gets a limiter for that read alone:
|
|
48
|
+
* `createHostLimiter(10, 10)`. A {@link Stage} passes one for all its batches.
|
|
48
49
|
*/
|
|
49
50
|
limiter?: HostLimiter;
|
|
50
51
|
/**
|
|
@@ -114,9 +115,9 @@ export interface SparqlConstructReaderOptions {
|
|
|
114
115
|
/**
|
|
115
116
|
* A streaming SPARQL CONSTRUCT reader.
|
|
116
117
|
*
|
|
117
|
-
* Each query runs on the {@link ReadOptions.limiter}, keyed on the
|
|
118
|
-
*
|
|
119
|
-
* reader pauses
|
|
118
|
+
* Each query runs on the {@link ReadOptions.limiter}, keyed on the endpoint
|
|
119
|
+
* ({@link endpointKeyOf}). While the endpoint is down (502, 503, 504 or a
|
|
120
|
+
* network error) the reader pauses it and tries again; on a 500 it splits the batch,
|
|
120
121
|
* down to a root it reports through {@link ReadOptions.onRootSkipped} and
|
|
121
122
|
* leaves out. See
|
|
122
123
|
* {@link https://ldelements.org/reference/pipeline#when-the-endpoint-fails | When the endpoint fails}.
|
|
@@ -169,8 +170,8 @@ export declare class SparqlConstructReader implements Reader {
|
|
|
169
170
|
*/
|
|
170
171
|
private readBatch;
|
|
171
172
|
/**
|
|
172
|
-
* Run a query on the limiter, pausing the
|
|
173
|
-
*
|
|
173
|
+
* Run a query on the limiter, pausing the endpoint and trying again while it is
|
|
174
|
+
* down; see {@link SparqlConstructReader}.
|
|
174
175
|
*/
|
|
175
176
|
private fetchWithRetries;
|
|
176
177
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxC,OAAO,
|
|
1
|
+
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxC,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EAAyB,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAU/E;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,YAAY;;CAIhD;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE9B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IAExB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;IAEtB;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAChE;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CACF,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,4BAA4B;IAgBjD;;;;;;;;OAQG;IACG,IAAI,CACR,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA+C/B;;;OAGG;YACW,SAAS;IAiCvB;;;OAGG;YACW,gBAAgB;IAwC9B;;;;;OAKG;YACW,oBAAoB;IAgClC;;;;;;;;OAQG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GACpD,OAAO,CAAC,qBAAqB,CAAC;CAIlC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI;IAUjE,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GACzB,aAAa,CAAC,IAAI,CAAC,CASrB"}
|
package/dist/sparql/reader.js
CHANGED
|
@@ -6,7 +6,7 @@ import { Transform } from 'node:stream';
|
|
|
6
6
|
import { StreamParser } from 'n3';
|
|
7
7
|
import { Parser } from '@traqula/parser-sparql-1-1';
|
|
8
8
|
import { Generator } from '@traqula/generator-sparql-1-1';
|
|
9
|
-
import { createHostLimiter, parseRetryAfter, } from '@lde/host-limiter';
|
|
9
|
+
import { createHostLimiter, endpointKeyOf, parseRetryAfter, } from '@lde/host-limiter';
|
|
10
10
|
import { quadToStringQuad } from 'rdf-string';
|
|
11
11
|
import { withDefaultGraph } from './graph.js';
|
|
12
12
|
import { injectValues } from './values.js';
|
|
@@ -18,8 +18,6 @@ import { classifyOutcome, isTransientError, statusOf } from './failure.js';
|
|
|
18
18
|
* matters when the reader is driven directly (without a Pipeline).
|
|
19
19
|
*/
|
|
20
20
|
const defaultTimeoutPolicy = new ConstantTimeoutPolicy(300_000);
|
|
21
|
-
/** The limiter of every read not given one, so they share one limit. */
|
|
22
|
-
const defaultLimiter = createHostLimiter(10, 10);
|
|
23
21
|
/**
|
|
24
22
|
* A reader could not run because the dataset lacks a supported distribution.
|
|
25
23
|
*/
|
|
@@ -43,9 +41,9 @@ export class NoItemsSelected extends NotSupported {
|
|
|
43
41
|
/**
|
|
44
42
|
* A streaming SPARQL CONSTRUCT reader.
|
|
45
43
|
*
|
|
46
|
-
* Each query runs on the {@link ReadOptions.limiter}, keyed on the
|
|
47
|
-
*
|
|
48
|
-
* reader pauses
|
|
44
|
+
* Each query runs on the {@link ReadOptions.limiter}, keyed on the endpoint
|
|
45
|
+
* ({@link endpointKeyOf}). While the endpoint is down (502, 503, 504 or a
|
|
46
|
+
* network error) the reader pauses it and tries again; on a 500 it splits the batch,
|
|
49
47
|
* down to a root it reports through {@link ReadOptions.onRootSkipped} and
|
|
50
48
|
* leaves out. See
|
|
51
49
|
* {@link https://ldelements.org/reference/pipeline#when-the-endpoint-fails | When the endpoint fails}.
|
|
@@ -128,8 +126,8 @@ export class SparqlConstructReader {
|
|
|
128
126
|
let skipped = 0;
|
|
129
127
|
const quads = await this.readBatch(bindings, query, {
|
|
130
128
|
endpoint,
|
|
131
|
-
key:
|
|
132
|
-
limiter: options?.limiter ??
|
|
129
|
+
key: endpointKeyOf(endpoint),
|
|
130
|
+
limiter: options?.limiter ?? createHostLimiter(10, 10),
|
|
133
131
|
policy: options?.timeout ?? defaultTimeoutPolicy,
|
|
134
132
|
onRootSkipped: (root, error) => {
|
|
135
133
|
skipped++;
|
|
@@ -173,8 +171,8 @@ export class SparqlConstructReader {
|
|
|
173
171
|
}
|
|
174
172
|
}
|
|
175
173
|
/**
|
|
176
|
-
* Run a query on the limiter, pausing the
|
|
177
|
-
*
|
|
174
|
+
* Run a query on the limiter, pausing the endpoint and trying again while it is
|
|
175
|
+
* down; see {@link SparqlConstructReader}.
|
|
178
176
|
*/
|
|
179
177
|
async fetchWithRetries(query, attempt) {
|
|
180
178
|
for (let retry = 0;; retry++) {
|
|
@@ -359,7 +357,7 @@ async function* noQuads() {
|
|
|
359
357
|
// A skipped root reads as no quads.
|
|
360
358
|
}
|
|
361
359
|
/**
|
|
362
|
-
* How long a reader pauses
|
|
360
|
+
* How long a reader pauses the endpoint before each retry after it
|
|
363
361
|
* went down, in milliseconds: tripled each time, about 10 minutes in all, to
|
|
364
362
|
* outlast a backend restart. A `Retry-After` header replaces a step.
|
|
365
363
|
*/
|
package/dist/stage.d.ts
CHANGED
|
@@ -201,10 +201,13 @@ export declare class Stage<Out = Quad> {
|
|
|
201
201
|
/** Whether this stage reads a source of its own rather than the dataset’s
|
|
202
202
|
* resolved distribution. @see {@link StageOptions.sourceFor} */
|
|
203
203
|
readonly sourcesOwnData: boolean;
|
|
204
|
+
/** Maximum concurrent in-flight SPARQL queries. @see {@link StageOptions.maxConcurrency} */
|
|
205
|
+
readonly maxConcurrency: number;
|
|
204
206
|
private readonly readers;
|
|
205
207
|
private readonly itemSelector?;
|
|
206
208
|
private readonly batchSize;
|
|
207
|
-
|
|
209
|
+
/** The limiter of a run not given one, so its batches back off together. */
|
|
210
|
+
private readonly limiter;
|
|
208
211
|
private readonly validation?;
|
|
209
212
|
private readonly project?;
|
|
210
213
|
private readonly queueCapacity?;
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAMrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGpE;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;CACvE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACtB,MAAM,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAUxD,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC;IAC3E,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAW,SAAQ,IAAI,CACtC,WAAW,EACX,SAAS,GAAG,eAAe,CAC5B;IACC,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB;iEAC6D;IAC7D,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C;oFACgF;IAChF,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC9C;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK,CAAC,GAAG,GAAG,IAAI;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC;qEACiE;IACjE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,4FAA4F;IAC5F,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAkC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiC;gBAEhD,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;IAqCtC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IA4G/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IA0L7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA0BrB;;;;;OAKG;YACW,IAAI;IA2BlB,8EAA8E;IAC9E,OAAO,KAAK,cAAc,GAOzB;CACF;AA0DD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
|
package/dist/stage.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHostLimiter, positiveIntOrDefault, } from '@lde/host-limiter';
|
|
1
2
|
import { NoItemsSelected, NotSupported } from './sparql/reader.js';
|
|
2
3
|
import { batch } from './batch.js';
|
|
3
4
|
import { AsyncQueue } from './asyncQueue.js';
|
|
@@ -10,10 +11,13 @@ export class Stage {
|
|
|
10
11
|
/** Whether this stage reads a source of its own rather than the dataset’s
|
|
11
12
|
* resolved distribution. @see {@link StageOptions.sourceFor} */
|
|
12
13
|
sourcesOwnData;
|
|
14
|
+
/** Maximum concurrent in-flight SPARQL queries. @see {@link StageOptions.maxConcurrency} */
|
|
15
|
+
maxConcurrency;
|
|
13
16
|
readers;
|
|
14
17
|
itemSelector;
|
|
15
18
|
batchSize;
|
|
16
|
-
|
|
19
|
+
/** The limiter of a run not given one, so its batches back off together. */
|
|
20
|
+
limiter;
|
|
17
21
|
validation;
|
|
18
22
|
project;
|
|
19
23
|
queueCapacity;
|
|
@@ -39,6 +43,8 @@ export class Stage {
|
|
|
39
43
|
this.itemSelector = options.itemSelector;
|
|
40
44
|
this.batchSize = options.batchSize ?? 10;
|
|
41
45
|
this.maxConcurrency = options.maxConcurrency ?? 10;
|
|
46
|
+
const ceiling = positiveIntOrDefault(this.maxConcurrency, 10);
|
|
47
|
+
this.limiter = createHostLimiter(ceiling, ceiling);
|
|
42
48
|
this.validation = options.validation;
|
|
43
49
|
this.expectsOutput = options.expectsOutput ?? false;
|
|
44
50
|
this.project = options.project;
|
|
@@ -54,7 +60,7 @@ export class Stage {
|
|
|
54
60
|
const timeout = options?.timeout;
|
|
55
61
|
const readOptions = {
|
|
56
62
|
timeout,
|
|
57
|
-
limiter: options?.limiter,
|
|
63
|
+
limiter: options?.limiter ?? this.limiter,
|
|
58
64
|
onRootSkipped: options?.onRootSkipped,
|
|
59
65
|
};
|
|
60
66
|
// Substituted once, before anything reads: the selector and the readers
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.49.0",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -26,9 +26,9 @@
|
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@lde/dataset": "^0.10.1",
|
|
28
28
|
"@lde/dataset-registry-client": "^0.12.1",
|
|
29
|
-
"@lde/distribution-health": "^0.
|
|
30
|
-
"@lde/distribution-probe": "^0.
|
|
31
|
-
"@lde/host-limiter": "^0.
|
|
29
|
+
"@lde/distribution-health": "^0.10.0",
|
|
30
|
+
"@lde/distribution-probe": "^0.8.0",
|
|
31
|
+
"@lde/host-limiter": "^0.5.0",
|
|
32
32
|
"@lde/sparql-importer": "^0.11.1",
|
|
33
33
|
"@lde/sparql-server": "^0.6.0",
|
|
34
34
|
"@rdfjs/namespace": "^2.0.1",
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"@traqula/generator-sparql-1-1": "^1.3.1",
|
|
38
38
|
"@traqula/parser-sparql-1-1": "^1.3.1",
|
|
39
39
|
"@traqula/rules-sparql-1-1": "^1.2.0",
|
|
40
|
-
"fetch-sparql-endpoint": "^7.
|
|
40
|
+
"fetch-sparql-endpoint": "^7.3.0",
|
|
41
41
|
"filenamify-url": "^4.0.0",
|
|
42
42
|
"is-network-error": "^1.3.2",
|
|
43
43
|
"n3": "^2.13.12",
|