@lde/pipeline 0.46.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pipeline.d.ts +13 -0
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +10 -0
- package/dist/pipelineEvent.d.ts +12 -1
- package/dist/pipelineEvent.d.ts.map +1 -1
- package/dist/pipelineEvent.js +1 -0
- package/dist/sparql/failure.d.ts +16 -0
- package/dist/sparql/failure.d.ts.map +1 -0
- package/dist/sparql/failure.js +42 -0
- package/dist/sparql/reader.d.ts +35 -7
- package/dist/sparql/reader.d.ts.map +1 -1
- package/dist/sparql/reader.js +124 -54
- package/dist/sparql/selector.d.ts.map +1 -1
- package/dist/sparql/selector.js +2 -20
- package/dist/sparql/timeoutPolicy.d.ts +2 -2
- package/dist/stage.d.ts +2 -2
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +11 -6
- package/package.json +4 -4
package/dist/pipeline.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Dataset } from '@lde/dataset';
|
|
2
|
+
import type { HostLimiter } from '@lde/host-limiter';
|
|
2
3
|
import type { Quad } from '@rdfjs/types';
|
|
3
4
|
import type { DatasetSelector } from './selector.js';
|
|
4
5
|
import { Stage } from './stage.js';
|
|
@@ -119,6 +120,17 @@ export interface PipelineOptions<Out = Quad> {
|
|
|
119
120
|
* once per dataset so state resets between datasets.
|
|
120
121
|
*/
|
|
121
122
|
timeout?: () => TimeoutPolicy;
|
|
123
|
+
/**
|
|
124
|
+
* The limiter every stage’s readers run their queries on, keyed on the
|
|
125
|
+
* dataset’s IRI. Each dataset’s limit adapts to how its endpoint responds,
|
|
126
|
+
* up to the limiter’s per-host ceiling; see {@link SparqlConstructReader}.
|
|
127
|
+
*
|
|
128
|
+
* Pass the limiter you also give a resolver that loads the same backend –
|
|
129
|
+
* keyed on the dataset too – so the two share one limit.
|
|
130
|
+
*
|
|
131
|
+
* Left out, the readers share a default: `createHostLimiter(10, 10)`.
|
|
132
|
+
*/
|
|
133
|
+
limiter?: HostLimiter;
|
|
122
134
|
}
|
|
123
135
|
export declare class Pipeline<Out = Quad> {
|
|
124
136
|
private readonly name;
|
|
@@ -131,6 +143,7 @@ export declare class Pipeline<Out = Quad> {
|
|
|
131
143
|
private readonly chaining?;
|
|
132
144
|
private readonly reporters;
|
|
133
145
|
private readonly timeoutFactory;
|
|
146
|
+
private readonly limiter?;
|
|
134
147
|
private readonly provenanceStore?;
|
|
135
148
|
private readonly pipelineVersion?;
|
|
136
149
|
constructor(options: PipelineOptions<Out>);
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAW7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AASpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAG9C,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,QAAQ,EAAE,CAAC;IAC1C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAW7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AASpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAG9C,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,QAAQ,EAAE,CAAC;IAC1C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;IAC9B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsB;IAChD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAc;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA0DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAyFZ,cAAc;IA8R5B,OAAO,CAAC,MAAM;IAyBd,OAAO,CAAC,iBAAiB;IAczB,+EAA+E;YACjE,aAAa;YAuBb,gBAAgB;IAgB9B,OAAO,CAAE,aAAa;IAStB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA+BlC;;;;;;OAMG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA6EtB,2EAA2E;YAC7D,eAAe;YAuBf,QAAQ;YA+EP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -240,6 +240,7 @@ export class Pipeline {
|
|
|
240
240
|
chaining;
|
|
241
241
|
reporters;
|
|
242
242
|
timeoutFactory;
|
|
243
|
+
limiter;
|
|
243
244
|
provenanceStore;
|
|
244
245
|
pipelineVersion;
|
|
245
246
|
constructor(options) {
|
|
@@ -281,6 +282,7 @@ export class Pipeline {
|
|
|
281
282
|
this.reporters = [options.reporter ?? []].flat();
|
|
282
283
|
this.timeoutFactory =
|
|
283
284
|
options.timeout ?? (() => new ConstantTimeoutPolicy(300_000));
|
|
285
|
+
this.limiter = options.limiter;
|
|
284
286
|
this.provenanceStore = options.provenanceStore;
|
|
285
287
|
this.pipelineVersion = options.pipelineVersion;
|
|
286
288
|
}
|
|
@@ -749,6 +751,14 @@ export class Pipeline {
|
|
|
749
751
|
});
|
|
750
752
|
},
|
|
751
753
|
timeout,
|
|
754
|
+
limiter: this.limiter,
|
|
755
|
+
onRootSkipped: (root, error) => this.report(runId, {
|
|
756
|
+
type: 'rootSkipped',
|
|
757
|
+
dataset,
|
|
758
|
+
stage: stage.name,
|
|
759
|
+
root,
|
|
760
|
+
error,
|
|
761
|
+
}),
|
|
752
762
|
valueUnread: unread && ((target, field, value) => unread.add(target, field, value)),
|
|
753
763
|
onComplete: (stageSummary) => {
|
|
754
764
|
summary = stageSummary;
|
package/dist/pipelineEvent.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { ValidityVerdict } from '@lde/distribution-health';
|
|
|
3
3
|
import type { ValidationReport } from './validator.js';
|
|
4
4
|
import type { ImportDetails } from './distribution/resolver.js';
|
|
5
5
|
import type { TimeoutTransitionEvent } from './sparql/timeoutPolicy.js';
|
|
6
|
+
import type { VariableBindings } from './sparql/reader.js';
|
|
6
7
|
/**
|
|
7
8
|
* Deliberately without `fatal`: the pipeline never ends the process; an
|
|
8
9
|
* unrecoverable failure rejects {@link Pipeline.run}.
|
|
@@ -220,6 +221,16 @@ export interface StageFailedEvent extends DatasetEventEnvelope<'stageFailed'> {
|
|
|
220
221
|
stage: string;
|
|
221
222
|
error: Error;
|
|
222
223
|
}
|
|
224
|
+
/**
|
|
225
|
+
* A stage left one root out: the endpoint still answered HTTP 500 to a query
|
|
226
|
+
* for that root alone, so the rest of its batch was read without it.
|
|
227
|
+
*/
|
|
228
|
+
export interface RootSkippedEvent extends DatasetEventEnvelope<'rootSkipped'> {
|
|
229
|
+
stage: string;
|
|
230
|
+
/** The root, as the stage’s item selector selected it. */
|
|
231
|
+
root: VariableBindings;
|
|
232
|
+
error: Error;
|
|
233
|
+
}
|
|
223
234
|
/** A stage was skipped for a dataset. */
|
|
224
235
|
export interface StageSkippedEvent extends DatasetEventEnvelope<'stageSkipped'> {
|
|
225
236
|
stage: string;
|
|
@@ -265,7 +276,7 @@ export interface TimeoutTightenedEvent extends DatasetEventEnvelope<'timeoutTigh
|
|
|
265
276
|
export interface TimeoutRelaxedEvent extends DatasetEventEnvelope<'timeoutRelaxed'>, TimeoutTransitionEvent {
|
|
266
277
|
}
|
|
267
278
|
/** Everything a pipeline run reports, discriminated on `type`. */
|
|
268
|
-
export type PipelineEvent = PipelineStartEvent | DatasetsSelectedEvent | SelectionEmptyEvent | PipelineCompleteEvent | DatasetStartEvent | DistributionProbedEvent | DistributionValidatedEvent | ImportStartedEvent | ImportFailedEvent | DistributionSelectedEvent | StageStartEvent | StageProgressEvent | StageCompleteEvent | StageFailedEvent | StageSkippedEvent | DatasetValidatedEvent | DatasetCompleteEvent | DatasetSkippedEvent | ItemsRejectedEvent | ValuesUnreadEvent | TimeoutTightenedEvent | TimeoutRelaxedEvent;
|
|
279
|
+
export type PipelineEvent = PipelineStartEvent | DatasetsSelectedEvent | SelectionEmptyEvent | PipelineCompleteEvent | DatasetStartEvent | DistributionProbedEvent | DistributionValidatedEvent | ImportStartedEvent | ImportFailedEvent | DistributionSelectedEvent | StageStartEvent | StageProgressEvent | StageCompleteEvent | StageFailedEvent | StageSkippedEvent | RootSkippedEvent | DatasetValidatedEvent | DatasetCompleteEvent | DatasetSkippedEvent | ItemsRejectedEvent | ValuesUnreadEvent | TimeoutTightenedEvent | TimeoutRelaxedEvent;
|
|
269
280
|
/** A {@link PipelineEvent} without the envelope fields the pipeline fills in. */
|
|
270
281
|
export type PipelineEventPayload = PipelineEvent extends infer Event ? Event extends PipelineEvent ? Omit<Event, 'defaultLevel' | 'timestamp' | 'runId'> : never : never;
|
|
271
282
|
/** The pipeline’s default {@link Level} for an event. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipelineEvent.d.ts","sourceRoot":"","sources":["../src/pipelineEvent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"pipelineEvent.d.ts","sourceRoot":"","sources":["../src/pipelineEvent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACxE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D;;;GAGG;AACH,MAAM,MAAM,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAExD;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,SAAS,MAAM;IAChD,IAAI,EAAE,IAAI,CAAC;IACX,YAAY,EAAE,KAAK,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,oEAAoE;AACpE,MAAM,WAAW,oBAAoB,CACnC,IAAI,SAAS,MAAM,CACnB,SAAQ,aAAa,CAAC,IAAI,CAAC;IAC3B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,8FAA8F;AAC9F,MAAM,WAAW,MAAM,CAAC,IAAI,SAAS,MAAM;IACzC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,0CAA0C;AAC1C,MAAM,MAAM,iBAAiB,GAAG,MAAM,CACpC,WAAW,GAAG,aAAa,GAAG,eAAe,GAAG,yBAAyB,CAC1E,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,cAAc,GAAG,iBAAiB,CAAC,CAAC;AAEzE,+CAA+C;AAC/C,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B;qBACiB;IACjB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,sBAAsB;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;oEAEgE;IAChE,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;CAC9C;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;4DACwD;IACxD,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,CAAC;CAC3C;AAED,0EAA0E;AAC1E,MAAM,WAAW,WAAW;IAC1B;;8DAE0D;IAC1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,4DAA4D;AAC5D,MAAM,WAAW,WAAW;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,kBAAmB,SAAQ,aAAa,CAAC,eAAe,CAAC;IACxE,IAAI,EAAE,MAAM,CAAC;CACd;AAED,oFAAoF;AACpF,MAAM,WAAW,qBAAsB,SAAQ,aAAa,CAAC,kBAAkB,CAAC;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAoB,SAAQ,aAAa,CAAC,gBAAgB,CAAC;IAC1E,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,sBAAsB;AACtB,MAAM,WAAW,qBACf,SAAQ,aAAa,CAAC,kBAAkB,CAAC,EAAE,WAAW;IACtD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,iDAAiD;AACjD,MAAM,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAC;AAErE,mDAAmD;AACnD,MAAM,WAAW,uBAAwB,SAAQ,oBAAoB,CAAC,oBAAoB,CAAC;IACzF,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,EAAE,0BAA0B,CAAC;CACpC;AAED,0GAA0G;AAC1G,MAAM,WAAW,0BAA2B,SAAQ,oBAAoB,CAAC,uBAAuB,CAAC;IAC/F,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,EAAE,eAAe,CAAC;CAC1B;AAED,mDAAmD;AACnD,MAAM,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,eAAe,CAAC,CAAC;AAEvE,uCAAuC;AACvC,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;CACd;AAED,0EAA0E;AAC1E,MAAM,WAAW,yBAA0B,SAAQ,oBAAoB,CAAC,sBAAsB,CAAC;IAC7F,YAAY,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,qCAAqC;AACrC,MAAM,WAAW,eAAgB,SAAQ,oBAAoB,CAAC,YAAY,CAAC;IACzE,KAAK,EAAE,MAAM,CAAC;CACf;AAED,yCAAyC;AACzC,MAAM,WAAW,kBACf,SAAQ,oBAAoB,CAAC,eAAe,CAAC,EAAE,WAAW;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,sCAAsC;AACtC,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB,CAAC,eAAe,CAAC;IAC/E,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB;uEACmE;IACnE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;iCAC6B;IAC7B,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC3C;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW;IAC1B,iDAAiD;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,8DAA8D;IAC9D,MAAM,EAAE,MAAM,CAAC;IACf;wEACoE;IACpE,UAAU,EAAE,SAAS,iBAAiB,EAAE,CAAC;IACzC,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;IAClB,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,8EAA8E;AAC9E,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC1C;AAED,wEAAwE;AACxE,MAAM,WAAW,iBAAiB;IAChC;oDACgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB,CAAC,aAAa,CAAC;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;CACd;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB,CAAC,aAAa,CAAC;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,0DAA0D;IAC1D,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,KAAK,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAsB,SAAQ,oBAAoB,CAAC,kBAAkB,CAAC;IACrF,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED,kDAAkD;AAClD,MAAM,WAAW,oBACf,SAAQ,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,WAAW;IAC5D,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,mBAAoB,SAAQ,oBAAoB,CAAC,gBAAgB,CAAC;IACjF,MAAM,EAAE,iBAAiB,CAAC;IAC1B,qEAAqE;IACrE,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB,CAAC,eAAe,CAAC;IAC/E,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,gEAAgE;AAChE,MAAM,WAAW,qBACf,SAAQ,oBAAoB,CAAC,kBAAkB,CAAC,EAAE,sBAAsB;CAAG;AAE7E,8DAA8D;AAC9D,MAAM,WAAW,mBACf,SAAQ,oBAAoB,CAAC,gBAAgB,CAAC,EAAE,sBAAsB;CAAG;AAE3E,kEAAkE;AAClE,MAAM,MAAM,aAAa,GACrB,kBAAkB,GAClB,qBAAqB,GACrB,mBAAmB,GACnB,qBAAqB,GACrB,iBAAiB,GACjB,uBAAuB,GACvB,0BAA0B,GAC1B,kBAAkB,GAClB,iBAAiB,GACjB,yBAAyB,GACzB,eAAe,GACf,kBAAkB,GAClB,kBAAkB,GAClB,gBAAgB,GAChB,iBAAiB,GACjB,gBAAgB,GAChB,qBAAqB,GACrB,oBAAoB,GACpB,mBAAmB,GACnB,kBAAkB,GAClB,iBAAiB,GACjB,qBAAqB,GACrB,mBAAmB,CAAC;AAExB,iFAAiF;AACjF,MAAM,MAAM,oBAAoB,GAAG,aAAa,SAAS,MAAM,KAAK,GAChE,KAAK,SAAS,aAAa,GACzB,IAAI,CAAC,KAAK,EAAE,cAAc,GAAG,WAAW,GAAG,OAAO,CAAC,GACnD,KAAK,GACP,KAAK,CAAC;AAEV,yDAAyD;AACzD,wBAAgB,OAAO,CAAC,KAAK,EAAE,oBAAoB,GAAG,KAAK,CAkC1D"}
|
package/dist/pipelineEvent.js
CHANGED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { TimeoutOutcome } from './timeoutPolicy.js';
|
|
2
|
+
/** The HTTP status `fetch-sparql-endpoint` put in an error’s message. */
|
|
3
|
+
export declare function statusOf(error: Error): number | undefined;
|
|
4
|
+
/** Whether the endpoint is down or struggling: HTTP 502, 503 or 504, or a
|
|
5
|
+
* network error. */
|
|
6
|
+
export declare function isTransientError(error: unknown): boolean;
|
|
7
|
+
/**
|
|
8
|
+
* Classify a fetch error for {@link TimeoutPolicy} reporting.
|
|
9
|
+
*
|
|
10
|
+
* - HTTP 504 → `'timeout'`: the upstream reported it ran out of time. This
|
|
11
|
+
* is the exact failure mode adaptive timeouts exist to react to.
|
|
12
|
+
* - `AbortError` / `TimeoutError`: our own `AbortSignal.timeout()` fired.
|
|
13
|
+
* - Anything else → `'error'`: neutral with respect to tightening.
|
|
14
|
+
*/
|
|
15
|
+
export declare function classifyOutcome(error: unknown): TimeoutOutcome;
|
|
16
|
+
//# sourceMappingURL=failure.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failure.d.ts","sourceRoot":"","sources":["../../src/sparql/failure.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAIzD,yEAAyE;AACzE,wBAAgB,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,GAAG,SAAS,CAGzD;AAED;qBACqB;AACrB,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAKxD;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,cAAc,CAkB9D"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import isNetworkError from 'is-network-error';
|
|
2
|
+
const transientStatusPattern = /HTTP status (\d+)/;
|
|
3
|
+
/** The HTTP status `fetch-sparql-endpoint` put in an error’s message. */
|
|
4
|
+
export function statusOf(error) {
|
|
5
|
+
const match = error.message.match(transientStatusPattern);
|
|
6
|
+
return match ? Number(match[1]) : undefined;
|
|
7
|
+
}
|
|
8
|
+
/** Whether the endpoint is down or struggling: HTTP 502, 503 or 504, or a
|
|
9
|
+
* network error. */
|
|
10
|
+
export function isTransientError(error) {
|
|
11
|
+
if (isNetworkError(error))
|
|
12
|
+
return true;
|
|
13
|
+
if (!(error instanceof Error))
|
|
14
|
+
return false;
|
|
15
|
+
const status = statusOf(error);
|
|
16
|
+
return status === 502 || status === 503 || status === 504;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Classify a fetch error for {@link TimeoutPolicy} reporting.
|
|
20
|
+
*
|
|
21
|
+
* - HTTP 504 → `'timeout'`: the upstream reported it ran out of time. This
|
|
22
|
+
* is the exact failure mode adaptive timeouts exist to react to.
|
|
23
|
+
* - `AbortError` / `TimeoutError`: our own `AbortSignal.timeout()` fired.
|
|
24
|
+
* - Anything else → `'error'`: neutral with respect to tightening.
|
|
25
|
+
*/
|
|
26
|
+
export function classifyOutcome(error) {
|
|
27
|
+
if (error instanceof Error) {
|
|
28
|
+
if (error.name === 'AbortError' || error.name === 'TimeoutError') {
|
|
29
|
+
return 'timeout';
|
|
30
|
+
}
|
|
31
|
+
if (error.cause instanceof Error) {
|
|
32
|
+
if (error.cause.name === 'AbortError' ||
|
|
33
|
+
error.cause.name === 'TimeoutError') {
|
|
34
|
+
return 'timeout';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
if (statusOf(error) === 504) {
|
|
38
|
+
return 'timeout';
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return 'error';
|
|
42
|
+
}
|
package/dist/sparql/reader.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { Dataset, Distribution } from '@lde/dataset';
|
|
|
2
2
|
import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
|
|
3
3
|
import type { NamedNode, Quad } from '@rdfjs/types';
|
|
4
4
|
import { Transform } from 'node:stream';
|
|
5
|
+
import { type HostLimiter } from '@lde/host-limiter';
|
|
5
6
|
import { type TimeoutPolicy } from './timeoutPolicy.js';
|
|
6
7
|
/**
|
|
7
8
|
* A reader could not run because the dataset lacks a supported distribution.
|
|
@@ -37,6 +38,21 @@ export interface ReadOptions {
|
|
|
37
38
|
* Pipeline runners use this to thread the per-dataset policy through.
|
|
38
39
|
*/
|
|
39
40
|
timeout?: TimeoutPolicy;
|
|
41
|
+
/**
|
|
42
|
+
* The limiter every query of a read goes through, keyed on the dataset’s
|
|
43
|
+
* IRI, so a dataset’s readers – and anything else keyed on the dataset, such
|
|
44
|
+
* as a resolver fetching from the same backend – share one adaptive limit.
|
|
45
|
+
* See {@link SparqlConstructReader}.
|
|
46
|
+
*
|
|
47
|
+
* Reads not given one share a default: `createHostLimiter(10, 10)`.
|
|
48
|
+
*/
|
|
49
|
+
limiter?: HostLimiter;
|
|
50
|
+
/**
|
|
51
|
+
* Called for each root the endpoint failed on by itself: a query for that
|
|
52
|
+
* one root still answered HTTP 500. The root is left out of the read, and
|
|
53
|
+
* the rest of the batch is read without it.
|
|
54
|
+
*/
|
|
55
|
+
onRootSkipped?: (root: VariableBindings, error: Error) => void;
|
|
40
56
|
}
|
|
41
57
|
export interface Reader {
|
|
42
58
|
read(dataset: Dataset, distribution: Distribution, options?: ReadOptions): Promise<AsyncIterable<Quad> | NotSupported>;
|
|
@@ -49,11 +65,6 @@ export interface SparqlConstructReaderOptions {
|
|
|
49
65
|
* SPARQL CONSTRUCT query to execute.
|
|
50
66
|
*/
|
|
51
67
|
query: string;
|
|
52
|
-
/**
|
|
53
|
-
* Number of retries for transient errors (network failures and HTTP 502/503/504).
|
|
54
|
-
* @default 3
|
|
55
|
-
*/
|
|
56
|
-
retries?: number;
|
|
57
68
|
/**
|
|
58
69
|
* Optional custom SparqlEndpointFetcher instance.
|
|
59
70
|
*
|
|
@@ -103,6 +114,13 @@ export interface SparqlConstructReaderOptions {
|
|
|
103
114
|
/**
|
|
104
115
|
* A streaming SPARQL CONSTRUCT reader.
|
|
105
116
|
*
|
|
117
|
+
* Each query runs on the {@link ReadOptions.limiter}, keyed on the dataset’s
|
|
118
|
+
* IRI. While the endpoint is down (502, 503, 504 or a network error) the
|
|
119
|
+
* reader pauses the dataset and tries again; on a 500 it splits the batch,
|
|
120
|
+
* down to a root it reports through {@link ReadOptions.onRootSkipped} and
|
|
121
|
+
* leaves out. See
|
|
122
|
+
* {@link https://ldelements.org/reference/pipeline#when-the-endpoint-fails | When the endpoint fails}.
|
|
123
|
+
*
|
|
106
124
|
* Queries **without** `#subjectFilter#` are parsed once in the constructor
|
|
107
125
|
* (fast path). Queries that contain the template are stored as raw strings
|
|
108
126
|
* and parsed at {@link read} time after substitution.
|
|
@@ -131,7 +149,6 @@ export declare class SparqlConstructReader implements Reader {
|
|
|
131
149
|
private readonly rawQuery;
|
|
132
150
|
private readonly preParsed?;
|
|
133
151
|
private readonly userFetcher?;
|
|
134
|
-
private readonly retries;
|
|
135
152
|
private readonly lineBuffer;
|
|
136
153
|
private readonly deduplicate;
|
|
137
154
|
private readonly generator;
|
|
@@ -146,6 +163,16 @@ export declare class SparqlConstructReader implements Reader {
|
|
|
146
163
|
* @returns AsyncIterable<Quad> stream of results.
|
|
147
164
|
*/
|
|
148
165
|
read(dataset: Dataset, distribution: Distribution, options?: ReadOptions): Promise<AsyncIterable<Quad>>;
|
|
166
|
+
/**
|
|
167
|
+
* Read one batch, splitting it on an HTTP 500 until the endpoint answers or
|
|
168
|
+
* one root is left; see {@link SparqlConstructReader}.
|
|
169
|
+
*/
|
|
170
|
+
private readBatch;
|
|
171
|
+
/**
|
|
172
|
+
* Run a query on the limiter, pausing the dataset and trying again while the
|
|
173
|
+
* endpoint is down; see {@link SparqlConstructReader}.
|
|
174
|
+
*/
|
|
175
|
+
private fetchWithRetries;
|
|
149
176
|
/**
|
|
150
177
|
* Run a single attempt against the endpoint with a per-call abort
|
|
151
178
|
* signal derived from {@link TimeoutPolicy.beforeRequest}. Reports the
|
|
@@ -159,7 +186,8 @@ export declare class SparqlConstructReader implements Reader {
|
|
|
159
186
|
* policy budget is bypassed in that case (see the JSDoc on
|
|
160
187
|
* {@link SparqlConstructReaderOptions.fetcher}). Otherwise a fresh
|
|
161
188
|
* {@link SparqlEndpointFetcher} is constructed per attempt with the
|
|
162
|
-
* policy-supplied timeout baked in
|
|
189
|
+
* policy-supplied timeout baked in, which notes the response’s
|
|
190
|
+
* `Retry-After` in `response`.
|
|
163
191
|
*/
|
|
164
192
|
private fetcherForAttempt;
|
|
165
193
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxC,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EAAyB,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAa/E;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,YAAY;;CAIhD;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE9B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IAExB;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;IAEtB;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAChE;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CACF,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,4BAA4B;IAgBjD;;;;;;;;OAQG;IACG,IAAI,CACR,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA+C/B;;;OAGG;YACW,SAAS;IAiCvB;;;OAGG;YACW,gBAAgB;IAwC9B;;;;;OAKG;YACW,oBAAoB;IAgClC;;;;;;;;OAQG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GACpD,OAAO,CAAC,qBAAqB,CAAC;CAIlC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI;IAUjE,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GACzB,aAAa,CAAC,IAAI,CAAC,CASrB"}
|
package/dist/sparql/reader.js
CHANGED
|
@@ -6,18 +6,20 @@ import { Transform } from 'node:stream';
|
|
|
6
6
|
import { StreamParser } from 'n3';
|
|
7
7
|
import { Parser } from '@traqula/parser-sparql-1-1';
|
|
8
8
|
import { Generator } from '@traqula/generator-sparql-1-1';
|
|
9
|
-
import
|
|
10
|
-
import pRetry from 'p-retry';
|
|
9
|
+
import { createHostLimiter, parseRetryAfter, } from '@lde/host-limiter';
|
|
11
10
|
import { quadToStringQuad } from 'rdf-string';
|
|
12
11
|
import { withDefaultGraph } from './graph.js';
|
|
13
12
|
import { injectValues } from './values.js';
|
|
14
|
-
import { ConstantTimeoutPolicy
|
|
13
|
+
import { ConstantTimeoutPolicy } from './timeoutPolicy.js';
|
|
14
|
+
import { classifyOutcome, isTransientError, statusOf } from './failure.js';
|
|
15
15
|
/**
|
|
16
16
|
* Fallback policy when no per-call `TimeoutPolicy` is supplied via
|
|
17
17
|
* {@link ReadOptions.timeout}. Pipeline always supplies one, so this only
|
|
18
18
|
* matters when the reader is driven directly (without a Pipeline).
|
|
19
19
|
*/
|
|
20
20
|
const defaultTimeoutPolicy = new ConstantTimeoutPolicy(300_000);
|
|
21
|
+
/** The limiter of every read not given one, so they share one limit. */
|
|
22
|
+
const defaultLimiter = createHostLimiter(10, 10);
|
|
21
23
|
/**
|
|
22
24
|
* A reader could not run because the dataset lacks a supported distribution.
|
|
23
25
|
*/
|
|
@@ -41,6 +43,13 @@ export class NoItemsSelected extends NotSupported {
|
|
|
41
43
|
/**
|
|
42
44
|
* A streaming SPARQL CONSTRUCT reader.
|
|
43
45
|
*
|
|
46
|
+
* Each query runs on the {@link ReadOptions.limiter}, keyed on the dataset’s
|
|
47
|
+
* IRI. While the endpoint is down (502, 503, 504 or a network error) the
|
|
48
|
+
* reader pauses the dataset and tries again; on a 500 it splits the batch,
|
|
49
|
+
* down to a root it reports through {@link ReadOptions.onRootSkipped} and
|
|
50
|
+
* leaves out. See
|
|
51
|
+
* {@link https://ldelements.org/reference/pipeline#when-the-endpoint-fails | When the endpoint fails}.
|
|
52
|
+
*
|
|
44
53
|
* Queries **without** `#subjectFilter#` are parsed once in the constructor
|
|
45
54
|
* (fast path). Queries that contain the template are stored as raw strings
|
|
46
55
|
* and parsed at {@link read} time after substitution.
|
|
@@ -69,13 +78,11 @@ export class SparqlConstructReader {
|
|
|
69
78
|
rawQuery;
|
|
70
79
|
preParsed;
|
|
71
80
|
userFetcher;
|
|
72
|
-
retries;
|
|
73
81
|
lineBuffer;
|
|
74
82
|
deduplicate;
|
|
75
83
|
generator = new Generator();
|
|
76
84
|
constructor(options) {
|
|
77
85
|
this.rawQuery = options.query;
|
|
78
|
-
this.retries = options.retries ?? 3;
|
|
79
86
|
this.lineBuffer = options.lineBuffer ?? false;
|
|
80
87
|
this.deduplicate = options.deduplicate ?? false;
|
|
81
88
|
if (!options.query.includes('#subjectFilter#')) {
|
|
@@ -113,29 +120,102 @@ export class SparqlConstructReader {
|
|
|
113
120
|
if (distribution.namedGraph) {
|
|
114
121
|
withDefaultGraph(ast, distribution.namedGraph);
|
|
115
122
|
}
|
|
116
|
-
const bindings = options?.bindings;
|
|
117
|
-
if (bindings !== undefined && bindings.length > 0) {
|
|
118
|
-
ast = injectValues(ast, bindings);
|
|
119
|
-
}
|
|
120
|
-
let query = this.generator.generate(ast);
|
|
121
123
|
assertSafeIri(dataset.iri.toString());
|
|
122
|
-
query =
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
124
|
+
const query = (bindings) => this.generator
|
|
125
|
+
.generate(bindings.length > 0 ? injectValues(ast, bindings) : ast)
|
|
126
|
+
.replaceAll('?dataset', `<${dataset.iri}>`);
|
|
127
|
+
const bindings = options?.bindings ?? [];
|
|
128
|
+
let skipped = 0;
|
|
129
|
+
const quads = await this.readBatch(bindings, query, {
|
|
130
|
+
endpoint,
|
|
131
|
+
key: dataset.iri.toString(),
|
|
132
|
+
limiter: options?.limiter ?? defaultLimiter,
|
|
133
|
+
policy: options?.timeout ?? defaultTimeoutPolicy,
|
|
134
|
+
onRootSkipped: (root, error) => {
|
|
135
|
+
skipped++;
|
|
136
|
+
options?.onRootSkipped?.(root, error);
|
|
137
|
+
// Not one root answered: the endpoint fails this query whatever the
|
|
138
|
+
// roots, so the batch fails rather than coming out empty.
|
|
139
|
+
if (skipped === bindings.length && skipped > 1)
|
|
140
|
+
throw error;
|
|
141
|
+
},
|
|
127
142
|
});
|
|
128
143
|
return this.deduplicate ? deduplicateQuads(quads) : quads;
|
|
129
144
|
}
|
|
145
|
+
/**
|
|
146
|
+
* Read one batch, splitting it on an HTTP 500 until the endpoint answers or
|
|
147
|
+
* one root is left; see {@link SparqlConstructReader}.
|
|
148
|
+
*/
|
|
149
|
+
async readBatch(bindings, query, attempt) {
|
|
150
|
+
try {
|
|
151
|
+
return await this.fetchWithRetries(query(bindings), attempt);
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
if (!(error instanceof Error) ||
|
|
155
|
+
statusOf(error) !== 500 ||
|
|
156
|
+
bindings.length === 0) {
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
159
|
+
if (bindings.length === 1) {
|
|
160
|
+
attempt.onRootSkipped(bindings[0], error);
|
|
161
|
+
return noQuads();
|
|
162
|
+
}
|
|
163
|
+
const middle = Math.ceil(bindings.length / 2);
|
|
164
|
+
const halves = [bindings.slice(0, middle), bindings.slice(middle)];
|
|
165
|
+
// Each half is asked for once the one before it has been read, so a
|
|
166
|
+
// split adds no load, and leaves no response open, of its own.
|
|
167
|
+
const readHalf = (half) => this.readBatch(half, query, attempt);
|
|
168
|
+
return (async function* () {
|
|
169
|
+
for (const half of halves) {
|
|
170
|
+
yield* await readHalf(half);
|
|
171
|
+
}
|
|
172
|
+
})();
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Run a query on the limiter, pausing the dataset and trying again while the
|
|
177
|
+
* endpoint is down; see {@link SparqlConstructReader}.
|
|
178
|
+
*/
|
|
179
|
+
async fetchWithRetries(query, attempt) {
|
|
180
|
+
for (let retry = 0;; retry++) {
|
|
181
|
+
try {
|
|
182
|
+
return await attempt.limiter.run(attempt.key, async (slot) => {
|
|
183
|
+
const response = {};
|
|
184
|
+
try {
|
|
185
|
+
return await this.fetchQuadsWithPolicy(attempt.endpoint, query, attempt.policy, response);
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
// A rejection lowers the limit, unless the slot is told otherwise.
|
|
189
|
+
if (isTransientError(error)) {
|
|
190
|
+
// Paused before the slot frees, so no query waiting for it goes
|
|
191
|
+
// out to an endpoint that is down.
|
|
192
|
+
if (retry < BACKOFF_MS.length) {
|
|
193
|
+
attempt.limiter.pause(attempt.key, Date.now() + backoffMs(retry, response.retryAfterMs));
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
else if (classifyOutcome(error) !== 'timeout') {
|
|
197
|
+
slot.ignore();
|
|
198
|
+
}
|
|
199
|
+
throw error;
|
|
200
|
+
}
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
if (!isTransientError(error) || retry === BACKOFF_MS.length) {
|
|
205
|
+
throw error;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
130
210
|
/**
|
|
131
211
|
* Run a single attempt against the endpoint with a per-call abort
|
|
132
212
|
* signal derived from {@link TimeoutPolicy.beforeRequest}. Reports the
|
|
133
213
|
* outcome via {@link TimeoutPolicy.afterRequest} regardless of whether
|
|
134
214
|
* the attempt resolved or threw.
|
|
135
215
|
*/
|
|
136
|
-
async fetchQuadsWithPolicy(endpointUrl, query, policy) {
|
|
216
|
+
async fetchQuadsWithPolicy(endpointUrl, query, policy, response) {
|
|
137
217
|
const timeoutMs = policy.beforeRequest({ endpoint: endpointUrl });
|
|
138
|
-
const fetcher = this.fetcherForAttempt(timeoutMs);
|
|
218
|
+
const fetcher = this.fetcherForAttempt(timeoutMs, response);
|
|
139
219
|
const start = Date.now();
|
|
140
220
|
try {
|
|
141
221
|
const quads = await this.fetchQuads(fetcher, endpointUrl.toString(), query);
|
|
@@ -162,12 +242,22 @@ export class SparqlConstructReader {
|
|
|
162
242
|
* policy budget is bypassed in that case (see the JSDoc on
|
|
163
243
|
* {@link SparqlConstructReaderOptions.fetcher}). Otherwise a fresh
|
|
164
244
|
* {@link SparqlEndpointFetcher} is constructed per attempt with the
|
|
165
|
-
* policy-supplied timeout baked in
|
|
245
|
+
* policy-supplied timeout baked in, which notes the response’s
|
|
246
|
+
* `Retry-After` in `response`.
|
|
166
247
|
*/
|
|
167
|
-
fetcherForAttempt(timeoutMs) {
|
|
248
|
+
fetcherForAttempt(timeoutMs, response) {
|
|
168
249
|
if (this.userFetcher)
|
|
169
250
|
return this.userFetcher;
|
|
170
|
-
return new SparqlEndpointFetcher({
|
|
251
|
+
return new SparqlEndpointFetcher({
|
|
252
|
+
timeout: timeoutMs,
|
|
253
|
+
fetch: async (input, init) => {
|
|
254
|
+
const answer = await fetch(input, init);
|
|
255
|
+
if (!answer.ok) {
|
|
256
|
+
response.retryAfterMs = parseRetryAfter(answer.headers.get('retry-after'));
|
|
257
|
+
}
|
|
258
|
+
return answer;
|
|
259
|
+
},
|
|
260
|
+
});
|
|
171
261
|
}
|
|
172
262
|
/**
|
|
173
263
|
* Fetch quads from the endpoint, optionally line-buffering the response
|
|
@@ -265,41 +355,21 @@ export async function* deduplicateQuads(quads) {
|
|
|
265
355
|
}
|
|
266
356
|
}
|
|
267
357
|
}
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
if (isNetworkError(error))
|
|
271
|
-
return true;
|
|
272
|
-
if (!(error instanceof Error))
|
|
273
|
-
return false;
|
|
274
|
-
const match = error.message.match(transientStatusPattern);
|
|
275
|
-
if (!match)
|
|
276
|
-
return false;
|
|
277
|
-
const status = Number(match[1]);
|
|
278
|
-
return status === 502 || status === 503 || status === 504;
|
|
358
|
+
async function* noQuads() {
|
|
359
|
+
// A skipped root reads as no quads.
|
|
279
360
|
}
|
|
280
361
|
/**
|
|
281
|
-
*
|
|
282
|
-
*
|
|
283
|
-
*
|
|
284
|
-
* is the exact failure mode adaptive timeouts exist to react to.
|
|
285
|
-
* - `AbortError` / `TimeoutError`: our own `AbortSignal.timeout()` fired.
|
|
286
|
-
* - Anything else → `'error'`: neutral with respect to tightening.
|
|
362
|
+
* How long a reader pauses its dataset before each retry after the endpoint
|
|
363
|
+
* went down, in milliseconds: tripled each time, about 10 minutes in all, to
|
|
364
|
+
* outlast a backend restart. A `Retry-After` header replaces a step.
|
|
287
365
|
*/
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
const match = error.message.match(transientStatusPattern);
|
|
300
|
-
if (match && Number(match[1]) === 504) {
|
|
301
|
-
return 'timeout';
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
return 'error';
|
|
366
|
+
const BACKOFF_MS = [5_000, 15_000, 45_000, 135_000, 405_000];
|
|
367
|
+
/**
|
|
368
|
+
* How long to pause before retry `retry`: what `Retry-After` asked, up to the
|
|
369
|
+
* longest step, or the step itself when the endpoint did not say.
|
|
370
|
+
*/
|
|
371
|
+
function backoffMs(retry, retryAfterMs) {
|
|
372
|
+
return retryAfterMs === undefined
|
|
373
|
+
? BACKOFF_MS[retry]
|
|
374
|
+
: Math.min(retryAfterMs, BACKOFF_MS[BACKOFF_MS.length - 1]);
|
|
305
375
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAcpD,MAAM,WAAW,yBAAyB;IACxC;;;;;;;;OAQG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoB;IAClD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;gBAEzC,OAAO,EAAE,yBAAyB;IA0BvC,MAAM,CACX,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,qBAAqB,CAAC,gBAAgB,CAAC;IA8E1C;;;;OAIG;YACW,uBAAuB;CA8BtC"}
|
package/dist/sparql/selector.js
CHANGED
|
@@ -2,9 +2,9 @@ import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
|
|
|
2
2
|
import { Parser } from '@traqula/parser-sparql-1-1';
|
|
3
3
|
import { Generator } from '@traqula/generator-sparql-1-1';
|
|
4
4
|
import { AstFactory, } from '@traqula/rules-sparql-1-1';
|
|
5
|
+
import { classifyOutcome } from './failure.js';
|
|
5
6
|
import { withDefaultGraph } from './graph.js';
|
|
6
|
-
import { ConstantTimeoutPolicy
|
|
7
|
-
const transientStatusPattern = /HTTP status (\d+)/;
|
|
7
|
+
import { ConstantTimeoutPolicy } from './timeoutPolicy.js';
|
|
8
8
|
/**
|
|
9
9
|
* Fallback policy when no per-call `TimeoutPolicy` is supplied via
|
|
10
10
|
* {@link SelectOptions.timeout}. Pipeline always supplies one, so this only
|
|
@@ -156,24 +156,6 @@ export class SparqlItemSelector {
|
|
|
156
156
|
}
|
|
157
157
|
}
|
|
158
158
|
}
|
|
159
|
-
function classifyOutcome(error) {
|
|
160
|
-
if (error instanceof Error) {
|
|
161
|
-
if (error.name === 'AbortError' || error.name === 'TimeoutError') {
|
|
162
|
-
return 'timeout';
|
|
163
|
-
}
|
|
164
|
-
if (error.cause instanceof Error) {
|
|
165
|
-
if (error.cause.name === 'AbortError' ||
|
|
166
|
-
error.cause.name === 'TimeoutError') {
|
|
167
|
-
return 'timeout';
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
const match = error.message.match(transientStatusPattern);
|
|
171
|
-
if (match && Number(match[1]) === 504) {
|
|
172
|
-
return 'timeout';
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
return 'error';
|
|
176
|
-
}
|
|
177
159
|
function isVariableTerm(v) {
|
|
178
160
|
return ('type' in v &&
|
|
179
161
|
v.type === 'term' &&
|
|
@@ -40,8 +40,8 @@ export interface AfterRequestContext {
|
|
|
40
40
|
export interface TimeoutPolicy {
|
|
41
41
|
/**
|
|
42
42
|
* Returns the timeout (in milliseconds) to apply to the upcoming request.
|
|
43
|
-
* Called once per attempt
|
|
44
|
-
*
|
|
43
|
+
* Called once per attempt, retries included, so a retry can already use a
|
|
44
|
+
* tightened budget.
|
|
45
45
|
*/
|
|
46
46
|
beforeRequest(context: BeforeRequestContext): number;
|
|
47
47
|
/**
|
package/dist/stage.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Dataset, Distribution } from '@lde/dataset';
|
|
2
2
|
import type { Quad } from '@rdfjs/types';
|
|
3
|
-
import type { Reader, VariableBindings } from './sparql/reader.js';
|
|
3
|
+
import type { ReadOptions, Reader, VariableBindings } from './sparql/reader.js';
|
|
4
4
|
import { NotSupported } from './sparql/reader.js';
|
|
5
5
|
import type { TimeoutPolicy } from './sparql/timeoutPolicy.js';
|
|
6
6
|
import type { Validator } from './validator.js';
|
|
@@ -172,7 +172,7 @@ export interface StageOptions<Out = Quad> {
|
|
|
172
172
|
onInvalid?: 'write' | 'skip' | 'halt';
|
|
173
173
|
};
|
|
174
174
|
}
|
|
175
|
-
export interface RunOptions {
|
|
175
|
+
export interface RunOptions extends Pick<ReadOptions, 'limiter' | 'onRootSkipped'> {
|
|
176
176
|
onProgress?: (itemsProcessed: number, quadsGenerated: number) => void;
|
|
177
177
|
/**
|
|
178
178
|
* Per-dataset {@link TimeoutPolicy} threaded through to readers and
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGpE;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;CACvE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACtB,MAAM,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAUxD,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC;IAC3E,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAW,SAAQ,IAAI,CACtC,WAAW,EACX,SAAS,GAAG,eAAe,CAC5B;IACC,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB;iEAC6D;IAC7D,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C;oFACgF;IAChF,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC9C;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK,CAAC,GAAG,GAAG,IAAI;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC;qEACiE;IACjE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAkC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiC;gBAEhD,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;IAmCtC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IA4G/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IA0L7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA0BrB;;;;;OAKG;YACW,IAAI;IA2BlB,8EAA8E;IAC9E,OAAO,KAAK,cAAc,GAOzB;CACF;AA0DD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
|
package/dist/stage.js
CHANGED
|
@@ -52,6 +52,11 @@ export class Stage {
|
|
|
52
52
|
}
|
|
53
53
|
async run(dataset, resolved, writer, options) {
|
|
54
54
|
const timeout = options?.timeout;
|
|
55
|
+
const readOptions = {
|
|
56
|
+
timeout,
|
|
57
|
+
limiter: options?.limiter,
|
|
58
|
+
onRootSkipped: options?.onRootSkipped,
|
|
59
|
+
};
|
|
55
60
|
// Substituted once, before anything reads: the selector and the readers
|
|
56
61
|
// must agree on where this stage's data comes from.
|
|
57
62
|
const distribution = this.sourceFor
|
|
@@ -61,13 +66,13 @@ export class Stage {
|
|
|
61
66
|
if (this.itemSelector) {
|
|
62
67
|
const result = await this.runWithSelector(tally.selected(this.itemSelector.select(distribution, this.batchSize, {
|
|
63
68
|
timeout,
|
|
64
|
-
})), dataset, distribution, writer, tally, options);
|
|
69
|
+
})), dataset, distribution, writer, tally, readOptions, options);
|
|
65
70
|
if (!(result instanceof NotSupported)) {
|
|
66
71
|
options?.onComplete?.(tally.summary());
|
|
67
72
|
}
|
|
68
73
|
return result;
|
|
69
74
|
}
|
|
70
|
-
const chains = await this.readAll(dataset, distribution,
|
|
75
|
+
const chains = await this.readAll(dataset, distribution, readOptions, tally);
|
|
71
76
|
if (chains instanceof NotSupported) {
|
|
72
77
|
return chains;
|
|
73
78
|
}
|
|
@@ -133,7 +138,7 @@ export class Stage {
|
|
|
133
138
|
throw new Error(`Stage '${this.name}' expected output but produced none`);
|
|
134
139
|
}
|
|
135
140
|
}
|
|
136
|
-
async runWithSelector(selector, dataset, distribution, writer, tally, options) {
|
|
141
|
+
async runWithSelector(selector, dataset, distribution, writer, tally, readOptions, options) {
|
|
137
142
|
// Peek the first batch to detect an empty selector before starting the
|
|
138
143
|
// writer (important because e.g. SparqlUpdateWriter does CLEAR GRAPH).
|
|
139
144
|
const batches = batch(selector, this.batchSize);
|
|
@@ -191,7 +196,7 @@ export class Stage {
|
|
|
191
196
|
track((async () => {
|
|
192
197
|
// Run all readers for this batch in parallel.
|
|
193
198
|
const readerOutputs = await Promise.all(this.readers.map(async (normalized) => {
|
|
194
|
-
const result = await this.read(normalized, dataset, distribution, {
|
|
199
|
+
const result = await this.read(normalized, dataset, distribution, { ...readOptions, bindings }, tally);
|
|
195
200
|
if (result instanceof NotSupported)
|
|
196
201
|
return [];
|
|
197
202
|
hasResults = true;
|
|
@@ -286,8 +291,8 @@ export class Stage {
|
|
|
286
291
|
// 'skip': discard
|
|
287
292
|
return [];
|
|
288
293
|
}
|
|
289
|
-
async readAll(dataset, distribution,
|
|
290
|
-
const results = await Promise.all(this.readers.map((normalized) => this.read(normalized, dataset, distribution,
|
|
294
|
+
async readAll(dataset, distribution, readOptions, tally) {
|
|
295
|
+
const results = await Promise.all(this.readers.map((normalized) => this.read(normalized, dataset, distribution, readOptions, tally)));
|
|
291
296
|
const chains = [];
|
|
292
297
|
for (const result of results) {
|
|
293
298
|
if (!(result instanceof NotSupported)) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.47.0",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -26,8 +26,9 @@
|
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@lde/dataset": "^0.10.1",
|
|
28
28
|
"@lde/dataset-registry-client": "^0.12.1",
|
|
29
|
-
"@lde/distribution-health": "^0.
|
|
30
|
-
"@lde/distribution-probe": "^0.
|
|
29
|
+
"@lde/distribution-health": "^0.9.0",
|
|
30
|
+
"@lde/distribution-probe": "^0.7.0",
|
|
31
|
+
"@lde/host-limiter": "^0.4.0",
|
|
31
32
|
"@lde/sparql-importer": "^0.11.1",
|
|
32
33
|
"@lde/sparql-server": "^0.6.0",
|
|
33
34
|
"@rdfjs/namespace": "^2.0.1",
|
|
@@ -40,7 +41,6 @@
|
|
|
40
41
|
"filenamify-url": "^4.0.0",
|
|
41
42
|
"is-network-error": "^1.3.2",
|
|
42
43
|
"n3": "^2.13.12",
|
|
43
|
-
"p-retry": "^8.0.1",
|
|
44
44
|
"rdf-string": "^2.0.1",
|
|
45
45
|
"tslib": "^2.3.0"
|
|
46
46
|
}
|