@lde/pipeline 0.30.15 → 0.30.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAGrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAG1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAGrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAG1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;CAC3D;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AAkFD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe;IA2C9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoBZ,cAAc;IA+K5B,+EAA+E;YACjE,aAAa;YAmBb,gBAAgB;IAW9B,OAAO,CAAE,aAAa;IAOtB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAMnB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA+DP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -6,6 +6,8 @@ import { SparqlDistributionResolver } from './distribution/index.js';
|
|
|
6
6
|
import { sourceFingerprint } from './provenance/sourceFingerprint.js';
|
|
7
7
|
import { shouldReprocess } from './provenance/reprocessDecision.js';
|
|
8
8
|
import { NetworkError, SparqlProbeResult, } from '@lde/distribution-probe';
|
|
9
|
+
import { ImportSuccessful } from '@lde/sparql-importer';
|
|
10
|
+
import { importOutcomeToVerdict, probeResultToVerdict, } from '@lde/distribution-health';
|
|
9
11
|
import { NotSupported } from './sparql/executor.js';
|
|
10
12
|
import { ConstantTimeoutPolicy, } from './sparql/timeoutPolicy.js';
|
|
11
13
|
/**
|
|
@@ -141,6 +143,13 @@ export class Pipeline {
|
|
|
141
143
|
probed = await this.distributionResolver.probe(dataset, {
|
|
142
144
|
onProbe: (distribution, result) => {
|
|
143
145
|
this.reporter?.distributionProbed?.(mapProbeResult(distribution, result));
|
|
146
|
+
// Shallow validity: the probe parse-validates small RDF bodies as a
|
|
147
|
+
// by-product. Surface that verdict per distribution, judged against
|
|
148
|
+
// the distribution's own observed fingerprint.
|
|
149
|
+
const verdict = probeResultToVerdict(result, sourceFingerprint(distribution, result));
|
|
150
|
+
if (verdict) {
|
|
151
|
+
this.reporter?.distributionValidated?.(distribution, verdict);
|
|
152
|
+
}
|
|
144
153
|
},
|
|
145
154
|
});
|
|
146
155
|
}
|
|
@@ -190,6 +199,13 @@ export class Pipeline {
|
|
|
190
199
|
return;
|
|
191
200
|
}
|
|
192
201
|
if (resolved instanceof NoDistributionAvailable) {
|
|
202
|
+
// A failed import is a deep RDF-validity verdict on the distribution
|
|
203
|
+
// attempted. Surface it per distribution even though the dataset produces
|
|
204
|
+
// no summary, so an invalid distribution is recorded rather than silently
|
|
205
|
+
// dropped.
|
|
206
|
+
if (resolved.importFailed) {
|
|
207
|
+
this.reporter?.distributionValidated?.(resolved.importFailed.distribution, importOutcomeToVerdict(resolved.importFailed, fingerprint));
|
|
208
|
+
}
|
|
193
209
|
// Record the failure so a dataset whose source is unchanged is not
|
|
194
210
|
// re-imported every run; it is retried at the next fingerprint change or
|
|
195
211
|
// version rotation.
|
|
@@ -198,6 +214,12 @@ export class Pipeline {
|
|
|
198
214
|
return;
|
|
199
215
|
}
|
|
200
216
|
this.reporter?.distributionSelected?.(dataset, resolved.distribution, resolved.importedFrom, resolved.importDuration, resolved.tripleCount);
|
|
217
|
+
// A completed data-dump import is a deep validity verdict on the imported
|
|
218
|
+
// distribution (valid, or empty when it yielded no triples). Native SPARQL
|
|
219
|
+
// endpoints are not imported, so they carry no deep verdict.
|
|
220
|
+
if (resolved.importedFrom) {
|
|
221
|
+
this.reporter?.distributionValidated?.(resolved.importedFrom, importOutcomeToVerdict(new ImportSuccessful(resolved.importedFrom, undefined, resolved.tripleCount), fingerprint));
|
|
222
|
+
}
|
|
201
223
|
const timeout = this.timeoutFactory();
|
|
202
224
|
const unsubscribe = timeout.subscribe?.({
|
|
203
225
|
onTighten: (event) => this.reporter?.timeoutTightened?.(event),
|
|
@@ -372,6 +394,7 @@ export class Pipeline {
|
|
|
372
394
|
}
|
|
373
395
|
}
|
|
374
396
|
function mapProbeResult(distribution, result) {
|
|
397
|
+
const fingerprint = sourceFingerprint(distribution, result);
|
|
375
398
|
if (result instanceof NetworkError) {
|
|
376
399
|
return {
|
|
377
400
|
distribution,
|
|
@@ -379,6 +402,7 @@ function mapProbeResult(distribution, result) {
|
|
|
379
402
|
available: false,
|
|
380
403
|
error: result.message,
|
|
381
404
|
warnings: [],
|
|
405
|
+
fingerprint,
|
|
382
406
|
};
|
|
383
407
|
}
|
|
384
408
|
return {
|
|
@@ -390,5 +414,6 @@ function mapProbeResult(distribution, result) {
|
|
|
390
414
|
statusCode: result.statusCode,
|
|
391
415
|
error: result.failureReason ?? undefined,
|
|
392
416
|
warnings: result.warnings,
|
|
417
|
+
fingerprint,
|
|
393
418
|
};
|
|
394
419
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Dataset, Distribution } from '@lde/dataset';
|
|
2
|
+
import type { ValidityVerdict } from '@lde/distribution-health';
|
|
2
3
|
import type { ValidationReport } from './validator.js';
|
|
3
4
|
import type { TimeoutTransitionEvent } from './sparql/timeoutPolicy.js';
|
|
4
5
|
export interface DistributionAnalysisResult {
|
|
@@ -8,6 +9,14 @@ export interface DistributionAnalysisResult {
|
|
|
8
9
|
statusCode?: number;
|
|
9
10
|
error?: string;
|
|
10
11
|
warnings: string[];
|
|
12
|
+
/**
|
|
13
|
+
* The source-change fingerprint observed for this distribution, or `null`
|
|
14
|
+
* when none could be established (e.g. a live SPARQL endpoint). The shared
|
|
15
|
+
* key against which a validity verdict's `validatedFingerprint` is matched,
|
|
16
|
+
* so a consumer can record it on the reachability rail. Always populated by
|
|
17
|
+
* the pipeline; optional so consumers constructing the result need not set it.
|
|
18
|
+
*/
|
|
19
|
+
fingerprint?: string | null;
|
|
11
20
|
}
|
|
12
21
|
export interface ProgressReporter {
|
|
13
22
|
pipelineStart?(name: string): void;
|
|
@@ -19,6 +28,13 @@ export interface ProgressReporter {
|
|
|
19
28
|
importStarted?(): void;
|
|
20
29
|
/** Called when importing a distribution fails. */
|
|
21
30
|
importFailed?(distribution: Distribution, error: string): void;
|
|
31
|
+
/**
|
|
32
|
+
* Called with the RDF-validity verdict for a distribution the pipeline
|
|
33
|
+
* attempted, whether valid or invalid, and even when the dataset is
|
|
34
|
+
* otherwise skipped (no summary produced). The verdict is a plain
|
|
35
|
+
* TypeScript value; consumers decide how (and whether) to record it as RDF.
|
|
36
|
+
*/
|
|
37
|
+
distributionValidated?(distribution: Distribution, verdict: ValidityVerdict): void;
|
|
22
38
|
distributionSelected?(dataset: Dataset, distribution: Distribution, importedFrom?: Distribution, importDuration?: number, tripleCount?: number): void;
|
|
23
39
|
stageStart?(stage: string): void;
|
|
24
40
|
stageProgress?(update: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D;;;;;OAKG;IACH,qBAAqB,CAAC,CACpB,YAAY,EAAE,YAAY,EAC1B,OAAO,EAAE,eAAe,GACvB,IAAI,CAAC;IACR,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD;;;;;;;;OAQG;IACH,gBAAgB,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACpE,eAAe,CAAC,CACd,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,GAC1D,IAAI,CAAC;IACR,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT;;;;OAIG;IACH,gBAAgB,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACvD;;;OAGG;IACH,cAAc,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;CACtD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.30.
|
|
3
|
+
"version": "0.30.17",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@lde/dataset": "0.7.7",
|
|
28
28
|
"@lde/dataset-registry-client": "0.8.3",
|
|
29
|
+
"@lde/distribution-health": "0.1.2",
|
|
29
30
|
"@lde/distribution-probe": "0.1.10",
|
|
30
31
|
"@lde/sparql-importer": "0.6.5",
|
|
31
32
|
"@lde/sparql-server": "0.4.11",
|