@lde/pipeline 0.25.0 → 0.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAGvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;YAgB5C,aAAa;IAwErB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Distribution } from '@lde/dataset';
|
|
2
2
|
import { ImportFailed, ImportSuccessful } from '@lde/sparql-importer';
|
|
3
3
|
import { NoDistributionAvailable, ResolvedDistribution, } from './resolver.js';
|
|
4
|
+
import { NetworkError } from './probe.js';
|
|
4
5
|
/**
|
|
5
6
|
* A {@link DistributionResolver} decorator that adds data-dump import logic.
|
|
6
7
|
*
|
|
@@ -28,11 +29,26 @@ export class ImportResolver {
|
|
|
28
29
|
return this.importDataset(dataset, result.probeResults, callbacks);
|
|
29
30
|
}
|
|
30
31
|
async importDataset(dataset, probeResults, callbacks) {
|
|
32
|
+
const successfulUrls = new Set(probeResults
|
|
33
|
+
.filter((r) => !(r instanceof NetworkError) && r.isSuccess())
|
|
34
|
+
.map((r) => r.url));
|
|
35
|
+
const candidates = dataset
|
|
36
|
+
.getDownloadDistributions()
|
|
37
|
+
.filter((d) => d.accessUrl && successfulUrls.has(d.accessUrl.toString()));
|
|
38
|
+
if (candidates.length === 0) {
|
|
39
|
+
return new NoDistributionAvailable(dataset, 'No importable distributions passed probing', probeResults);
|
|
40
|
+
}
|
|
31
41
|
const importStart = Date.now();
|
|
32
42
|
callbacks?.onImportStart?.();
|
|
33
|
-
const importResult = await this.options.importer.import(
|
|
43
|
+
const importResult = await this.options.importer.import(candidates);
|
|
34
44
|
if (importResult instanceof ImportSuccessful) {
|
|
35
|
-
|
|
45
|
+
try {
|
|
46
|
+
await this.options.server.start();
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
callbacks?.onImportFailed?.(importResult.distribution, error instanceof Error ? error.message : String(error));
|
|
50
|
+
return new NoDistributionAvailable(dataset, 'SPARQL server failed to start after import', probeResults);
|
|
51
|
+
}
|
|
36
52
|
const distribution = Distribution.sparql(this.options.server.queryEndpoint, importResult.identifier);
|
|
37
53
|
distribution.subjectFilter = importResult.distribution.subjectFilter;
|
|
38
54
|
return new ResolvedDistribution(distribution, probeResults, importResult.distribution, Date.now() - importStart, importResult.tripleCount);
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAsCD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBZ,cAAc;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAsCD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBZ,cAAc;IAgE5B;;;OAGG;YACW,QAAQ;IA2CtB,2EAA2E;YAC7D,eAAe;YAcf,QAAQ;YAmDP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -87,17 +87,24 @@ export class Pipeline {
|
|
|
87
87
|
}
|
|
88
88
|
async processDataset(dataset) {
|
|
89
89
|
this.reporter?.datasetStart?.(dataset);
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
90
|
+
let resolved;
|
|
91
|
+
try {
|
|
92
|
+
resolved = await this.distributionResolver.resolve(dataset, {
|
|
93
|
+
onProbe: (distribution, result) => {
|
|
94
|
+
this.reporter?.distributionProbed?.(mapProbeResult(distribution, result));
|
|
95
|
+
},
|
|
96
|
+
onImportStart: () => {
|
|
97
|
+
this.reporter?.importStarted?.();
|
|
98
|
+
},
|
|
99
|
+
onImportFailed: (distribution, error) => {
|
|
100
|
+
this.reporter?.importFailed?.(distribution, error);
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
catch (error) {
|
|
105
|
+
this.reporter?.datasetSkipped?.(dataset, `Distribution resolution failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
101
108
|
if (resolved instanceof NoDistributionAvailable) {
|
|
102
109
|
this.reporter?.datasetSkipped?.(dataset, resolved.message);
|
|
103
110
|
return;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.25.
|
|
3
|
+
"version": "0.25.1",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"dependencies": {
|
|
26
26
|
"@lde/dataset": "0.7.2",
|
|
27
27
|
"@lde/dataset-registry-client": "0.7.4",
|
|
28
|
-
"@lde/sparql-importer": "0.4.
|
|
28
|
+
"@lde/sparql-importer": "0.4.2",
|
|
29
29
|
"@lde/sparql-server": "0.4.10",
|
|
30
30
|
"@rdfjs/types": "^2.0.1",
|
|
31
31
|
"@traqula/generator-sparql-1-1": "^1.0.3",
|