@lde/pipeline 0.6.30 → 0.6.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Dataset, Distribution } from '@lde/dataset';
|
|
2
2
|
import type { Importer } from '@lde/sparql-importer';
|
|
3
3
|
import { ImportFailed } from '@lde/sparql-importer';
|
|
4
|
+
import type { SparqlServer } from '@lde/sparql-server';
|
|
4
5
|
import { type ProbeResultType } from './probe.js';
|
|
5
6
|
export declare class ResolvedDistribution {
|
|
6
7
|
readonly distribution: Distribution;
|
|
@@ -16,9 +17,11 @@ export declare class NoDistributionAvailable {
|
|
|
16
17
|
}
|
|
17
18
|
export interface DistributionResolver {
|
|
18
19
|
resolve(dataset: Dataset): Promise<ResolvedDistribution | NoDistributionAvailable>;
|
|
20
|
+
cleanup?(): Promise<void>;
|
|
19
21
|
}
|
|
20
22
|
export interface SparqlDistributionResolverOptions {
|
|
21
23
|
importer?: Importer;
|
|
24
|
+
server?: SparqlServer;
|
|
22
25
|
timeout?: number;
|
|
23
26
|
}
|
|
24
27
|
/**
|
|
@@ -33,8 +36,10 @@ export interface SparqlDistributionResolverOptions {
|
|
|
33
36
|
*/
|
|
34
37
|
export declare class SparqlDistributionResolver implements DistributionResolver {
|
|
35
38
|
private readonly importer?;
|
|
39
|
+
private readonly server?;
|
|
36
40
|
private readonly timeout;
|
|
37
41
|
constructor(options?: SparqlDistributionResolverOptions);
|
|
38
42
|
resolve(dataset: Dataset): Promise<ResolvedDistribution | NoDistributionAvailable>;
|
|
43
|
+
cleanup(): Promise<void>;
|
|
39
44
|
}
|
|
40
45
|
//# sourceMappingURL=resolver.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAoB,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAA4B,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAE5E,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;gBAD/B,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE;CAE3C;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAoB,MAAM,sBAAsB,CAAC;AACtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAA4B,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAE5E,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;gBAD/B,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE;CAE3C;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAMjD,OAAO,CACX,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IA0DpD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
|
|
@@ -33,9 +33,11 @@ export class NoDistributionAvailable {
|
|
|
33
33
|
*/
|
|
34
34
|
export class SparqlDistributionResolver {
|
|
35
35
|
importer;
|
|
36
|
+
server;
|
|
36
37
|
timeout;
|
|
37
38
|
constructor(options) {
|
|
38
39
|
this.importer = options?.importer;
|
|
40
|
+
this.server = options?.server;
|
|
39
41
|
this.timeout = options?.timeout ?? 5000;
|
|
40
42
|
}
|
|
41
43
|
async resolve(dataset) {
|
|
@@ -54,6 +56,12 @@ export class SparqlDistributionResolver {
|
|
|
54
56
|
if (this.importer) {
|
|
55
57
|
const importResult = await this.importer.import(dataset);
|
|
56
58
|
if (importResult instanceof ImportSuccessful) {
|
|
59
|
+
// Start server if provided, using its query endpoint.
|
|
60
|
+
if (this.server) {
|
|
61
|
+
await this.server.start();
|
|
62
|
+
const distribution = Distribution.sparql(this.server.queryEndpoint, importResult.identifier);
|
|
63
|
+
return new ResolvedDistribution(distribution, results);
|
|
64
|
+
}
|
|
57
65
|
const distribution = Distribution.sparql(importResult.distribution.accessUrl, importResult.identifier);
|
|
58
66
|
return new ResolvedDistribution(distribution, results);
|
|
59
67
|
}
|
|
@@ -63,4 +71,7 @@ export class SparqlDistributionResolver {
|
|
|
63
71
|
}
|
|
64
72
|
return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available', results);
|
|
65
73
|
}
|
|
74
|
+
async cleanup() {
|
|
75
|
+
await this.server?.stop();
|
|
76
|
+
}
|
|
66
77
|
}
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAGpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;KACnD,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAmBD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAoB9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAGpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;KACnD,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAmBD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAoB9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;YA4Bd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -74,6 +74,9 @@ export class Pipeline {
|
|
|
74
74
|
catch {
|
|
75
75
|
// Stage error for this dataset; continue to next dataset.
|
|
76
76
|
}
|
|
77
|
+
finally {
|
|
78
|
+
await this.distributionResolver.cleanup?.();
|
|
79
|
+
}
|
|
77
80
|
this.reporter?.datasetComplete(datasetIri);
|
|
78
81
|
}
|
|
79
82
|
async runStage(dataset, distribution, stage) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.32",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "https://github.com/ldengine/lde",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -23,9 +23,10 @@
|
|
|
23
23
|
"!**/*.tsbuildinfo"
|
|
24
24
|
],
|
|
25
25
|
"dependencies": {
|
|
26
|
-
"@lde/dataset": "0.6.
|
|
27
|
-
"@lde/dataset-registry-client": "0.6.
|
|
28
|
-
"@lde/sparql-importer": "0.2.
|
|
26
|
+
"@lde/dataset": "0.6.10",
|
|
27
|
+
"@lde/dataset-registry-client": "0.6.17",
|
|
28
|
+
"@lde/sparql-importer": "0.2.10",
|
|
29
|
+
"@lde/sparql-server": "0.4.10",
|
|
29
30
|
"@rdfjs/types": "^2.0.1",
|
|
30
31
|
"fetch-sparql-endpoint": "^7.1.0",
|
|
31
32
|
"filenamify-url": "^3.0.0",
|