@lde/pipeline 0.24.4 → 0.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAGvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;YAgB5C,aAAa;IAwErB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Distribution } from '@lde/dataset';
|
|
2
2
|
import { ImportFailed, ImportSuccessful } from '@lde/sparql-importer';
|
|
3
3
|
import { NoDistributionAvailable, ResolvedDistribution, } from './resolver.js';
|
|
4
|
+
import { NetworkError } from './probe.js';
|
|
4
5
|
/**
|
|
5
6
|
* A {@link DistributionResolver} decorator that adds data-dump import logic.
|
|
6
7
|
*
|
|
@@ -28,11 +29,26 @@ export class ImportResolver {
|
|
|
28
29
|
return this.importDataset(dataset, result.probeResults, callbacks);
|
|
29
30
|
}
|
|
30
31
|
async importDataset(dataset, probeResults, callbacks) {
|
|
32
|
+
const successfulUrls = new Set(probeResults
|
|
33
|
+
.filter((r) => !(r instanceof NetworkError) && r.isSuccess())
|
|
34
|
+
.map((r) => r.url));
|
|
35
|
+
const candidates = dataset
|
|
36
|
+
.getDownloadDistributions()
|
|
37
|
+
.filter((d) => d.accessUrl && successfulUrls.has(d.accessUrl.toString()));
|
|
38
|
+
if (candidates.length === 0) {
|
|
39
|
+
return new NoDistributionAvailable(dataset, 'No importable distributions passed probing', probeResults);
|
|
40
|
+
}
|
|
31
41
|
const importStart = Date.now();
|
|
32
42
|
callbacks?.onImportStart?.();
|
|
33
|
-
const importResult = await this.options.importer.import(
|
|
43
|
+
const importResult = await this.options.importer.import(candidates);
|
|
34
44
|
if (importResult instanceof ImportSuccessful) {
|
|
35
|
-
|
|
45
|
+
try {
|
|
46
|
+
await this.options.server.start();
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
callbacks?.onImportFailed?.(importResult.distribution, error instanceof Error ? error.message : String(error));
|
|
50
|
+
return new NoDistributionAvailable(dataset, 'SPARQL server failed to start after import', probeResults);
|
|
51
|
+
}
|
|
36
52
|
const distribution = Distribution.sparql(this.options.server.queryEndpoint, importResult.identifier);
|
|
37
53
|
distribution.subjectFilter = importResult.distribution.subjectFilter;
|
|
38
54
|
return new ResolvedDistribution(distribution, probeResults, importResult.distribution, Date.now() - importStart, importResult.tripleCount);
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAsCD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAsCD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBZ,cAAc;IAgE5B;;;OAGG;YACW,QAAQ;IA2CtB,2EAA2E;YAC7D,eAAe;YAcf,QAAQ;YAmDP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -80,21 +80,31 @@ export class Pipeline {
|
|
|
80
80
|
for await (const dataset of datasets) {
|
|
81
81
|
await this.processDataset(dataset);
|
|
82
82
|
}
|
|
83
|
-
this.reporter?.pipelineComplete?.({
|
|
83
|
+
this.reporter?.pipelineComplete?.({
|
|
84
|
+
duration: Date.now() - start,
|
|
85
|
+
memoryUsageBytes: process.memoryUsage().rss,
|
|
86
|
+
});
|
|
84
87
|
}
|
|
85
88
|
async processDataset(dataset) {
|
|
86
89
|
this.reporter?.datasetStart?.(dataset);
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
90
|
+
let resolved;
|
|
91
|
+
try {
|
|
92
|
+
resolved = await this.distributionResolver.resolve(dataset, {
|
|
93
|
+
onProbe: (distribution, result) => {
|
|
94
|
+
this.reporter?.distributionProbed?.(mapProbeResult(distribution, result));
|
|
95
|
+
},
|
|
96
|
+
onImportStart: () => {
|
|
97
|
+
this.reporter?.importStarted?.();
|
|
98
|
+
},
|
|
99
|
+
onImportFailed: (distribution, error) => {
|
|
100
|
+
this.reporter?.importFailed?.(distribution, error);
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
catch (error) {
|
|
105
|
+
this.reporter?.datasetSkipped?.(dataset, `Distribution resolution failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
98
108
|
if (resolved instanceof NoDistributionAvailable) {
|
|
99
109
|
this.reporter?.datasetSkipped?.(dataset, resolved.message);
|
|
100
110
|
return;
|
|
@@ -119,7 +129,9 @@ export class Pipeline {
|
|
|
119
129
|
await this.distributionResolver.cleanup?.();
|
|
120
130
|
}
|
|
121
131
|
await this.writer.flush?.(dataset);
|
|
122
|
-
this.reporter?.datasetComplete?.(dataset
|
|
132
|
+
this.reporter?.datasetComplete?.(dataset, {
|
|
133
|
+
memoryUsageBytes: process.memoryUsage().rss,
|
|
134
|
+
});
|
|
123
135
|
}
|
|
124
136
|
/**
|
|
125
137
|
* Run a stage with reporting and return whether it was supported.
|
|
@@ -134,7 +146,11 @@ export class Pipeline {
|
|
|
134
146
|
onProgress: (items, quads) => {
|
|
135
147
|
itemsProcessed = items;
|
|
136
148
|
quadsGenerated = quads;
|
|
137
|
-
this.reporter?.stageProgress?.({
|
|
149
|
+
this.reporter?.stageProgress?.({
|
|
150
|
+
itemsProcessed,
|
|
151
|
+
quadsGenerated,
|
|
152
|
+
memoryUsageBytes: process.memoryUsage().rss,
|
|
153
|
+
});
|
|
138
154
|
},
|
|
139
155
|
});
|
|
140
156
|
if (result instanceof NotSupported) {
|
|
@@ -22,6 +22,7 @@ export interface ProgressReporter {
|
|
|
22
22
|
stageProgress?(update: {
|
|
23
23
|
itemsProcessed: number;
|
|
24
24
|
quadsGenerated: number;
|
|
25
|
+
memoryUsageBytes: number;
|
|
25
26
|
}): void;
|
|
26
27
|
stageComplete?(stage: string, result: {
|
|
27
28
|
itemsProcessed: number;
|
|
@@ -32,10 +33,13 @@ export interface ProgressReporter {
|
|
|
32
33
|
/** Called after a stage completes if it has a validator. */
|
|
33
34
|
stageValidated?(stage: string, report: ValidationReport): void;
|
|
34
35
|
stageSkipped?(stage: string, reason: string): void;
|
|
35
|
-
datasetComplete?(dataset: Dataset
|
|
36
|
+
datasetComplete?(dataset: Dataset, result: {
|
|
37
|
+
memoryUsageBytes: number;
|
|
38
|
+
}): void;
|
|
36
39
|
datasetSkipped?(dataset: Dataset, reason: string): void;
|
|
37
40
|
pipelineComplete?(result: {
|
|
38
41
|
duration: number;
|
|
42
|
+
memoryUsageBytes: number;
|
|
39
43
|
}): void;
|
|
40
44
|
}
|
|
41
45
|
//# sourceMappingURL=progressReporter.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,4DAA4D;IAC5D,cAAc,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC/D,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD,eAAe,CAAC,CACd,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAA;KAAE,GACnC,IAAI,CAAC;IACR,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,GAAG,IAAI,CAAC;CACV"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.25.1",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"dependencies": {
|
|
26
26
|
"@lde/dataset": "0.7.2",
|
|
27
27
|
"@lde/dataset-registry-client": "0.7.4",
|
|
28
|
-
"@lde/sparql-importer": "0.4.
|
|
28
|
+
"@lde/sparql-importer": "0.4.2",
|
|
29
29
|
"@lde/sparql-server": "0.4.10",
|
|
30
30
|
"@rdfjs/types": "^2.0.1",
|
|
31
31
|
"@traqula/generator-sparql-1-1": "^1.0.3",
|