@lde/pipeline 0.25.1 → 0.25.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAMrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAGvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;YAgB5C,aAAa;IAkFrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Distribution } from '@lde/dataset';
|
|
2
|
-
import { ImportFailed, ImportSuccessful } from '@lde/sparql-importer';
|
|
2
|
+
import { ImportFailed, ImportSuccessful, NotSupported, } from '@lde/sparql-importer';
|
|
3
3
|
import { NoDistributionAvailable, ResolvedDistribution, } from './resolver.js';
|
|
4
4
|
import { NetworkError } from './probe.js';
|
|
5
5
|
/**
|
|
@@ -56,6 +56,11 @@ export class ImportResolver {
|
|
|
56
56
|
if (importResult instanceof ImportFailed) {
|
|
57
57
|
callbacks?.onImportFailed?.(importResult.distribution, importResult.error);
|
|
58
58
|
}
|
|
59
|
+
if (importResult instanceof NotSupported) {
|
|
60
|
+
const failedDistribution = importResult.distribution ?? candidates[0];
|
|
61
|
+
callbacks?.onImportFailed?.(failedDistribution, 'No supported import format');
|
|
62
|
+
return new NoDistributionAvailable(dataset, 'No supported import format available', probeResults);
|
|
63
|
+
}
|
|
59
64
|
return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available', probeResults, importResult instanceof ImportFailed ? importResult : undefined);
|
|
60
65
|
}
|
|
61
66
|
async cleanup() {
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAgED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoBZ,cAAc;IAkE5B;;;OAGG;YACW,QAAQ;IA6CtB,2EAA2E;YAC7D,eAAe;YAcf,QAAQ;YAmDP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -5,20 +5,41 @@ import { NoDistributionAvailable, } from './distribution/resolver.js';
|
|
|
5
5
|
import { SparqlDistributionResolver } from './distribution/index.js';
|
|
6
6
|
import { NetworkError, SparqlProbeResult, } from './distribution/probe.js';
|
|
7
7
|
import { NotSupported } from './sparql/executor.js';
|
|
8
|
+
/**
|
|
9
|
+
* Split an async iterable into `count` branches that can be consumed
|
|
10
|
+
* independently. Backpressure is enforced by the slowest consumer —
|
|
11
|
+
* the source only advances once every branch has consumed the current item.
|
|
12
|
+
*/
|
|
13
|
+
function tee(source, count) {
|
|
14
|
+
const iterator = source[Symbol.asyncIterator]();
|
|
15
|
+
let current;
|
|
16
|
+
let consumed = 0;
|
|
17
|
+
function advance() {
|
|
18
|
+
if (!current || consumed >= count) {
|
|
19
|
+
consumed = 0;
|
|
20
|
+
current = iterator.next();
|
|
21
|
+
}
|
|
22
|
+
consumed++;
|
|
23
|
+
return current;
|
|
24
|
+
}
|
|
25
|
+
return Array.from({ length: count }, () => ({
|
|
26
|
+
[Symbol.asyncIterator]() {
|
|
27
|
+
return {
|
|
28
|
+
async next() {
|
|
29
|
+
return advance();
|
|
30
|
+
},
|
|
31
|
+
};
|
|
32
|
+
},
|
|
33
|
+
}));
|
|
34
|
+
}
|
|
8
35
|
class FanOutWriter {
|
|
9
36
|
writers;
|
|
10
37
|
constructor(writers) {
|
|
11
38
|
this.writers = writers;
|
|
12
39
|
}
|
|
13
40
|
async write(dataset, quads) {
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
collected.push(quad);
|
|
17
|
-
for (const w of this.writers) {
|
|
18
|
-
await w.write(dataset, (async function* () {
|
|
19
|
-
yield* collected;
|
|
20
|
-
})());
|
|
21
|
-
}
|
|
41
|
+
const branches = tee(quads, this.writers.length);
|
|
42
|
+
await Promise.all(this.writers.map((writer, index) => writer.write(dataset, branches[index])));
|
|
22
43
|
}
|
|
23
44
|
async flush(dataset) {
|
|
24
45
|
for (const w of this.writers)
|
|
@@ -80,9 +101,11 @@ export class Pipeline {
|
|
|
80
101
|
for await (const dataset of datasets) {
|
|
81
102
|
await this.processDataset(dataset);
|
|
82
103
|
}
|
|
104
|
+
const finalMemory = process.memoryUsage();
|
|
83
105
|
this.reporter?.pipelineComplete?.({
|
|
84
106
|
duration: Date.now() - start,
|
|
85
|
-
memoryUsageBytes:
|
|
107
|
+
memoryUsageBytes: finalMemory.rss,
|
|
108
|
+
heapUsedBytes: finalMemory.heapUsed,
|
|
86
109
|
});
|
|
87
110
|
}
|
|
88
111
|
async processDataset(dataset) {
|
|
@@ -129,8 +152,10 @@ export class Pipeline {
|
|
|
129
152
|
await this.distributionResolver.cleanup?.();
|
|
130
153
|
}
|
|
131
154
|
await this.writer.flush?.(dataset);
|
|
155
|
+
const datasetMemory = process.memoryUsage();
|
|
132
156
|
this.reporter?.datasetComplete?.(dataset, {
|
|
133
|
-
memoryUsageBytes:
|
|
157
|
+
memoryUsageBytes: datasetMemory.rss,
|
|
158
|
+
heapUsedBytes: datasetMemory.heapUsed,
|
|
134
159
|
});
|
|
135
160
|
}
|
|
136
161
|
/**
|
|
@@ -146,10 +171,12 @@ export class Pipeline {
|
|
|
146
171
|
onProgress: (items, quads) => {
|
|
147
172
|
itemsProcessed = items;
|
|
148
173
|
quadsGenerated = quads;
|
|
174
|
+
const stageMemory = process.memoryUsage();
|
|
149
175
|
this.reporter?.stageProgress?.({
|
|
150
176
|
itemsProcessed,
|
|
151
177
|
quadsGenerated,
|
|
152
|
-
memoryUsageBytes:
|
|
178
|
+
memoryUsageBytes: stageMemory.rss,
|
|
179
|
+
heapUsedBytes: stageMemory.heapUsed,
|
|
153
180
|
});
|
|
154
181
|
},
|
|
155
182
|
});
|
|
@@ -23,6 +23,7 @@ export interface ProgressReporter {
|
|
|
23
23
|
itemsProcessed: number;
|
|
24
24
|
quadsGenerated: number;
|
|
25
25
|
memoryUsageBytes: number;
|
|
26
|
+
heapUsedBytes: number;
|
|
26
27
|
}): void;
|
|
27
28
|
stageComplete?(stage: string, result: {
|
|
28
29
|
itemsProcessed: number;
|
|
@@ -35,11 +36,13 @@ export interface ProgressReporter {
|
|
|
35
36
|
stageSkipped?(stage: string, reason: string): void;
|
|
36
37
|
datasetComplete?(dataset: Dataset, result: {
|
|
37
38
|
memoryUsageBytes: number;
|
|
39
|
+
heapUsedBytes: number;
|
|
38
40
|
}): void;
|
|
39
41
|
datasetSkipped?(dataset: Dataset, reason: string): void;
|
|
40
42
|
pipelineComplete?(result: {
|
|
41
43
|
duration: number;
|
|
42
44
|
memoryUsageBytes: number;
|
|
45
|
+
heapUsedBytes: number;
|
|
43
46
|
}): void;
|
|
44
47
|
}
|
|
45
48
|
//# sourceMappingURL=progressReporter.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,4DAA4D;IAC5D,cAAc,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC/D,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD,eAAe,CAAC,CACd,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,GAC1D,IAAI,CAAC;IACR,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;CACV"}
|