@lde/pipeline 0.32.0 → 0.33.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -295,7 +295,7 @@ After all stages for a dataset have run, the pipeline calls `validator.report(da
|
|
|
295
295
|
|
|
296
296
|
### Writer
|
|
297
297
|
|
|
298
|
-
Writes generated quads to a destination. A `Writer` is transactional: each pipeline run opens one run on it (`openRun(context)`), writes every dataset through the resulting `RunWriter`, and ends with exactly one `commit()` (on success) or `abort(error)` (on failure). The run lifecycle is the home of destination-level concerns such as atomic swaps, deletion sweeps and cross-pod locks; the pipeline drives `openRun → write* → commit/abort` uniformly and never branches on the writer’s update mode.
|
|
298
|
+
Writes generated quads to a destination. A `Writer` is transactional: each pipeline run opens one run on it (`openRun(context)`), writes every dataset through the resulting `RunWriter`, and ends with exactly one `commit()` (on success) or `abort(error)` (on failure). The run lifecycle is the home of destination-level concerns such as atomic swaps, deletion sweeps and cross-pod locks; the pipeline drives `openRun → write* → commit/abort` uniformly and never branches on the writer’s update mode. After a dataset’s stages complete, its `flush(dataset, outcome)` says whether the dataset succeeded, so a writer can gate destructive finalization (e.g. an In-place stale-document sweep) on `'success'`.
|
|
299
299
|
|
|
300
300
|
- `SparqlUpdateWriter` — writes to a SPARQL endpoint via UPDATE queries; writes are visible as they land, so `commit`/`abort` are no-ops
|
|
301
301
|
- `FileWriter` — writes to local files; each file is streamed to a sibling temp file and atomically renamed on flush, `commit` finalizes any files still open, and `abort` discards temp output
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAGrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAGrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;CAC3D;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AAqID,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe;IA+C9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAyCZ,cAAc;IAqN5B,+EAA+E;YACjE,aAAa;YAmBb,gBAAgB;IAW9B,OAAO,CAAE,aAAa;IAOtB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAMnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA4BlC;;;;;OAKG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YAuEP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -56,7 +56,18 @@ class FanOutWriter {
|
|
|
56
56
|
this.writers = writers;
|
|
57
57
|
}
|
|
58
58
|
async openRun(context) {
|
|
59
|
-
|
|
59
|
+
// Opening a writer's run can have side effects (a cross-pod lock, a fresh
|
|
60
|
+
// collection). If one writer opens but a sibling then fails, the pipeline
|
|
61
|
+
// never gets a RunWriter to abort, so roll the opened ones back here
|
|
62
|
+
// before rethrowing — otherwise their locks and collections leak.
|
|
63
|
+
const results = await Promise.allSettled(this.writers.map((writer) => writer.openRun(context)));
|
|
64
|
+
const opened = results.flatMap((result) => result.status === 'fulfilled' ? [result.value] : []);
|
|
65
|
+
const failure = results.find((result) => result.status === 'rejected');
|
|
66
|
+
if (failure) {
|
|
67
|
+
await Promise.allSettled(opened.map((run) => run.abort(failure.reason)));
|
|
68
|
+
throw failure.reason;
|
|
69
|
+
}
|
|
70
|
+
return new FanOutRunWriter(opened);
|
|
60
71
|
}
|
|
61
72
|
}
|
|
62
73
|
/**
|
|
@@ -78,9 +89,9 @@ class FanOutRunWriter {
|
|
|
78
89
|
const branches = tee(quads, this.runs.length);
|
|
79
90
|
await Promise.all(this.runs.map((run, index) => run.write(dataset, branches[index])));
|
|
80
91
|
}
|
|
81
|
-
async flush(dataset) {
|
|
92
|
+
async flush(dataset, outcome) {
|
|
82
93
|
for (const run of this.runs)
|
|
83
|
-
await run.flush?.(dataset);
|
|
94
|
+
await run.flush?.(dataset, outcome);
|
|
84
95
|
}
|
|
85
96
|
async reset(dataset) {
|
|
86
97
|
for (const run of this.runs)
|
|
@@ -334,7 +345,7 @@ export class Pipeline {
|
|
|
334
345
|
await this.distributionResolver.cleanup?.();
|
|
335
346
|
unsubscribe?.();
|
|
336
347
|
}
|
|
337
|
-
await runWriter.flush?.(dataset);
|
|
348
|
+
await runWriter.flush?.(dataset, stageFailed ? 'failed' : 'success');
|
|
338
349
|
await this.reportValidators(dataset);
|
|
339
350
|
// A dataset whose stages threw produced incomplete output; record it as
|
|
340
351
|
// ‘failed’ rather than freezing a broken result under a ‘success’ record.
|
|
@@ -486,7 +497,7 @@ export class Pipeline {
|
|
|
486
497
|
const scratchRun = await scratchWriter.openRun(context);
|
|
487
498
|
try {
|
|
488
499
|
await this.runChainedStage(dataset, stageDistribution, chainedStage, scratchRun, timeout);
|
|
489
|
-
await scratchRun.flush(dataset);
|
|
500
|
+
await scratchRun.flush(dataset, 'success');
|
|
490
501
|
await scratchRun.commit();
|
|
491
502
|
}
|
|
492
503
|
catch (error) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Dataset } from '@lde/dataset';
|
|
2
|
-
import { RunContext, RunWriter, Writer } from './writer.js';
|
|
2
|
+
import { DatasetOutcome, RunContext, RunWriter, Writer } from './writer.js';
|
|
3
3
|
export interface FileWriterOptions {
|
|
4
4
|
/**
|
|
5
5
|
* Output directory for written files.
|
|
@@ -33,10 +33,12 @@ export interface FileWriterOptions {
|
|
|
33
33
|
}
|
|
34
34
|
/**
|
|
35
35
|
* The run writer a {@link FileWriter} opens: per-dataset `flush` and `reset`
|
|
36
|
-
* are always available, so direct callers need no optional chaining.
|
|
36
|
+
* are always available, so direct callers need no optional chaining. The
|
|
37
|
+
* dataset outcome does not change what a flush does here – a failed dataset's
|
|
38
|
+
* partial output still materializes, matching the pre-transactional behaviour.
|
|
37
39
|
*/
|
|
38
40
|
export interface FileRunWriter extends RunWriter {
|
|
39
|
-
flush(dataset: Dataset): Promise<void>;
|
|
41
|
+
flush(dataset: Dataset, outcome?: DatasetOutcome): Promise<void>;
|
|
40
42
|
reset(dataset: Dataset): Promise<void>;
|
|
41
43
|
}
|
|
42
44
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAOvC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAOvC,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE5E,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,GAAG,CAAC;CACtC;AAgBD;;;;;GAKG;AACH,MAAM,WAAW,aAAc,SAAQ,SAAS;IAC9C,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjE,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACxC;AAED;;;;;;;;;;GAUG;AACH,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAyB;IACnD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA4B;gBAE1C,OAAO,EAAE,iBAAiB;IAQhC,OAAO,CAAC,QAAQ,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC;YAwB9C,UAAU;YAqCV,SAAS;YAOT,SAAS;YAgCT,WAAW;IAczB;;;;OAIG;YACW,cAAc;IAQ5B,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,WAAW;YAIL,eAAe;IA6B7B,OAAO,CAAC,YAAY;CAUrB"}
|
package/dist/writer/writer.d.ts
CHANGED
|
@@ -44,6 +44,12 @@ export interface DatasetWriter<Item = Quad> {
|
|
|
44
44
|
*/
|
|
45
45
|
write(dataset: Dataset, items: AsyncIterable<Item>): Promise<void>;
|
|
46
46
|
}
|
|
47
|
+
/**
|
|
48
|
+
* How a dataset’s processing ended, handed to {@link RunWriter.flush}:
|
|
49
|
+
* `'success'` when every stage completed, `'failed'` when a stage hard-failed
|
|
50
|
+
* and the dataset’s output is incomplete.
|
|
51
|
+
*/
|
|
52
|
+
export type DatasetOutcome = 'success' | 'failed';
|
|
47
53
|
/**
|
|
48
54
|
* One open run transaction on a destination: per-dataset writes bracketed by
|
|
49
55
|
* exactly one {@link commit} or {@link abort}. Obtained from
|
|
@@ -55,13 +61,16 @@ export interface DatasetWriter<Item = Quad> {
|
|
|
55
61
|
export interface RunWriter<Item = Quad> extends DatasetWriter<Item> {
|
|
56
62
|
/**
|
|
57
63
|
* Finalize writing for a dataset. Called after all stages complete for that
|
|
58
|
-
* dataset
|
|
64
|
+
* dataset – successful or not; `outcome` says which, so a writer can gate
|
|
65
|
+
* destructive finalization (an In-place writer’s stale-document sweep) on
|
|
66
|
+
* `'success'`: a failed dataset’s output is incomplete, and sweeping against
|
|
67
|
+
* it would delete data the run never got to rewrite.
|
|
59
68
|
*
|
|
60
69
|
* Writers that buffer output across multiple {@link write} calls (e.g. to
|
|
61
70
|
* share Turtle prefix declarations) should implement this to flush remaining
|
|
62
71
|
* data and release per-dataset resources.
|
|
63
72
|
*/
|
|
64
|
-
flush?(dataset: Dataset): Promise<void>;
|
|
73
|
+
flush?(dataset: Dataset, outcome: DatasetOutcome): Promise<void>;
|
|
65
74
|
/**
|
|
66
75
|
* Discard a dataset’s already-written output so a subsequent pass starts
|
|
67
76
|
* from a clean slate. Called by the pipeline before it re-runs all stages
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;OAKG;IACH,eAAe,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;CACvC;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,GAAG,IAAI;IACxC;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS,CAAC,IAAI,GAAG,IAAI,CAAE,SAAQ,aAAa,CAAC,IAAI,CAAC;IACjE
|
|
1
|
+
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;OAKG;IACH,eAAe,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;CACvC;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,GAAG,IAAI;IACxC;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,QAAQ,CAAC;AAElD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS,CAAC,IAAI,GAAG,IAAI,CAAE,SAAQ,aAAa,CAAC,IAAI,CAAC;IACjE;;;;;;;;;;OAUG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjE;;;;;;;;;OASG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExC;;;;;;;;;OASG;IACH,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAExB;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,MAAM,CAAC,IAAI,GAAG,IAAI;IACjC;;;;OAIG;IACH,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;CACxD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.33.1",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@lde/dataset": "^0.7.8",
|
|
28
28
|
"@lde/dataset-registry-client": "^0.8.5",
|
|
29
|
-
"@lde/distribution-health": "^0.2.
|
|
30
|
-
"@lde/distribution-probe": "^0.2.
|
|
29
|
+
"@lde/distribution-health": "^0.2.4",
|
|
30
|
+
"@lde/distribution-probe": "^0.2.4",
|
|
31
31
|
"@lde/sparql-importer": "^0.6.6",
|
|
32
32
|
"@lde/sparql-server": "^0.4.12",
|
|
33
33
|
"@rdfjs/namespace": "^2.0.1",
|