@lde/pipeline 0.35.5 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/combineReporters.js +1 -0
- package/dist/pipeline.d.ts +15 -0
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +41 -1
- package/dist/progressReporter.d.ts +7 -1
- package/dist/progressReporter.d.ts.map +1 -1
- package/dist/writer/writer.d.ts +22 -0
- package/dist/writer/writer.d.ts.map +1 -1
- package/package.json +3 -3
package/dist/combineReporters.js
CHANGED
|
@@ -32,6 +32,7 @@ export function combineReporters(reporters) {
|
|
|
32
32
|
const combined = {
|
|
33
33
|
pipelineStart: (...args) => forward('pipelineStart', ...args),
|
|
34
34
|
datasetsSelected: (...args) => forward('datasetsSelected', ...args),
|
|
35
|
+
selectionEmpty: (...args) => forward('selectionEmpty', ...args),
|
|
35
36
|
datasetStart: (...args) => forward('datasetStart', ...args),
|
|
36
37
|
distributionProbed: (...args) => forward('distributionProbed', ...args),
|
|
37
38
|
importStarted: (...args) => forward('importStarted', ...args),
|
package/dist/pipeline.d.ts
CHANGED
|
@@ -9,6 +9,21 @@ import type { ProvenanceStore } from './provenance/store.js';
|
|
|
9
9
|
import type { StageOutputResolver } from './stageOutputResolver.js';
|
|
10
10
|
import type { ProgressReporter } from './progressReporter.js';
|
|
11
11
|
import { type TimeoutPolicy } from './sparql/timeoutPolicy.js';
|
|
12
|
+
/**
|
|
13
|
+
* Why a run was abandoned rather than committed: its selector produced no
|
|
14
|
+
* datasets, so the run holds no membership a writer could safely act on.
|
|
15
|
+
*
|
|
16
|
+
* Every writer reads the run’s selection as the authoritative membership of its
|
|
17
|
+
* destination – that is what lets an In-place writer sweep departed sources and
|
|
18
|
+
* a Blue/green writer ship a fresh collection. An empty selection would make
|
|
19
|
+
* that reading catastrophic (delete everything / swap in an empty collection),
|
|
20
|
+
* and it is indistinguishable from a registry outage or a criteria typo, so the
|
|
21
|
+
* pipeline aborts instead. Handed to {@link RunWriter.abort}, whose contract –
|
|
22
|
+
* leave the live destination as it was – is exactly the wanted behaviour.
|
|
23
|
+
*/
|
|
24
|
+
export declare class EmptySelection extends Error {
|
|
25
|
+
constructor();
|
|
26
|
+
}
|
|
12
27
|
/**
|
|
13
28
|
* Context handed to a {@link PipelinePlugin.beforeStageWrite} transform: the
|
|
14
29
|
* `dataset` whose merged output is being written and the `stage` that produced
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA6DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA6DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAiFZ,cAAc;IAgO5B,+EAA+E;YACjE,aAAa;YAyBb,gBAAgB;IAa9B,OAAO,CAAE,aAAa;IAStB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA4BlC;;;;;OAKG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA8EP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -13,6 +13,24 @@ import { importOutcomeToVerdict, probeResultToVerdict, } from '@lde/distribution
|
|
|
13
13
|
import { NotSupported } from './sparql/reader.js';
|
|
14
14
|
import { combineReporters } from './combineReporters.js';
|
|
15
15
|
import { ConstantTimeoutPolicy, } from './sparql/timeoutPolicy.js';
|
|
16
|
+
/**
|
|
17
|
+
* Why a run was abandoned rather than committed: its selector produced no
|
|
18
|
+
* datasets, so the run holds no membership a writer could safely act on.
|
|
19
|
+
*
|
|
20
|
+
* Every writer reads the run’s selection as the authoritative membership of its
|
|
21
|
+
* destination – that is what lets an In-place writer sweep departed sources and
|
|
22
|
+
* a Blue/green writer ship a fresh collection. An empty selection would make
|
|
23
|
+
* that reading catastrophic (delete everything / swap in an empty collection),
|
|
24
|
+
* and it is indistinguishable from a registry outage or a criteria typo, so the
|
|
25
|
+
* pipeline aborts instead. Handed to {@link RunWriter.abort}, whose contract –
|
|
26
|
+
* leave the live destination as it was – is exactly the wanted behaviour.
|
|
27
|
+
*/
|
|
28
|
+
export class EmptySelection extends Error {
|
|
29
|
+
constructor() {
|
|
30
|
+
super('Selection produced no datasets; abandoning the run rather than treating an empty selection as membership. The destination is unchanged.');
|
|
31
|
+
this.name = 'EmptySelection';
|
|
32
|
+
}
|
|
33
|
+
}
|
|
16
34
|
/**
|
|
17
35
|
* Split an async iterable into `count` branches that can be consumed
|
|
18
36
|
* independently. Backpressure is enforced by the slowest consumer –
|
|
@@ -301,17 +319,39 @@ export class Pipeline {
|
|
|
301
319
|
const runWriter = this.beforeDatasetWrite
|
|
302
320
|
? new DatasetTransformWriter(openedWriter, this.beforeDatasetWrite)
|
|
303
321
|
: openedWriter;
|
|
322
|
+
let abandoned = false;
|
|
304
323
|
try {
|
|
305
324
|
for await (const dataset of datasets) {
|
|
306
325
|
selectedSources.push(dataset.iri.toString());
|
|
307
326
|
await this.processDataset(dataset, runWriter, context);
|
|
308
327
|
}
|
|
309
|
-
|
|
328
|
+
// Committing an empty selection would hand every writer an empty
|
|
329
|
+
// membership as if it were the truth: an In-place writer’s sweep reads
|
|
330
|
+
// every indexed source as departed and deletes the lot, and a Blue/green
|
|
331
|
+
// writer swaps an empty collection live and drops the one that held the
|
|
332
|
+
// data. A selector returning nothing is indistinguishable from a registry
|
|
333
|
+
// outage or a criteria typo, so the run is abandoned instead – leaving
|
|
334
|
+
// the destination exactly as it was, which is what `abort` already
|
|
335
|
+
// guarantees for both writers. To empty a destination on purpose, drop it.
|
|
336
|
+
abandoned = selectedSources.length === 0;
|
|
337
|
+
if (!abandoned) {
|
|
338
|
+
await runWriter.commit();
|
|
339
|
+
}
|
|
310
340
|
}
|
|
311
341
|
catch (error) {
|
|
312
342
|
await runWriter.abort(error);
|
|
313
343
|
throw error;
|
|
314
344
|
}
|
|
345
|
+
// Outside the try, so the run is aborted exactly once. Inside it, an abort
|
|
346
|
+
// that rejected (a writer whose lock release fails, a fan-out branch that
|
|
347
|
+
// rethrows) would fall into the catch and be aborted a second time, and
|
|
348
|
+
// would make an abandoned run throw – which is the one thing abandoning is
|
|
349
|
+
// meant not to do.
|
|
350
|
+
if (abandoned) {
|
|
351
|
+
const reason = new EmptySelection();
|
|
352
|
+
this.reporter?.selectionEmpty?.(reason.message);
|
|
353
|
+
await runWriter.abort(reason);
|
|
354
|
+
}
|
|
315
355
|
const finalMemory = process.memoryUsage();
|
|
316
356
|
this.reporter?.pipelineComplete?.({
|
|
317
357
|
duration: Date.now() - start,
|
|
@@ -21,6 +21,12 @@ export interface DistributionAnalysisResult {
|
|
|
21
21
|
export interface ProgressReporter {
|
|
22
22
|
pipelineStart?(name: string): void;
|
|
23
23
|
datasetsSelected?(count: number, duration: number): void;
|
|
24
|
+
/**
|
|
25
|
+
* Called when the run is abandoned because the selection was empty – no
|
|
26
|
+
* commit, destination unchanged. Distinct from a completed run that indexed
|
|
27
|
+
* nothing: the writers were told to abort, so nothing was swept or swapped.
|
|
28
|
+
*/
|
|
29
|
+
selectionEmpty?(reason: string): void;
|
|
24
30
|
datasetStart?(dataset: Dataset): void;
|
|
25
31
|
/** Called each time a single distribution probe completes. */
|
|
26
32
|
distributionProbed?(result: DistributionAnalysisResult): void;
|
|
@@ -53,7 +59,7 @@ export interface ProgressReporter {
|
|
|
53
59
|
/**
|
|
54
60
|
* Called once per (dataset, validator) pair after all stages for a dataset
|
|
55
61
|
* have run. Fires regardless of whether any stage actually invoked
|
|
56
|
-
* `validate()`
|
|
62
|
+
* `validate()` – the report reflects the validator’s accumulated state.
|
|
57
63
|
* When no stage produced data, the report typically carries
|
|
58
64
|
* `quadsValidated: 0` and `conforms: true` (the SHACL vacuous-truth
|
|
59
65
|
* default); consumers that want to distinguish ‘not tested’ from ‘tested
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D;;;;;OAKG;IACH,qBAAqB,CAAC,CACpB,YAAY,EAAE,YAAY,EAC1B,OAAO,EAAE,eAAe,GACvB,IAAI,CAAC;IACR,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD;;;;;;;;OAQG;IACH,gBAAgB,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACpE,eAAe,CAAC,CACd,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,GAC1D,IAAI,CAAC;IACR,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT;;;;OAIG;IACH,gBAAgB,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACvD;;;OAGG;IACH,cAAc,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;CACtD"}
|
|
1
|
+
{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD;;;;OAIG;IACH,cAAc,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D;;;;;OAKG;IACH,qBAAqB,CAAC,CACpB,YAAY,EAAE,YAAY,EAC1B,OAAO,EAAE,eAAe,GACvB,IAAI,CAAC;IACR,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD;;;;;;;;OAQG;IACH,gBAAgB,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACpE,eAAe,CAAC,CACd,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,GAC1D,IAAI,CAAC;IACR,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT;;;;OAIG;IACH,gBAAgB,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACvD;;;OAGG;IACH,cAAc,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;CACtD"}
|
package/dist/writer/writer.d.ts
CHANGED
|
@@ -101,6 +101,28 @@ export interface RunWriter<Item = Quad> extends DatasetWriter<Item> {
|
|
|
101
101
|
* @param error The failure that ended the run
|
|
102
102
|
*/
|
|
103
103
|
abort(error: unknown): Promise<void>;
|
|
104
|
+
/**
|
|
105
|
+
* End the run releasing its run-level resources (locks, connections) but
|
|
106
|
+
* **keeping** whatever it built – the half-way house between
|
|
107
|
+
* {@link commit} and {@link abort}.
|
|
108
|
+
*
|
|
109
|
+
* It exists for the one case where undoing is the more destructive choice: a
|
|
110
|
+
* caller coordinating several writers whose outputs reference each other,
|
|
111
|
+
* where one has already gone live pointing at what another built. Dropping
|
|
112
|
+
* that half-built output would break the live one, so the caller asks for it
|
|
113
|
+
* to be kept and reconciled by a later run instead
|
|
114
|
+
* ([ADR 19](https://github.com/ldelements/lde/blob/main/docs/decisions/0019-filter-across-collections-through-declared-joins.md)).
|
|
115
|
+
*
|
|
116
|
+
* Optional: it falls back to {@link abort}, which is already correct for
|
|
117
|
+
* every writer whose `abort` does not discard built state (an In-place
|
|
118
|
+
* rebuild releasing its lock, a writer with no run-level state at all). A
|
|
119
|
+
* writer whose `abort` DOES discard – a Blue/green rebuild dropping its
|
|
120
|
+
* fresh collection – must implement this, or a coordinating caller has to
|
|
121
|
+
* choose between leaking its lock and breaking a live destination.
|
|
122
|
+
*
|
|
123
|
+
* @param error The failure that ended the run
|
|
124
|
+
*/
|
|
125
|
+
abandon?(error: unknown): Promise<void>;
|
|
104
126
|
}
|
|
105
127
|
/**
|
|
106
128
|
* A destination for pipeline output: a factory of per-run transactions.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;OAKG;IACH,eAAe,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;CACvC;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,GAAG,IAAI;IACxC;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,QAAQ,CAAC;AAElD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS,CAAC,IAAI,GAAG,IAAI,CAAE,SAAQ,aAAa,CAAC,IAAI,CAAC;IACjE;;;;;;;;;;OAUG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjE;;;;;;;;;OASG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExC;;;;;;;;;OASG;IACH,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAExB;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;OAKG;IACH,eAAe,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;CACvC;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,GAAG,IAAI;IACxC;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,QAAQ,CAAC;AAElD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS,CAAC,IAAI,GAAG,IAAI,CAAE,SAAQ,aAAa,CAAC,IAAI,CAAC;IACjE;;;;;;;;;;OAUG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjE;;;;;;;;;OASG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExC;;;;;;;;;OASG;IACH,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAExB;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,OAAO,CAAC,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,MAAM,CAAC,IAAI,GAAG,IAAI;IACjC;;;;OAIG;IACH,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;CACxD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.36.0",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -34,8 +34,8 @@
|
|
|
34
34
|
"@rdfjs/types": "^2.0.1",
|
|
35
35
|
"@tpluscode/rdf-ns-builders": "^5.0.0",
|
|
36
36
|
"@traqula/generator-sparql-1-1": "^1.1.8",
|
|
37
|
-
"@traqula/parser-sparql-1-1": "^1.
|
|
38
|
-
"@traqula/rules-sparql-1-1": "^1.
|
|
37
|
+
"@traqula/parser-sparql-1-1": "^1.2.0",
|
|
38
|
+
"@traqula/rules-sparql-1-1": "^1.2.0",
|
|
39
39
|
"fetch-sparql-endpoint": "^7.1.1",
|
|
40
40
|
"filenamify-url": "^4.0.0",
|
|
41
41
|
"is-network-error": "^1.3.2",
|