@lde/pipeline 0.30.16 → 0.30.18

package/README.md

@@ -276,6 +276,21 @@ Writes generated quads to a destination:
 - `SparqlUpdateWriter` — writes to a SPARQL endpoint via UPDATE queries
 - `FileWriter` — writes to local files
 
+### Reporter
+
+A `ProgressReporter` observes the run, receiving lifecycle events such as `pipelineStart`, `stageComplete`, `datasetValidated` and `pipelineComplete`. Every method is optional, so a reporter implements only the events it cares about.
+
+Pass a single reporter, or an array to have several observe the same run — for example a console reporter alongside one that collects validation verdicts:
+
+```typescript
+new Pipeline({
+  // …
+  reporter: [new ConsoleReporter(), verdictCollector],
+});
+```
+
+Each reporter receives every event, in array order; a reporter that does not implement a given event is skipped for it.
+
 ### Provenance store
 
 A `ProvenanceStore` gives the pipeline a small per-dataset memory, so a future run can skip datasets that are genuinely unchanged. It is purely a storage seam: the framework owns the skip decision (see [`sourceFingerprint`](#source-change-fingerprint) and `shouldReprocess`), the store owns only how each record is persisted.

package/dist/combineReporters.d.ts

@@ -0,0 +1,16 @@
+import type { ProgressReporter } from './progressReporter.js';
+/**
+ * Combine several {@link ProgressReporter}s into one that forwards every
+ * lifecycle call to each child that implements it. Lets a single run be
+ * observed by more than one reporter – e.g. a console reporter alongside a
+ * verdict-collecting one.
+ *
+ * Each method is dispatched to the children in array order; a child that does
+ * not implement a given (optional) method is skipped for that call.
+ *
+ * Internal to the package: not re-exported from `index.ts`. {@link Pipeline}
+ * uses it to normalise a `reporter` array into the single reporter its call
+ * sites expect, so the broader API need not grow a new public symbol.
+ */
+export declare function combineReporters(reporters: readonly ProgressReporter[]): ProgressReporter;
+//# sourceMappingURL=combineReporters.d.ts.map

package/dist/combineReporters.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"combineReporters.d.ts","sourceRoot":"","sources":["../src/combineReporters.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAU9D;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,SAAS,gBAAgB,EAAE,GACrC,gBAAgB,CA4ClB"}

package/dist/combineReporters.js

@@ -0,0 +1,51 @@
+/**
+ * Combine several {@link ProgressReporter}s into one that forwards every
+ * lifecycle call to each child that implements it. Lets a single run be
+ * observed by more than one reporter – e.g. a console reporter alongside a
+ * verdict-collecting one.
+ *
+ * Each method is dispatched to the children in array order; a child that does
+ * not implement a given (optional) method is skipped for that call.
+ *
+ * Internal to the package: not re-exported from `index.ts`. {@link Pipeline}
+ * uses it to normalise a `reporter` array into the single reporter its call
+ * sites expect, so the broader API need not grow a new public symbol.
+ */
+export function combineReporters(reporters) {
+    const forward = (method, ...args) => {
+        for (const reporter of reporters) {
+            // Cast to the concrete signature for `method`: indexing by a generic key
+            // yields a union of method types TS won't call directly, even though the
+            // arguments are correlated.
+            const handler = reporter[method];
+            // Every method is optional; notify only the children that implement it.
+            handler?.(...args);
+        }
+    };
+    // Listing every method explicitly (rather than a Proxy) keeps the forwarding
+    // type-safe: typing the result as `Required<ProgressReporter>` forces a new
+    // entry here whenever the interface grows, so a forgotten method fails to
+    // compile instead of silently going unforwarded.
+    const combined = {
+        pipelineStart: (...args) => forward('pipelineStart', ...args),
+        datasetsSelected: (...args) => forward('datasetsSelected', ...args),
+        datasetStart: (...args) => forward('datasetStart', ...args),
+        distributionProbed: (...args) => forward('distributionProbed', ...args),
+        importStarted: (...args) => forward('importStarted', ...args),
+        importFailed: (...args) => forward('importFailed', ...args),
+        distributionValidated: (...args) => forward('distributionValidated', ...args),
+        distributionSelected: (...args) => forward('distributionSelected', ...args),
+        stageStart: (...args) => forward('stageStart', ...args),
+        stageProgress: (...args) => forward('stageProgress', ...args),
+        stageComplete: (...args) => forward('stageComplete', ...args),
+        stageFailed: (...args) => forward('stageFailed', ...args),
+        stageSkipped: (...args) => forward('stageSkipped', ...args),
+        datasetValidated: (...args) => forward('datasetValidated', ...args),
+        datasetComplete: (...args) => forward('datasetComplete', ...args),
+        datasetSkipped: (...args) => forward('datasetSkipped', ...args),
+        pipelineComplete: (...args) => forward('pipelineComplete', ...args),
+        timeoutTightened: (...args) => forward('timeoutTightened', ...args),
+        timeoutRelaxed: (...args) => forward('timeoutRelaxed', ...args),
+    };
+    return combined;
+}

package/dist/pipeline.d.ts

@@ -42,7 +42,13 @@ export interface PipelineOptions {
         stageOutputResolver: StageOutputResolver;
         outputDir: string;
     };
-    reporter?: ProgressReporter;
+    /**
+     * Observer(s) notified of pipeline lifecycle events. Pass an array to have
+     * several reporters observe the same run – e.g. a console reporter alongside
+     * a verdict-collecting one; every reporter receives each event in array
+     * order. A single reporter may be passed directly.
+     */
+    reporter?: ProgressReporter | readonly ProgressReporter[];
     /**
      * Optional per-dataset processing memory. When set, the pipeline skips a
      * dataset whose source-change fingerprint and {@link pipelineVersion} both

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAGrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAG1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;CAC3D;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AAkFD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe;IA2C9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoBZ,cAAc;IA+K5B,+EAA+E;YACjE,aAAa;YAmBb,gBAAgB;IAW9B,OAAO,CAAE,aAAa;IAOtB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAMnB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA+DP,SAAS;CAczB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAGrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAG1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;CAC3D;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AAkFD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe;IA+C9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoBZ,cAAc;IA+K5B,+EAA+E;YACjE,aAAa;YAmBb,gBAAgB;IAW9B,OAAO,CAAE,aAAa;IAOtB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAMnB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA+DP,SAAS;CAczB"}

package/dist/pipeline.js

@@ -9,6 +9,7 @@ import { NetworkError, SparqlProbeResult, } from '@lde/distribution-probe';
 import { ImportSuccessful } from '@lde/sparql-importer';
 import { importOutcomeToVerdict, probeResultToVerdict, } from '@lde/distribution-health';
 import { NotSupported } from './sparql/executor.js';
+import { combineReporters } from './combineReporters.js';
 import { ConstantTimeoutPolicy, } from './sparql/timeoutPolicy.js';
 /**
  * Split an async iterable into `count` branches that can be consumed
@@ -113,7 +114,11 @@ export class Pipeline {
         this.distributionResolver =
             options.distributionResolver ?? new SparqlDistributionResolver();
         this.chaining = options.chaining;
-        this.reporter = options.reporter;
+        // `Array.isArray` narrows the array branch but not the readonly-array out of
+        // the else branch, so cast the single-reporter case explicitly.
+        this.reporter = Array.isArray(options.reporter)
+            ? combineReporters(options.reporter)
+            : options.reporter;
         this.timeoutFactory =
             options.timeout ?? (() => new ConstantTimeoutPolicy(300_000));
         this.provenanceStore = options.provenanceStore;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.30.16",
+  "version": "0.30.18",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -26,7 +26,7 @@
   "dependencies": {
     "@lde/dataset": "0.7.7",
     "@lde/dataset-registry-client": "0.8.3",
-    "@lde/distribution-health": "0.1.1",
+    "@lde/distribution-health": "0.1.2",
     "@lde/distribution-probe": "0.1.10",
     "@lde/sparql-importer": "0.6.5",
     "@lde/sparql-server": "0.4.11",