@lde/pipeline 0.42.1 → 0.44.0

package/dist/pipeline.js

@@ -9,8 +9,9 @@ import { sourceFingerprint } from './provenance/sourceFingerprint.js';
 import { shouldReprocess } from './provenance/reprocessDecision.js';
 import { NetworkError, SparqlProbeResult, } from '@lde/distribution-probe';
 import { importOutcomeToVerdict, probeResultToVerdict, } from '@lde/distribution-health';
-import { NotSupported } from './sparql/reader.js';
-import { combineReporters } from './combineReporters.js';
+import { NoItemsSelected, NotSupported } from './sparql/reader.js';
+import { levelOf, } from './pipelineEvent.js';
+import { UnreadValueTally } from './unreadValueTally.js';
 import { ConstantTimeoutPolicy, } from './sparql/timeoutPolicy.js';
 /**
  * Why a run was abandoned rather than committed: its selector produced no
@@ -237,7 +238,7 @@ export class Pipeline {
     beforeDatasetWrite;
     distributionResolver;
     chaining;
-    reporter;
+    reporters;
     timeoutFactory;
     provenanceStore;
     pipelineVersion;
@@ -277,11 +278,7 @@ export class Pipeline {
         this.distributionResolver =
             options.distributionResolver ?? new SparqlDistributionResolver();
         this.chaining = options.chaining;
-        // `Array.isArray` narrows the array branch but not the readonly-array out of
-        // the else branch, so cast the single-reporter case explicitly.
-        this.reporter = Array.isArray(options.reporter)
-            ? combineReporters(options.reporter)
-            : options.reporter;
+        this.reporters = [options.reporter ?? []].flat();
         this.timeoutFactory =
             options.timeout ?? (() => new ConstantTimeoutPolicy(300_000));
         this.provenanceStore = options.provenanceStore;
@@ -289,7 +286,8 @@ export class Pipeline {
     }
     async run() {
         const start = Date.now();
-        this.reporter?.pipelineStart?.(this.name);
+        const runId = randomUUID();
+        this.report(runId, { type: 'pipelineStart', name: this.name });
         // Fail fast on a store that cannot persist records (e.g. a provenance
         // file on a mount the runtime user cannot write to): every dataset would
         // process fine, but the records would be lost and skip-unchanged would
@@ -297,17 +295,21 @@ export class Pipeline {
         await this.provenanceStore?.check?.();
         const selectStart = Date.now();
         const datasets = await this.datasetSelector.select();
-        this.reporter?.datasetsSelected?.(datasets.total, Date.now() - selectStart);
+        this.report(runId, {
+            type: 'datasetsSelected',
+            count: datasets.total,
+            durationMs: Date.now() - selectStart,
+        });
         // The run transaction: one openRun → write* → commit/abort per pipeline
         // run. `selectedSources` accumulates every selected dataset – including
         // ones skipped as unchanged – so a writer's commit can sweep by registry
         // membership.
         const selectedSources = [];
         const context = {
-            runId: randomUUID(),
+            runId,
             startedAt: new Date().toISOString(),
             selectedSources: () => selectedSources,
-            itemsRejected: (dataset, rejection) => this.reporter?.itemsRejected?.(dataset, rejection),
+            itemsRejected: (dataset, rejection) => this.report(runId, { type: 'itemsRejected', dataset, rejection }),
             provenance: this.provenanceStore,
         };
         const openedWriter = await this.writer.openRun(context);
@@ -349,36 +351,61 @@ export class Pipeline {
         // meant not to do.
         if (abandoned) {
             const reason = new EmptySelection();
-            this.reporter?.selectionEmpty?.(reason.message);
+            this.report(runId, { type: 'selectionEmpty', message: reason.message });
             await runWriter.abort(reason);
         }
         const finalMemory = process.memoryUsage();
-        this.reporter?.pipelineComplete?.({
-            duration: Date.now() - start,
+        this.report(runId, {
+            type: 'pipelineComplete',
+            durationMs: Date.now() - start,
             memoryUsageBytes: finalMemory.rss,
             heapUsedBytes: finalMemory.heapUsed,
         });
     }
     async processDataset(dataset, runWriter, context) {
-        this.reporter?.datasetStart?.(dataset);
+        const { runId } = context;
+        const datasetStart = Date.now();
+        this.report(runId, { type: 'datasetStart', dataset });
+        const skip = (reason) => this.report(runId, { type: 'datasetSkipped', dataset, reason });
+        const skipOnError = (code, prefix, thrown) => {
+            const error = thrown instanceof Error
+                ? thrown
+                : new Error(String(thrown), { cause: thrown });
+            this.report(runId, {
+                type: 'datasetSkipped',
+                dataset,
+                reason: { code, message: `${prefix}: ${error.message}` },
+                error,
+            });
+        };
         // Probe phase: gather probe results and the source-to-be, without importing.
         let probed;
         try {
             probed = await this.distributionResolver.probe(dataset, {
                 onProbe: (distribution, result) => {
-                    this.reporter?.distributionProbed?.(mapProbeResult(distribution, result));
+                    this.report(runId, {
+                        type: 'distributionProbed',
+                        dataset,
+                        distribution,
+                        result: mapProbeResult(distribution, result),
+                    });
                     // Shallow validity: the probe parse-validates small RDF bodies as a
                     // by-product. Surface that verdict per distribution, judged against
                     // the distribution's own observed fingerprint.
                     const verdict = probeResultToVerdict(result, sourceFingerprint(distribution, result));
                     if (verdict) {
-                        this.reporter?.distributionValidated?.(distribution, verdict);
+                        this.report(runId, {
+                            type: 'distributionValidated',
+                            dataset,
+                            distribution,
+                            verdict,
+                        });
                     }
                 },
             });
         }
         catch (error) {
-            this.reporter?.datasetSkipped?.(dataset, `Distribution probing failed: ${error instanceof Error ? error.message : String(error)}`);
+            skipOnError('probeFailed', 'Distribution probing failed', error);
             return;
         }
         // Derive the source-change fingerprint from the probed source: null for a
@@ -405,24 +432,26 @@ export class Pipeline {
                 sourceFingerprint: fingerprint,
                 pipelineVersion: this.pipelineVersion,
             }, stored)) {
-                this.reporter?.datasetSkipped?.(dataset, 'Unchanged since last run');
+                skip({ code: 'unchanged', message: 'Unchanged since last run' });
                 return;
             }
         }
         // Resolve phase: import a data dump only when the source is one.
+        const importCallbacks = {
+            onImportStart: () => this.report(runId, { type: 'importStarted', dataset }),
+            onImportFailed: (distribution, error) => this.report(runId, {
+                type: 'importFailed',
+                dataset,
+                distribution,
+                error,
+            }),
+        };
         let resolved;
         try {
-            resolved = await this.distributionResolver.resolve(probed, {
-                onImportStart: () => {
-                    this.reporter?.importStarted?.();
-                },
-                onImportFailed: (distribution, error) => {
-                    this.reporter?.importFailed?.(distribution, error);
-                },
-            });
+            resolved = await this.distributionResolver.resolve(probed, importCallbacks);
         }
         catch (error) {
-            this.reporter?.datasetSkipped?.(dataset, `Distribution resolution failed: ${error instanceof Error ? error.message : String(error)}`);
+            skipOnError('resolveFailed', 'Distribution resolution failed', error);
             return;
         }
         if (resolved instanceof NoDistributionAvailable) {
@@ -431,24 +460,36 @@ export class Pipeline {
             // no summary, so an invalid distribution is recorded rather than silently
             // dropped.
             if (resolved.importFailed) {
-                this.reporter?.distributionValidated?.(resolved.importFailed.distribution, importOutcomeToVerdict(resolved.importFailed, fingerprint));
+                this.report(runId, {
+                    type: 'distributionValidated',
+                    dataset,
+                    distribution: resolved.importFailed.distribution,
+                    verdict: importOutcomeToVerdict(resolved.importFailed, fingerprint),
+                });
             }
             // Record the failure so a dataset whose source is unchanged is not
             // re-imported every run; it is retried at the next fingerprint change or
             // version rotation.
-            await this.recordOutcome(dataset, fingerprint, 'failed');
-            this.reporter?.datasetSkipped?.(dataset, resolved.message);
+            await this.recordOutcome(runId, dataset, fingerprint, 'failed');
+            skip({ code: 'noDistributionAvailable', message: resolved.message });
             return;
         }
-        this.reportSelectedDistribution(dataset, resolved, fingerprint);
+        this.reportSelectedDistribution(runId, dataset, resolved, fingerprint);
         const timeout = this.timeoutFactory();
         const unsubscribe = timeout.subscribe?.({
-            onTighten: (event) => this.reporter?.timeoutTightened?.(event),
-            onRelax: (event) => this.reporter?.timeoutRelaxed?.(event),
+            onTighten: (transition) => this.report(runId, {
+                ...transition,
+                type: 'timeoutTightened',
+                dataset,
+            }),
+            onRelax: (transition) => this.report(runId, { ...transition, type: 'timeoutRelaxed', dataset }),
         });
         let stageFailed = false;
+        // What the kept attempt’s stages could not read, reported once the dataset
+        // completes.
+        let unread;
         try {
-            stageFailed = await this.runStages(dataset, resolved.distribution, timeout, runWriter, context);
+            ({ stageFailed, unread } = await this.runStages(dataset, resolved.distribution, timeout, runWriter, context));
             // Reactive fallback: an endpoint that passed probing but could not serve
             // the analysis stages is empirically incapable. Switch to the dataset’s
             // data dump and re-run all stages locally, discarding the
@@ -463,10 +504,7 @@ export class Pipeline {
                 // path. The dataset stays recorded as failed (stageFailed is already
                 // true) and processing continues with the next dataset.
                 try {
-                    const fallback = await this.distributionResolver.resolveFallback(probed, {
-                        onImportStart: () => this.reporter?.importStarted?.(),
-                        onImportFailed: (distribution, error) => this.reporter?.importFailed?.(distribution, error),
-                    });
+                    const fallback = await this.distributionResolver.resolveFallback(probed, importCallbacks);
                     if (fallback instanceof ResolvedDistribution) {
                         // The dump is now the dataset's effective source: report it as
                         // selected/validated under its own change fingerprint so the next
@@ -480,11 +518,11 @@ export class Pipeline {
                                 fallbackFingerprint = sourceFingerprint(dump, dumpProbeResult);
                             }
                         }
-                        this.reportSelectedDistribution(dataset, fallback, fallbackFingerprint);
+                        this.reportSelectedDistribution(runId, dataset, fallback, fallbackFingerprint);
                         // Discard the endpoint-sourced partial output before the re-run so
                         // the dump-sourced stats replace it rather than appending to it.
                         await runWriter.reset?.(dataset);
-                        stageFailed = await this.runStages(dataset, fallback.distribution, timeout, runWriter, context);
+                        ({ stageFailed, unread } = await this.runStages(dataset, fallback.distribution, timeout, runWriter, context));
                         // Adopt the dump’s fingerprint only once the re-run has succeeded.
                         // If the reset or the re-run throws, or a stage fails against the
                         // dump, the dataset is recorded ‘failed’ under the endpoint’s
@@ -499,11 +537,16 @@ export class Pipeline {
                         // A failed dump import is a deep validity verdict on that dump –
                         // surface it rather than silently keeping the endpoint's partial
                         // output, matching the primary NoDistributionAvailable path.
-                        this.reporter?.distributionValidated?.(fallback.importFailed.distribution, importOutcomeToVerdict(fallback.importFailed, fingerprint));
+                        this.report(runId, {
+                            type: 'distributionValidated',
+                            dataset,
+                            distribution: fallback.importFailed.distribution,
+                            verdict: importOutcomeToVerdict(fallback.importFailed, fingerprint),
+                        });
                     }
                 }
                 catch (error) {
-                    this.reporter?.stageFailed?.('reactive-dump-fallback', error instanceof Error ? error : new Error(String(error)));
+                    this.reportStageFailed(runId, dataset, 'reactive-dump-fallback', error);
                 }
             }
         }
@@ -518,21 +561,60 @@ export class Pipeline {
             // A per-dataset writer failure must not abort the whole run: isolate it
             // like a stage failure so the remaining datasets still process. The
             // dataset is recorded 'failed' below and retried on the next run.
-            this.reporter?.stageFailed?.('write', error instanceof Error ? error : new Error(String(error)));
+            this.reportStageFailed(runId, dataset, 'write', error);
             stageFailed = true;
         }
-        await this.reportValidators(dataset);
+        await this.reportValidators(runId, dataset);
         // A dataset whose stages threw produced incomplete output; record it as
         // ‘failed’ rather than freezing a broken result under a ‘success’ record.
-        await this.recordOutcome(dataset, fingerprint, stageFailed ? 'failed' : 'success');
+        await this.recordOutcome(runId, dataset, fingerprint, stageFailed ? 'failed' : 'success');
+        if (!stageFailed && unread !== undefined) {
+            for (const entry of unread.entries()) {
+                this.report(runId, { type: 'valuesUnread', dataset, unread: entry });
+            }
+        }
         const datasetMemory = process.memoryUsage();
-        this.reporter?.datasetComplete?.(dataset, {
+        this.report(runId, {
+            type: 'datasetComplete',
+            dataset,
+            durationMs: Date.now() - datasetStart,
             memoryUsageBytes: datasetMemory.rss,
             heapUsedBytes: datasetMemory.heapUsed,
         });
     }
+    report(runId, payload) {
+        if (this.reporters.length === 0)
+            return;
+        const event = {
+            ...payload,
+            defaultLevel: levelOf(payload),
+            timestamp: new Date(),
+            runId,
+        };
+        for (const reporter of this.reporters) {
+            // A reporter only observes the run, so its failure must not change it:
+            // the other reporters still get the event, and the run carries on.
+            try {
+                reporter.onEvent(event);
+            }
+            catch (error) {
+                process.emitWarning(`A reporter threw on a ${event.type} event: ${error instanceof Error ? error.message : String(error)}`, {
+                    type: 'ReporterWarning',
+                    detail: error instanceof Error ? error.stack : undefined,
+                });
+            }
+        }
+    }
+    reportStageFailed(runId, dataset, stage, error) {
+        this.report(runId, {
+            type: 'stageFailed',
+            dataset,
+            stage,
+            error: error instanceof Error ? error : new Error(String(error)),
+        });
+    }
     /** Persist the processing record for a dataset, when a store is configured. */
-    async recordOutcome(dataset, fingerprint, status) {
+    async recordOutcome(runId, dataset, fingerprint, status) {
         if (!this.provenanceStore)
             return;
         try {
@@ -548,10 +630,10 @@ export class Pipeline {
             // reprocesses next run, its record not yet updated – but it must be
             // surfaced: a store that fails every write silently disables
             // skip-unchanged.
-            this.reporter?.stageFailed?.('provenance', error instanceof Error ? error : new Error(String(error)));
+            this.reportStageFailed(runId, dataset, 'provenance', error);
         }
     }
-    async reportValidators(dataset) {
+    async reportValidators(runId, dataset) {
         const validators = new Set();
         for (const stage of this.collectStages(this.stages)) {
             if (stage.validator)
@@ -559,7 +641,7 @@ export class Pipeline {
         }
         for (const validator of validators) {
             const report = await validator.report(dataset);
-            this.reporter?.datasetValidated?.(dataset, report);
+            this.report(runId, { type: 'datasetValidated', dataset, report });
         }
     }
     // Walks every stage and sub-stage to collect validators; it reads only
@@ -594,24 +676,36 @@ export class Pipeline {
      * yielded no triples); native SPARQL endpoints are not imported and carry no
      * deep verdict.
      */
-    reportSelectedDistribution(dataset, resolved, fingerprint) {
-        this.reporter?.distributionSelected?.(dataset, resolved.distribution, resolved.imported);
+    reportSelectedDistribution(runId, dataset, resolved, fingerprint) {
+        this.report(runId, {
+            type: 'distributionSelected',
+            dataset,
+            distribution: resolved.distribution,
+            imported: resolved.imported,
+        });
         if (resolved.imported) {
-            this.reporter?.distributionValidated?.(resolved.imported.from, importOutcomeToVerdict({
-                type: 'successful',
+            this.report(runId, {
+                type: 'distributionValidated',
+                dataset,
                 distribution: resolved.imported.from,
-                tripleCount: resolved.imported.tripleCount,
-                warnings: [],
-            }, fingerprint));
+                verdict: importOutcomeToVerdict({
+                    type: 'successful',
+                    distribution: resolved.imported.from,
+                    tripleCount: resolved.imported.tripleCount,
+                    warnings: [],
+                }, fingerprint),
+            });
         }
     }
     /**
      * Run every top-level stage against one distribution, catching and reporting
      * per-stage failures so one failing stage does not abort the rest. Returns
      * whether any stage hard-failed – the signal the reactive dump fallback
-     * reacts to.
+     * reacts to – and what the stages could not read, which belongs to this
+     * attempt alone.
      */
     async runStages(dataset, distribution, timeout, runWriter, context) {
+        const unread = new UnreadValueTally();
         let stageFailed = false;
         for (const stage of this.stages) {
             try {
@@ -619,22 +713,22 @@ export class Pipeline {
                     await this.runChain(dataset, distribution, stage, runWriter, context, timeout);
                 }
                 else {
-                    await this.runStage(dataset, distribution, stage, this.stageWriter(runWriter, stage.name), timeout);
+                    await this.runStage(context.runId, dataset, distribution, stage, this.stageWriter(runWriter, stage.name), timeout, unread);
                 }
             }
             catch (error) {
                 stageFailed = true;
-                this.reporter?.stageFailed?.(stage.name, error instanceof Error ? error : new Error(String(error)));
+                this.reportStageFailed(context.runId, dataset, stage.name, error);
             }
         }
-        return stageFailed;
+        return { stageFailed, unread };
     }
     /**
      * Run a stage with reporting and return whether it was supported.
      * Returns `true` if the stage produced results, `false` if NotSupported.
      */
-    async runStage(dataset, distribution, stage, writer, timeout) {
-        this.reporter?.stageStart?.(stage.name);
+    async runStage(runId, dataset, distribution, stage, writer, timeout, unread) {
+        this.report(runId, { type: 'stageStart', dataset, stage: stage.name });
         const stageStart = Date.now();
         let itemsProcessed = 0;
         let quadsGenerated = 0;
@@ -643,7 +737,10 @@ export class Pipeline {
                 itemsProcessed = items;
                 quadsGenerated = quads;
                 const stageMemory = process.memoryUsage();
-                this.reporter?.stageProgress?.({
+                this.report(runId, {
+                    type: 'stageProgress',
+                    dataset,
+                    stage: stage.name,
                     itemsProcessed,
                     quadsGenerated,
                     memoryUsageBytes: stageMemory.rss,
@@ -651,21 +748,35 @@ export class Pipeline {
                 });
             },
             timeout,
+            valueUnread: unread && ((target, field, value) => unread.add(target, field, value)),
         });
         if (result instanceof NotSupported) {
-            this.reporter?.stageSkipped?.(stage.name, result.message);
+            this.report(runId, {
+                type: 'stageSkipped',
+                dataset,
+                stage: stage.name,
+                reason: {
+                    code: result instanceof NoItemsSelected
+                        ? 'noItemsSelected'
+                        : 'notSupported',
+                    message: result.message,
+                },
+            });
             return false;
         }
-        this.reporter?.stageComplete?.(stage.name, {
+        this.report(runId, {
+            type: 'stageComplete',
+            dataset,
+            stage: stage.name,
             itemsProcessed,
             quadsGenerated,
-            duration: Date.now() - stageStart,
+            durationMs: Date.now() - stageStart,
         });
         return true;
     }
     /** Run a stage in chained mode, throwing if the stage is not supported. */
-    async runChainedStage(dataset, distribution, stage, writer, timeout) {
-        const supported = await this.runStage(dataset, distribution, stage, writer, timeout);
+    async runChainedStage(runId, dataset, distribution, stage, writer, timeout) {
+        const supported = await this.runStage(runId, dataset, distribution, stage, writer, timeout);
         if (!supported) {
             throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
         }
@@ -686,7 +797,7 @@ export class Pipeline {
             });
             const scratchRun = await scratchWriter.openRun(context);
             try {
-                await this.runChainedStage(dataset, stageDistribution, chainedStage, scratchRun, timeout);
+                await this.runChainedStage(context.runId, dataset, stageDistribution, chainedStage, scratchRun, timeout);
                 await scratchRun.flush(dataset, 'success');
                 await scratchRun.commit();
             }
@@ -739,7 +850,6 @@ function mapProbeResult(distribution, result) {
     const fingerprint = sourceFingerprint(distribution, result);
     if (result instanceof NetworkError) {
         return {
-            distribution,
             type: 'network-error',
             available: false,
             error: result.message,
@@ -748,7 +858,6 @@ function mapProbeResult(distribution, result) {
         };
     }
     return {
-        distribution,
         type: result instanceof SparqlProbeResult
             ? 'sparql'
             : 'data-dump',