@lde/pipeline 0.19.0 → 0.21.0

package/README.md

@@ -128,6 +128,34 @@ new Stage({
 
 This keeps SPARQL doing the heavy lifting while TypeScript handles the edge cases. See [@lde/pipeline-void](../pipeline-void)'s `VocabularyExecutor` for a real-world example of this pattern.
 
+### Validation
+
+Stages can optionally validate their output quads against a `Validator`. Validation operates on the combined output of all executors per batch, so shapes that span multiple executors' output are validated correctly. Quads are buffered, validated, and then written or discarded based on the `onInvalid` policy. When no validator is configured, quads stream directly with zero overhead.
+
+```typescript
+import { ShaclValidator } from '@lde/pipeline-shacl-validator';
+
+new Stage({
+  name: 'transform',
+  executors: await SparqlConstructExecutor.fromFile('transform.rq'),
+  validation: {
+    validator: new ShaclValidator({
+      shapesFile: './shapes.ttl',
+      reportDir: './validation',
+    }),
+    onInvalid: 'write', // 'write' (default) | 'skip' | 'halt'
+  },
+});
+```
+
+| `onInvalid` | Behaviour                                          |
+| ----------- | -------------------------------------------------- |
+| `'write'`   | Write quads even if validation fails **(default)** |
+| `'skip'`    | Discard the batch silently                         |
+| `'halt'`    | Throw an error, stopping the pipeline              |
+
+`Validator` is an interface, so you can implement your own validation strategy. See [@lde/pipeline-shacl-validator](../pipeline-shacl-validator) for the SHACL implementation.
+
 ### Writer
 
 Writes generated quads to a destination:

package/dist/distribution/importResolver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;YAgB5C,aAAa;IAyCrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
+{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;YAgB5C,aAAa;IA0CrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}

package/dist/distribution/importResolver.js

@@ -29,6 +29,7 @@ export class ImportResolver {
     }
     async importDataset(dataset, probeResults, callbacks) {
         const importStart = Date.now();
+        callbacks?.onImportStart?.();
         const importResult = await this.options.importer.import(dataset);
         if (importResult instanceof ImportSuccessful) {
             await this.options.server.start();

package/dist/distribution/resolver.d.ts

@@ -20,6 +20,8 @@ export declare class NoDistributionAvailable {
 export interface ResolveCallbacks {
     /** Called each time a single distribution probe completes. */
     onProbe?: (distribution: Distribution, result: ProbeResultType) => void;
+    /** Called when a data-dump import begins. */
+    onImportStart?: () => void;
     /** Called when importing a distribution fails. */
     onImportFailed?: (distribution: Distribution, error: string) => void;
 }

package/dist/distribution/resolver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAA4B,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAE5E,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;IACpC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM;gBAJpB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA,EAC3B,cAAc,CAAC,EAAE,MAAM,YAAA,EACvB,WAAW,CAAC,EAAE,MAAM,YAAA;CAEhC;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,sDAAsD;AACtD,MAAM,WAAW,gBAAgB;IAC/B,8DAA8D;IAC9D,OAAO,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IACxE,kDAAkD;IAClD,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACtE;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,iCAAiC;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;GAQG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAIjD,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CA6B3D"}
+{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAA4B,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAE5E,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;IACpC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM;gBAJpB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA,EAC3B,cAAc,CAAC,EAAE,MAAM,YAAA,EACvB,WAAW,CAAC,EAAE,MAAM,YAAA;CAEhC;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,sDAAsD;AACtD,MAAM,WAAW,gBAAgB;IAC/B,8DAA8D;IAC9D,OAAO,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IACxE,6CAA6C;IAC7C,aAAa,CAAC,EAAE,MAAM,IAAI,CAAC;IAC3B,kDAAkD;IAClD,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACtE;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,iCAAiC;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;GAQG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAIjD,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CA6B3D"}

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+export * from './validator.js';
 export * from './asyncQueue.js';
 export * from './batch.js';
 export * from './pipeline.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC"}

package/dist/index.js

@@ -1,3 +1,4 @@
+export * from './validator.js';
 export * from './asyncQueue.js';
 export * from './batch.js';
 export * from './pipeline.js';

package/dist/pipeline.d.ts

@@ -35,9 +35,14 @@ export declare class Pipeline {
     constructor(options: PipelineOptions);
     run(): Promise<void>;
     private processDataset;
+    /**
+     * Run a stage with reporting and return whether it was supported.
+     * Returns `true` if the stage produced results, `false` if NotSupported.
+     */
     private runStage;
-    private runChain;
+    /** Run a stage in chained mode, throwing if the stage is not supported. */
     private runChainedStage;
+    private runChain;
     private readFiles;
 }
 //# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAsCD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAeZ,cAAc;YAkDd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAsCD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAeZ,cAAc;IAqD5B;;;OAGG;YACW,QAAQ;IAuCtB,2EAA2E;YAC7D,eAAe;YAcf,QAAQ;YAmDP,SAAS;CAUzB"}

package/dist/pipeline.js

@@ -88,6 +88,9 @@ export class Pipeline {
             onProbe: (distribution, result) => {
                 this.reporter?.distributionProbed?.(mapProbeResult(distribution, result));
             },
+            onImportStart: () => {
+                this.reporter?.importStarted?.();
+            },
             onImportFailed: (distribution, error) => {
                 this.reporter?.importFailed?.(distribution, error);
             },
@@ -118,12 +121,16 @@ export class Pipeline {
         await this.writer.flush?.(dataset);
         this.reporter?.datasetComplete?.(dataset);
     }
-    async runStage(dataset, distribution, stage) {
+    /**
+     * Run a stage with reporting and return whether it was supported.
+     * Returns `true` if the stage produced results, `false` if NotSupported.
+     */
+    async runStage(dataset, distribution, stage, writer = this.writer) {
         this.reporter?.stageStart?.(stage.name);
         const stageStart = Date.now();
         let itemsProcessed = 0;
         let quadsGenerated = 0;
-        const result = await stage.run(dataset, distribution, this.writer, {
+        const result = await stage.run(dataset, distribution, writer, {
             onProgress: (items, quads) => {
                 itemsProcessed = items;
                 quadsGenerated = quads;
@@ -132,13 +139,24 @@ export class Pipeline {
         });
         if (result instanceof NotSupported) {
             this.reporter?.stageSkipped?.(stage.name, result.message);
+            return false;
         }
-        else {
-            this.reporter?.stageComplete?.(stage.name, {
-                itemsProcessed,
-                quadsGenerated,
-                duration: Date.now() - stageStart,
-            });
+        this.reporter?.stageComplete?.(stage.name, {
+            itemsProcessed,
+            quadsGenerated,
+            duration: Date.now() - stageStart,
+        });
+        if (stage.validator) {
+            const report = await stage.validator.report(dataset);
+            this.reporter?.stageValidated?.(stage.name, report);
+        }
+        return true;
+    }
+    /** Run a stage in chained mode, throwing if the stage is not supported. */
+    async runChainedStage(dataset, distribution, stage, writer) {
+        const supported = await this.runStage(dataset, distribution, stage, writer);
+        if (!supported) {
+            throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
         }
     }
     async runChain(dataset, distribution, stage) {
@@ -173,28 +191,6 @@ export class Pipeline {
             await stageOutputResolver.cleanup();
         }
     }
-    async runChainedStage(dataset, distribution, stage, stageWriter) {
-        this.reporter?.stageStart?.(stage.name);
-        const stageStart = Date.now();
-        let itemsProcessed = 0;
-        let quadsGenerated = 0;
-        const result = await stage.run(dataset, distribution, stageWriter, {
-            onProgress: (items, quads) => {
-                itemsProcessed = items;
-                quadsGenerated = quads;
-                this.reporter?.stageProgress?.({ itemsProcessed, quadsGenerated });
-            },
-        });
-        if (result instanceof NotSupported) {
-            this.reporter?.stageSkipped?.(stage.name, result.message);
-            throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
-        }
-        this.reporter?.stageComplete?.(stage.name, {
-            itemsProcessed,
-            quadsGenerated,
-            duration: Date.now() - stageStart,
-        });
-    }
     async *readFiles(paths) {
         for (const path of paths) {
             const stream = createReadStream(path);

package/dist/progressReporter.d.ts

@@ -1,4 +1,5 @@
 import type { Dataset, Distribution } from '@lde/dataset';
+import type { ValidationReport } from './validator.js';
 export interface DistributionAnalysisResult {
     distribution: Distribution;
     type: 'sparql' | 'data-dump' | 'network-error';
@@ -12,6 +13,8 @@ export interface ProgressReporter {
     datasetStart?(dataset: Dataset): void;
     /** Called each time a single distribution probe completes. */
     distributionProbed?(result: DistributionAnalysisResult): void;
+    /** Called when a data-dump import begins. */
+    importStarted?(): void;
     /** Called when importing a distribution fails. */
     importFailed?(distribution: Distribution, error: string): void;
     distributionSelected?(dataset: Dataset, distribution: Distribution, importedFrom?: Distribution, importDuration?: number, tripleCount?: number): void;
@@ -26,6 +29,8 @@ export interface ProgressReporter {
         duration: number;
     }): void;
     stageFailed?(stage: string, error: Error): void;
+    /** Called after a stage completes if it has a validator. */
+    stageValidated?(stage: string, report: ValidationReport): void;
     stageSkipped?(stage: string, reason: string): void;
     datasetComplete?(dataset: Dataset): void;
     datasetSkipped?(dataset: Dataset, reason: string): void;

package/dist/progressReporter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1D,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD,eAAe,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACzC,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACvD"}
+{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,4DAA4D;IAC5D,cAAc,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC/D,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD,eAAe,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACzC,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACvD"}

package/dist/stage.d.ts

@@ -2,6 +2,7 @@ import { Dataset, Distribution } from '@lde/dataset';
 import type { Quad } from '@rdfjs/types';
 import type { Executor, VariableBindings } from './sparql/executor.js';
 import { NotSupported } from './sparql/executor.js';
+import type { Validator } from './validator.js';
 import type { Writer } from './writer/writer.js';
 /** Transforms a quad stream, optionally using dataset metadata. */
 export type QuadTransform = (quads: AsyncIterable<Quad>, dataset: Dataset) => AsyncIterable<Quad>;
@@ -15,6 +16,12 @@ export interface StageOptions {
     maxConcurrency?: number;
     /** Child stages that chain off this stage's output. */
     stages?: Stage[];
+    /** Optional validation of the combined quads produced by all executors per batch. */
+    validation?: {
+        validator: Validator;
+        /** What to do when a batch fails validation. @default 'write' */
+        onInvalid?: 'write' | 'skip' | 'halt';
+    };
 }
 export interface RunOptions {
     onProgress?: (itemsProcessed: number, quadsGenerated: number) => void;
@@ -26,9 +33,17 @@ export declare class Stage {
     private readonly itemSelector?;
     private readonly batchSize;
     private readonly maxConcurrency;
+    private readonly validation?;
     constructor(options: StageOptions);
+    /** The validator for this stage, if configured. */
+    get validator(): Validator | undefined;
     run(dataset: Dataset, distribution: Distribution, writer: Writer, options?: RunOptions): Promise<NotSupported | void>;
     private runWithSelector;
+    /**
+     * Validate a buffer of quads. Throws on halt, returns the quads to write
+     * (empty array when skipping invalid batches).
+     */
+    private validateBuffer;
     private executeAll;
 }
 /** Selects items (as variable bindings) for executors to process. Pagination is an implementation detail. */

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAW3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAmBjB,eAAe;YA+Gf,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAYjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAqCjB,eAAe;IAuH7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}

package/dist/stage.js

@@ -8,6 +8,7 @@ export class Stage {
     itemSelector;
     batchSize;
     maxConcurrency;
+    validation;
     constructor(options) {
         this.name = options.name;
         this.stages = options.stages ?? [];
@@ -17,6 +18,11 @@ export class Stage {
         this.itemSelector = options.itemSelector;
         this.batchSize = options.batchSize ?? 10;
         this.maxConcurrency = options.maxConcurrency ?? 10;
+        this.validation = options.validation;
+    }
+    /** The validator for this stage, if configured. */
+    get validator() {
+        return this.validation?.validator;
     }
     async run(dataset, distribution, writer, options) {
         if (this.itemSelector) {
@@ -26,7 +32,23 @@ export class Stage {
         if (streams instanceof NotSupported) {
             return streams;
         }
-        await writer.write(dataset, mergeStreams(streams));
+        if (this.validation) {
+            const buffer = [];
+            for (const stream of streams) {
+                for await (const quad of stream) {
+                    buffer.push(quad);
+                }
+            }
+            const accepted = await this.validateBuffer(buffer, dataset);
+            if (accepted.length > 0) {
+                await writer.write(dataset, (async function* () {
+                    yield* accepted;
+                })());
+            }
+        }
+        else {
+            await writer.write(dataset, mergeStreams(streams));
+        }
     }
     async runWithSelector(selector, dataset, distribution, writer, options) {
         // Peek the first batch to detect an empty selector before starting the
@@ -68,30 +90,36 @@ export class Stage {
                 for await (const bindings of allBatches) {
                     if (firstError)
                         break;
-                    for (const executor of this.executors) {
+                    // Respect maxConcurrency: wait for a slot to open.
+                    if (inFlight.size >= this.maxConcurrency) {
+                        await Promise.race(inFlight);
                         if (firstError)
                             break;
-                        // Respect maxConcurrency: wait for a slot to open.
-                        if (inFlight.size >= this.maxConcurrency) {
-                            await Promise.race(inFlight);
-                            if (firstError)
-                                break;
-                        }
-                        track((async () => {
+                    }
+                    track((async () => {
+                        const batchQuads = [];
+                        for (const executor of this.executors) {
                             const result = await executor.execute(dataset, distribution, {
                                 bindings,
                             });
                             if (!(result instanceof NotSupported)) {
                                 hasResults = true;
                                 for await (const quad of result) {
-                                    await queue.push(quad);
-                                    quadsGenerated++;
+                                    batchQuads.push(quad);
                                 }
                             }
-                            itemsProcessed += bindings.length;
-                            options?.onProgress?.(itemsProcessed, quadsGenerated);
-                        })());
-                    }
+                        }
+                        let accepted = batchQuads;
+                        if (this.validation && batchQuads.length > 0) {
+                            accepted = await this.validateBuffer(batchQuads, dataset);
+                        }
+                        for (const quad of accepted) {
+                            await queue.push(quad);
+                            quadsGenerated++;
+                        }
+                        itemsProcessed += bindings.length;
+                        options?.onProgress?.(itemsProcessed, quadsGenerated);
+                    })());
                 }
             }
             catch (err) {
@@ -121,6 +149,22 @@ export class Stage {
             return new NotSupported('All executors returned NotSupported');
         }
     }
+    /**
+     * Validate a buffer of quads. Throws on halt, returns the quads to write
+     * (empty array when skipping invalid batches).
+     */
+    async validateBuffer(buffer, dataset) {
+        const validationResult = await this.validation.validator.validate(buffer, dataset);
+        const onInvalid = this.validation.onInvalid ?? 'write';
+        if (!validationResult.conforms && onInvalid === 'halt') {
+            throw new Error(`Validation failed: ${validationResult.violations} violation(s)${validationResult.message ? `. ${validationResult.message}` : ''}`);
+        }
+        if (validationResult.conforms || onInvalid === 'write') {
+            return buffer;
+        }
+        // 'skip': discard
+        return [];
+    }
     async executeAll(dataset, distribution) {
         const results = await Promise.all(this.executors.map((executor) => executor.execute(dataset, distribution)));
         const streams = [];

package/dist/validator.d.ts

@@ -0,0 +1,21 @@
+import type { Quad } from '@rdfjs/types';
+import type { Dataset } from '@lde/dataset';
+/** Validates RDF quads against a set of constraints. */
+export interface Validator {
+    /** Validate a batch of quads. Accumulates results internally. */
+    validate(quads: Quad[], dataset: Dataset): Promise<ValidationResult>;
+    /** Return accumulated validation results for a dataset. */
+    report(dataset: Dataset): Promise<ValidationReport>;
+}
+/** Result of validating a single batch. */
+export interface ValidationResult {
+    violations: number;
+    conforms: boolean;
+    /** Human-readable detail, e.g. path to the report file. Included in error messages on halt. */
+    message?: string;
+}
+/** Accumulated validation results for a dataset. */
+export interface ValidationReport extends ValidationResult {
+    quadsValidated: number;
+}
+//# sourceMappingURL=validator.d.ts.map

package/dist/validator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validator.d.ts","sourceRoot":"","sources":["../src/validator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE5C,wDAAwD;AACxD,MAAM,WAAW,SAAS;IACxB,iEAAiE;IACjE,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAErE,2DAA2D;IAC3D,MAAM,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACrD;AAED,2CAA2C;AAC3C,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,+FAA+F;IAC/F,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,oDAAoD;AACpD,MAAM,WAAW,gBAAiB,SAAQ,gBAAgB;IACxD,cAAc,EAAE,MAAM,CAAC;CACxB"}

package/dist/validator.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.19.0",
+  "version": "0.21.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"