@lde/pipeline 0.21.0 → 0.22.1

package/README.md

@@ -130,7 +130,11 @@ This keeps SPARQL doing the heavy lifting while TypeScript handles the edge case
 
 ### Validation
 
-Stages can optionally validate their output quads against a `Validator`. Validation operates on the combined output of all executors per batch, so shapes that span multiple executors' output are validated correctly. Quads are buffered, validated, and then written or discarded based on the `onInvalid` policy. When no validator is configured, quads stream directly with zero overhead.
+Stages can optionally validate their output quads against a `Validator`. Validation operates on the **combined output of all executors per batch**, not on individual quads or per-executor output. A batch produces a complete result set — a self-contained cluster of linked resources — that can be meaningfully matched against SHACL shapes. Even with a single executor, each batch is a complete unit; with multiple executors, shapes that reference triples from different executors are validated correctly.
+
+Validating individual quads would be meaningless, since a single quad carries no structural context for shape matching. Validating the full pipeline output would also be problematic: because the pipeline streams results in batches, it doesn’t know where resource cluster boundaries fall. Batching the output could split a valid cluster across two batches, causing partial resources to fail validation even though the complete cluster is valid.
+
+Quads are buffered, validated, and then written or discarded based on the `onInvalid` policy. When no validator is configured, quads stream directly with zero overhead.
 
 ```typescript
 import { ShaclValidator } from '@lde/pipeline-shacl-validator';

package/dist/sparql/values.d.ts

@@ -1,10 +1,16 @@
-import { type QueryConstruct } from '@traqula/rules-sparql-1-1';
+import { type Pattern, type QueryConstruct, type QuerySelect } from '@traqula/rules-sparql-1-1';
 import type { VariableBindings } from './executor.js';
+/**
+ * Find the first SubSelect within a list of patterns, looking through
+ * intermediate group patterns (the parser wraps `{ SELECT }` in a group).
+ */
+export declare function findSubSelect(patterns: Pattern[]): QuerySelect | undefined;
 /**
  * Inject a VALUES clause into a parsed CONSTRUCT query for the given binding rows.
  *
  * Each row's keys become SPARQL variables; NamedNode values become IRIs in the
- * VALUES block. The VALUES clause is prepended to the query's WHERE patterns.
+ * VALUES block. The VALUES clause is injected into the innermost subquery so
+ * that SPARQL engines can constrain scans early.
  *
  * The caller owns parsing and stringifying; this function operates on the AST.
  */

package/dist/sparql/values.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"values.d.ts","sourceRoot":"","sources":["../../src/sparql/values.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAItD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,gBAAgB,EAAE,GAC3B,cAAc,CAoBhB"}
+{"version":3,"file":"values.d.ts","sourceRoot":"","sources":["../../src/sparql/values.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,OAAO,EAGZ,KAAK,cAAc,EACnB,KAAK,WAAW,EAEjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAItD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,WAAW,GAAG,SAAS,CAW1E;AA0DD;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,gBAAgB,EAAE,GAC3B,cAAc,CAoBhB"}

package/dist/sparql/values.js

@@ -1,10 +1,72 @@
 import { AstFactory, } from '@traqula/rules-sparql-1-1';
 const F = new AstFactory();
+/**
+ * Find the first SubSelect within a list of patterns, looking through
+ * intermediate group patterns (the parser wraps `{ SELECT }` in a group).
+ */
+export function findSubSelect(patterns) {
+    for (const pattern of patterns) {
+        if (F.isQuerySelect(pattern)) {
+            return pattern;
+        }
+        if (pattern.subType === 'group') {
+            const found = findSubSelect(pattern.patterns);
+            if (found)
+                return found;
+        }
+    }
+    return undefined;
+}
+/**
+ * Single-pass find-and-replace: walk through patterns to locate the SubSelect
+ * (looking through group wrappers) and return a new array with it replaced.
+ * Returns `undefined` if no SubSelect was found.
+ */
+function mapSubSelect(patterns, replacer) {
+    for (let index = 0; index < patterns.length; index++) {
+        const pattern = patterns[index];
+        if (F.isQuerySelect(pattern)) {
+            const newPatterns = [...patterns];
+            newPatterns[index] = replacer(pattern);
+            return newPatterns;
+        }
+        if (pattern.subType === 'group') {
+            const group = pattern;
+            const innerResult = mapSubSelect(group.patterns, replacer);
+            if (innerResult) {
+                const newPatterns = [...patterns];
+                newPatterns[index] = F.patternGroup(innerResult, F.gen());
+                return newPatterns;
+            }
+        }
+    }
+    return undefined;
+}
+/**
+ * Recursively walk through nested SubSelect patterns and inject the VALUES
+ * clause into the innermost WHERE clause. This ensures that SPARQL engines
+ * constrain scans at the deepest level rather than only at the outer scope.
+ *
+ * For flat queries (no SubSelect), the base case injects directly — identical
+ * to the previous behavior.
+ */
+function injectIntoInnermost(where, valuesPattern) {
+    const mapped = mapSubSelect(where.patterns, (subSelect) => ({
+        ...subSelect,
+        where: injectIntoInnermost(subSelect.where, valuesPattern),
+    }));
+    if (!mapped) {
+        // Base case: no SubSelect — inject here.
+        return F.patternGroup([valuesPattern, ...where.patterns], F.gen());
+    }
+    return F.patternGroup(mapped, F.gen());
+}
 /**
  * Inject a VALUES clause into a parsed CONSTRUCT query for the given binding rows.
  *
  * Each row's keys become SPARQL variables; NamedNode values become IRIs in the
- * VALUES block. The VALUES clause is prepended to the query's WHERE patterns.
+ * VALUES block. The VALUES clause is injected into the innermost subquery so
+ * that SPARQL engines can constrain scans early.
  *
  * The caller owns parsing and stringifying; this function operates on the AST.
  */
@@ -18,6 +80,6 @@ export function injectValues(query, bindings) {
     const valuesPattern = F.patternValues(variables, values, F.gen());
     return {
         ...query,
-        where: F.patternGroup([valuesPattern, ...query.where.patterns], F.gen()),
+        where: injectIntoInnermost(query.where, valuesPattern),
     };
 }

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAYjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAqCjB,eAAe;IAuH7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAYjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAkDjB,eAAe;IA2I7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}

package/dist/stage.js

@@ -39,11 +39,22 @@ export class Stage {
                     buffer.push(quad);
                 }
             }
-            const accepted = await this.validateBuffer(buffer, dataset);
-            if (accepted.length > 0) {
-                await writer.write(dataset, (async function* () {
-                    yield* accepted;
-                })());
+            const onInvalid = this.validation.onInvalid ?? 'write';
+            if (onInvalid === 'write') {
+                await Promise.all([
+                    writer.write(dataset, (async function* () {
+                        yield* buffer;
+                    })()),
+                    this.validation.validator.validate(buffer, dataset),
+                ]);
+            }
+            else {
+                const accepted = await this.validateBuffer(buffer, dataset);
+                if (accepted.length > 0) {
+                    await writer.write(dataset, (async function* () {
+                        yield* accepted;
+                    })());
+                }
             }
         }
         else {
@@ -74,6 +85,8 @@ export class Stage {
         let itemsProcessed = 0;
         let quadsGenerated = 0;
         let hasResults = false;
+        const onInvalid = this.validation?.onInvalid ?? 'write';
+        const pendingValidations = [];
         const dispatch = async () => {
             const inFlight = new Set();
             let firstError;
@@ -109,13 +122,25 @@ export class Stage {
                                 }
                             }
                         }
-                        let accepted = batchQuads;
-                        if (this.validation && batchQuads.length > 0) {
-                            accepted = await this.validateBuffer(batchQuads, dataset);
+                        if (this.validation &&
+                            batchQuads.length > 0 &&
+                            onInvalid !== 'write') {
+                            // 'skip' or 'halt': must await validation before deciding to write.
+                            const accepted = await this.validateBuffer(batchQuads, dataset);
+                            for (const quad of accepted) {
+                                await queue.push(quad);
+                                quadsGenerated++;
+                            }
                         }
-                        for (const quad of accepted) {
-                            await queue.push(quad);
-                            quadsGenerated++;
+                        else {
+                            for (const quad of batchQuads) {
+                                await queue.push(quad);
+                                quadsGenerated++;
+                            }
+                            if (this.validation && batchQuads.length > 0) {
+                                // 'write' mode: validate concurrently without blocking the write path.
+                                pendingValidations.push(this.validation.validator.validate(batchQuads, dataset));
+                            }
                         }
                         itemsProcessed += bindings.length;
                         options?.onProgress?.(itemsProcessed, quadsGenerated);
@@ -127,6 +152,8 @@ export class Stage {
             }
             // Wait for all remaining in-flight tasks to settle.
             await Promise.all(inFlight);
+            // Ensure all background validations complete before report() is called.
+            await Promise.all(pendingValidations);
             if (firstError) {
                 queue.abort(firstError);
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.21.0",
+  "version": "0.22.1",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -33,7 +33,7 @@
     "@traqula/rules-sparql-1-1": "^1.0.3",
     "fetch-sparql-endpoint": "^7.1.0",
     "filenamify-url": "^4.0.0",
-    "n3": "^2.0.1",
+    "n3": "^2.0.3",
     "tslib": "^2.3.0"
   }
 }