@lde/pipeline 0.22.0 → 0.23.0

package/README.md

@@ -130,7 +130,11 @@ This keeps SPARQL doing the heavy lifting while TypeScript handles the edge case
 
 ### Validation
 
-Stages can optionally validate their output quads against a `Validator`. Validation operates on the combined output of all executors per batch, so shapes that span multiple executors' output are validated correctly. Quads are buffered, validated, and then written or discarded based on the `onInvalid` policy. When no validator is configured, quads stream directly with zero overhead.
+Stages can optionally validate their output quads against a `Validator`. Validation operates on the **combined output of all executors per batch**, not on individual quads or per-executor output. A batch produces a complete result set — a self-contained cluster of linked resources — that can be meaningfully matched against SHACL shapes. Even with a single executor, each batch is a complete unit; with multiple executors, shapes that reference triples from different executors are validated correctly.
+
+Validating individual quads would be meaningless, since a single quad carries no structural context for shape matching. Validating the full pipeline output would also be problematic: because the pipeline streams results in batches, it doesn’t know where resource cluster boundaries fall. Batching the output could split a valid cluster across two batches, causing partial resources to fail validation even though the complete cluster is valid.
+
+Quads are buffered, validated, and then written or discarded based on the `onInvalid` policy. When no validator is configured, quads stream directly with zero overhead.
 
 ```typescript
 import { ShaclValidator } from '@lde/pipeline-shacl-validator';

package/dist/sparql/executor.d.ts

@@ -33,6 +33,11 @@ export interface SparqlConstructExecutorOptions {
      * @default 300000 (5 minutes)
      */
     timeout?: number;
+    /**
+     * Number of retries for transient HTTP errors (502, 503, 504).
+     * @default 3
+     */
+    retries?: number;
     /**
      * Optional custom SparqlEndpointFetcher instance.
      */
@@ -69,6 +74,7 @@ export declare class SparqlConstructExecutor implements Executor {
     private readonly rawQuery;
     private readonly preParsed?;
     private readonly fetcher;
+    private readonly retries;
     private readonly generator;
     constructor(options: SparqlConstructExecutorOptions);
     /**

package/dist/sparql/executor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AASpD;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAkBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAiC/B;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAUpD;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAmBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAuC/B;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}

package/dist/sparql/executor.js

@@ -3,6 +3,7 @@ import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 import { Parser } from '@traqula/parser-sparql-1-1';
 import { Generator } from '@traqula/generator-sparql-1-1';
+import pRetry from 'p-retry';
 import { withDefaultGraph } from './graph.js';
 import { injectValues } from './values.js';
 /**
@@ -45,9 +46,11 @@ export class SparqlConstructExecutor {
     rawQuery;
     preParsed;
     fetcher;
+    retries;
     generator = new Generator();
     constructor(options) {
         this.rawQuery = options.query;
+        this.retries = options.retries ?? 3;
         if (!options.query.includes('#subjectFilter#')) {
             const parsed = new Parser().parse(options.query);
             if (parsed.type !== 'query' || parsed.subType !== 'construct') {
@@ -92,7 +95,10 @@ export class SparqlConstructExecutor {
         }
         let query = this.generator.generate(ast);
         query = query.replaceAll('?dataset', `<${dataset.iri}>`);
-        return await this.fetcher.fetchTriples(endpoint.toString(), query);
+        return await pRetry(() => this.fetcher.fetchTriples(endpoint.toString(), query), {
+            retries: this.retries,
+            shouldRetry: ({ error }) => isTransientHttpError(error),
+        });
     }
     /**
      * Create an executor from a query file.
@@ -111,3 +117,13 @@ export class SparqlConstructExecutor {
 export async function readQueryFile(filename) {
     return (await readFile(resolve(filename))).toString();
 }
+const transientStatusPattern = /HTTP status (\d+)/;
+function isTransientHttpError(error) {
+    if (!(error instanceof Error))
+        return false;
+    const match = error.message.match(transientStatusPattern);
+    if (!match)
+        return false;
+    const status = Number(match[1]);
+    return status === 502 || status === 503 || status === 504;
+}

package/dist/sparql/values.d.ts

@@ -1,10 +1,16 @@
-import { type QueryConstruct } from '@traqula/rules-sparql-1-1';
+import { type Pattern, type QueryConstruct, type QuerySelect } from '@traqula/rules-sparql-1-1';
 import type { VariableBindings } from './executor.js';
+/**
+ * Find the first SubSelect within a list of patterns, looking through
+ * intermediate group patterns (the parser wraps `{ SELECT }` in a group).
+ */
+export declare function findSubSelect(patterns: Pattern[]): QuerySelect | undefined;
 /**
  * Inject a VALUES clause into a parsed CONSTRUCT query for the given binding rows.
  *
  * Each row's keys become SPARQL variables; NamedNode values become IRIs in the
- * VALUES block. The VALUES clause is prepended to the query's WHERE patterns.
+ * VALUES block. The VALUES clause is injected into the innermost subquery so
+ * that SPARQL engines can constrain scans early.
  *
  * The caller owns parsing and stringifying; this function operates on the AST.
  */

package/dist/sparql/values.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"values.d.ts","sourceRoot":"","sources":["../../src/sparql/values.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAItD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,gBAAgB,EAAE,GAC3B,cAAc,CAoBhB"}
+{"version":3,"file":"values.d.ts","sourceRoot":"","sources":["../../src/sparql/values.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,OAAO,EAGZ,KAAK,cAAc,EACnB,KAAK,WAAW,EAEjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAItD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,WAAW,GAAG,SAAS,CAW1E;AA0DD;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,gBAAgB,EAAE,GAC3B,cAAc,CAoBhB"}

package/dist/sparql/values.js

@@ -1,10 +1,72 @@
 import { AstFactory, } from '@traqula/rules-sparql-1-1';
 const F = new AstFactory();
+/**
+ * Find the first SubSelect within a list of patterns, looking through
+ * intermediate group patterns (the parser wraps `{ SELECT }` in a group).
+ */
+export function findSubSelect(patterns) {
+    for (const pattern of patterns) {
+        if (F.isQuerySelect(pattern)) {
+            return pattern;
+        }
+        if (pattern.subType === 'group') {
+            const found = findSubSelect(pattern.patterns);
+            if (found)
+                return found;
+        }
+    }
+    return undefined;
+}
+/**
+ * Single-pass find-and-replace: walk through patterns to locate the SubSelect
+ * (looking through group wrappers) and return a new array with it replaced.
+ * Returns `undefined` if no SubSelect was found.
+ */
+function mapSubSelect(patterns, replacer) {
+    for (let index = 0; index < patterns.length; index++) {
+        const pattern = patterns[index];
+        if (F.isQuerySelect(pattern)) {
+            const newPatterns = [...patterns];
+            newPatterns[index] = replacer(pattern);
+            return newPatterns;
+        }
+        if (pattern.subType === 'group') {
+            const group = pattern;
+            const innerResult = mapSubSelect(group.patterns, replacer);
+            if (innerResult) {
+                const newPatterns = [...patterns];
+                newPatterns[index] = F.patternGroup(innerResult, F.gen());
+                return newPatterns;
+            }
+        }
+    }
+    return undefined;
+}
+/**
+ * Recursively walk through nested SubSelect patterns and inject the VALUES
+ * clause into the innermost WHERE clause. This ensures that SPARQL engines
+ * constrain scans at the deepest level rather than only at the outer scope.
+ *
+ * For flat queries (no SubSelect), the base case injects directly — identical
+ * to the previous behavior.
+ */
+function injectIntoInnermost(where, valuesPattern) {
+    const mapped = mapSubSelect(where.patterns, (subSelect) => ({
+        ...subSelect,
+        where: injectIntoInnermost(subSelect.where, valuesPattern),
+    }));
+    if (!mapped) {
+        // Base case: no SubSelect — inject here.
+        return F.patternGroup([valuesPattern, ...where.patterns], F.gen());
+    }
+    return F.patternGroup(mapped, F.gen());
+}
 /**
  * Inject a VALUES clause into a parsed CONSTRUCT query for the given binding rows.
  *
  * Each row's keys become SPARQL variables; NamedNode values become IRIs in the
- * VALUES block. The VALUES clause is prepended to the query's WHERE patterns.
+ * VALUES block. The VALUES clause is injected into the innermost subquery so
+ * that SPARQL engines can constrain scans early.
  *
  * The caller owns parsing and stringifying; this function operates on the AST.
  */
@@ -18,6 +80,6 @@ export function injectValues(query, bindings) {
     const valuesPattern = F.patternValues(variables, values, F.gen());
     return {
         ...query,
-        where: F.patternGroup([valuesPattern, ...query.where.patterns], F.gen()),
+        where: injectIntoInnermost(query.where, valuesPattern),
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.22.0",
+  "version": "0.23.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -34,6 +34,7 @@
     "fetch-sparql-endpoint": "^7.1.0",
     "filenamify-url": "^4.0.0",
     "n3": "^2.0.3",
+    "p-retry": "^7.1.1",
     "tslib": "^2.3.0"
   }
 }