@lde/pipeline-shacl-sampler 0.2.0

package/README.md

@@ -0,0 +1,92 @@
+# @lde/pipeline-shacl-sampler
+
+Per-class sampling stages for [`@lde/pipeline`](../pipeline),
+derived from SHACL shapes.
+
+Given a SHACL shapes file, this package builds one
+[`Stage`](../pipeline/src/stage.ts) per `sh:targetClass`. Each stage
+pairs an [`ItemSelector`](../pipeline/src/sparql/selector.ts) that picks
+N instances of the target class from the distribution’s SPARQL endpoint
+with a CONSTRUCT executor that, for every path chain the SHACL declares
+(walked recursively through `sh:node`, `sh:class`,
+`sh:qualifiedValueShape`, and `sh:or` branches, stopping at leaf
+constraints or shape cycles), pulls in the triples reachable along that
+chain’s terminal node. The resulting quads are a sample subgraph rich
+enough for
+[`@lde/pipeline-shacl-validator`](../pipeline-shacl-validator) to
+validate without false-positive ‘missing nested node’ violations.
+
+## Usage
+
+```typescript
+import { Pipeline } from '@lde/pipeline';
+import { shaclSampleStages } from '@lde/pipeline-shacl-sampler';
+import { ShaclValidator } from '@lde/pipeline-shacl-validator';
+
+const shapesFile = 'https://docs.nde.nl/schema-profile/shacl.ttl';
+const validator = new ShaclValidator({ shapesFile, reportDir: './validation' });
+
+const stages = await shaclSampleStages({
+  shapesFile,
+  samplesPerClass: 50,
+  validator,
+});
+
+await new Pipeline({ /* … */, stages }).run();
+```
+
+## Options
+
+| Option            | Default           | Description                                                                                                            |
+| ----------------- | ----------------- | ---------------------------------------------------------------------------------------------------------------------- |
+| `shapesFile`      | —                 | URL or local path to the SHACL shapes file. Any format `rdf-dereference` accepts.                                      |
+| `samplesPerClass` | `50`              | Number of top-level resources to sample per `sh:targetClass`.                                                          |
+| `timeout`         | `60000`           | SPARQL query timeout in milliseconds.                                                                                  |
+| `batchSize`       | `samplesPerClass` | Maximum sampled subjects per executor call. Lower values spread work across multiple parallel queries.                 |
+| `maxConcurrency`  | `10`              | Maximum concurrent in-flight executor batches per stage.                                                               |
+| `validator`       | —                 | Optional [`Validator`](../pipeline/src/validator.ts) attached to every generated stage (typically a `ShaclValidator`). |
+| `onInvalid`       | `'write'`         | Behaviour when a sampled batch fails validation: `'write'` \| `'skip'` \| `'halt'`. Only used when `validator` is set. |
+
+## Limitations
+
+- Only plain-IRI `sh:path` values are supported. Sequence, alternative
+  and inverse paths throw at extraction time.
+- `sh:targetClass` is the only target form recognised; `sh:targetNode`,
+  `sh:targetSubjectsOf`, `sh:targetObjectsOf` and `sh:sparqlTarget` are
+  not yet supported.
+
+## Related work
+
+### `extract-cbd-shape`
+
+The TREEcg / W3C TREE-incubation
+[`extract-cbd-shape`](https://github.com/TREEcg/extract-cbd-shape)
+library implements a per-entity walk over an in-memory `RdfStore`,
+falling back to an HTTP dereference of the focus node whenever a
+required path is missing from the local store. It is the right tool
+for streaming hypermedia consumers (e.g. LDES clients) that already
+hold a current context and can fetch more of it over HTTP.
+
+It is the wrong tool for this package’s setting. We assemble sample
+subgraphs against a remote SPARQL endpoint with millions of triples;
+the per-entity round-trip pattern would issue _N samples × M target
+classes_ dereferences per dataset and assumes content-negotiable
+entity IRIs that resolve to RDF — rarely true for the cultural
+heritage datasets this package was built for.
+
+### SHACL2SPARQL
+
+The Corman, Reutter & Savković 2019
+[translation](https://link.springer.com/chapter/10.1007/978-3-030-30796-7_27)
+of SHACL constraints to SPARQL targets validation, not sample
+subgraph extraction. No production JavaScript implementation exists.
+
+### Why batch CONSTRUCT per `sh:targetClass`
+
+For each top-level shape, this package emits one `Stage` whose
+CONSTRUCT executor receives a batch of sampled subjects and walks
+every path chain the SHACL declares server-side. A capable SPARQL
+endpoint (e.g. QLever) evaluates the property-path UNIONs in a
+single round-trip per batch, regardless of chain count. The
+alternative — extracting per entity in the client — would multiply
+round-trips by the sample size.

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { extractTargetShapes, type TargetShape } from './pathExtractor.js';
+export { shaclSampleStages, type ShaclSampleStagesOptions, } from './sampleStages.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,KAAK,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EACL,iBAAiB,EACjB,KAAK,wBAAwB,GAC9B,MAAM,mBAAmB,CAAC"}

package/dist/index.js

@@ -0,0 +1,2 @@
+export { extractTargetShapes } from './pathExtractor.js';
+export { shaclSampleStages, } from './sampleStages.js';

package/dist/pathExtractor.d.ts

@@ -0,0 +1,38 @@
+import type { NamedNode } from '@rdfjs/types';
+/**
+ * A SHACL `sh:targetClass` shape distilled into the path chains the sampler
+ * needs to walk to feed the validator a closed sample subgraph.
+ */
+export interface TargetShape {
+    /** The class targeted by `sh:targetClass`. */
+    targetClass: NamedNode;
+    /**
+     * Property-path chains rooted at a sampled instance of {@link targetClass}.
+     *
+     * Each chain is the sequence of `sh:path` IRIs leading from the sampled
+     * subject to a node whose direct triples are needed for validation —
+     * because the SHACL declares a nested-shape constraint on that path
+     * (`sh:node`, `sh:class`, `sh:qualifiedValueShape`, or `sh:or` branches
+     * that reference those). Chains continue recursively into every shape
+     * thus referenced and stop on a cycle (a shape revisited on the current
+     * stack) or on a leaf property shape whose constraints reference no
+     * further shape.
+     *
+     * Each chain is *additive*: shorter prefixes are also present in the
+     * list when they themselves terminate at a nested-shape property.
+     */
+    pathChains: NamedNode[][];
+}
+/**
+ * Load a SHACL shapes file and extract its `sh:targetClass` shapes into
+ * {@link TargetShape}s.
+ *
+ * Multiple NodeShapes can target the same class — they are merged into a
+ * single entry. Only plain-IRI `sh:path` values are supported; sequence,
+ * alternative and inverse paths throw.
+ *
+ * @param shapesFile URL or local path to the SHACL shapes file. Any format
+ *   supported by `rdf-dereference` (Turtle, JSON-LD, N-Triples, …).
+ */
+export declare function extractTargetShapes(shapesFile: string): Promise<TargetShape[]>;
+//# sourceMappingURL=pathExtractor.d.ts.map

package/dist/pathExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pathExtractor.d.ts","sourceRoot":"","sources":["../src/pathExtractor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAQ,MAAM,cAAc,CAAC;AAsBpD;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,8CAA8C;IAC9C,WAAW,EAAE,SAAS,CAAC;IACvB;;;;;;;;;;;;;;OAcG;IACH,UAAU,EAAE,SAAS,EAAE,EAAE,CAAC;CAC3B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,mBAAmB,CACvC,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,WAAW,EAAE,CAAC,CAkCxB"}

package/dist/pathExtractor.js

@@ -0,0 +1,170 @@
+import { DataFactory, Store } from 'n3';
+import { rdfDereferencer } from 'rdf-dereference';
+const { namedNode } = DataFactory;
+const SHACL = 'http://www.w3.org/ns/shacl#';
+const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
+const sh = {
+    targetClass: namedNode(`${SHACL}targetClass`),
+    property: namedNode(`${SHACL}property`),
+    path: namedNode(`${SHACL}path`),
+    node: namedNode(`${SHACL}node`),
+    class: namedNode(`${SHACL}class`),
+    or: namedNode(`${SHACL}or`),
+    qualifiedValueShape: namedNode(`${SHACL}qualifiedValueShape`),
+};
+const rdfFirst = namedNode(`${RDF}first`);
+const rdfRest = namedNode(`${RDF}rest`);
+const rdfNil = namedNode(`${RDF}nil`);
+/**
+ * Load a SHACL shapes file and extract its `sh:targetClass` shapes into
+ * {@link TargetShape}s.
+ *
+ * Multiple NodeShapes can target the same class — they are merged into a
+ * single entry. Only plain-IRI `sh:path` values are supported; sequence,
+ * alternative and inverse paths throw.
+ *
+ * @param shapesFile URL or local path to the SHACL shapes file. Any format
+ *   supported by `rdf-dereference` (Turtle, JSON-LD, N-Triples, …).
+ */
+export async function extractTargetShapes(shapesFile) {
+    const store = await loadShapes(shapesFile);
+    const classToShapes = new Map();
+    for (const quad of store.getQuads(null, sh.targetClass, null, null)) {
+        if (quad.object.termType !== 'NamedNode')
+            continue;
+        const key = quad.object.value;
+        const list = classToShapes.get(key) ?? [];
+        list.push(quad.subject);
+        classToShapes.set(key, list);
+    }
+    const result = [];
+    for (const [classIri, shapeIris] of classToShapes) {
+        const pathChains = [];
+        const seen = new Set();
+        for (const shapeIri of shapeIris) {
+            for (const chain of expandShape(store, shapeIri, new Set(), classToShapes)) {
+                const key = chainKey(chain);
+                if (!seen.has(key)) {
+                    seen.add(key);
+                    pathChains.push(chain);
+                }
+            }
+        }
+        result.push({ targetClass: namedNode(classIri), pathChains });
+    }
+    return result;
+}
+/**
+ * Walk one shape's property graph, producing every path chain that ends at a
+ * nested-shape property. Cycle-detected via the {@link stack} of shape keys
+ * the current recursion has entered but not yet left.
+ */
+function expandShape(store, shape, stack, classToShapes) {
+    const key = termKey(shape);
+    if (stack.has(key))
+        return [];
+    stack.add(key);
+    const chains = [];
+    const seen = new Set();
+    for (const propQuad of store.getQuads(shape, sh.property, null, null)) {
+        const propShape = propQuad.object;
+        const analysis = valueShapeAnalysis(store, propShape, classToShapes);
+        if (!analysis.emit)
+            continue;
+        const path = readPath(store, propShape);
+        addUnique(chains, seen, [path]);
+        for (const nestedRef of analysis.refs) {
+            for (const subChain of expandShape(store, nestedRef, stack, classToShapes)) {
+                addUnique(chains, seen, [path, ...subChain]);
+            }
+        }
+    }
+    stack.delete(key);
+    return chains;
+}
+function valueShapeAnalysis(store, constraintShape, classToShapes) {
+    const refs = [];
+    let emit = false;
+    for (const q of store.getQuads(constraintShape, sh.node, null, null)) {
+        emit = true;
+        refs.push(q.object);
+    }
+    for (const q of store.getQuads(constraintShape, sh.qualifiedValueShape, null, null)) {
+        emit = true;
+        refs.push(q.object);
+    }
+    for (const q of store.getQuads(constraintShape, sh.class, null, null)) {
+        emit = true;
+        if (q.object.termType !== 'NamedNode')
+            continue;
+        for (const target of classToShapes.get(q.object.value) ?? []) {
+            refs.push(target);
+        }
+    }
+    for (const q of store.getQuads(constraintShape, sh.or, null, null)) {
+        for (const branch of orListBranches(store, q.object)) {
+            const sub = valueShapeAnalysis(store, branch, classToShapes);
+            if (sub.emit)
+                emit = true;
+            refs.push(...sub.refs);
+        }
+    }
+    return { emit, refs };
+}
+function orListBranches(store, listHead) {
+    const branches = [];
+    let current = listHead;
+    while (!(current.termType === 'NamedNode' && current.value === rdfNil.value)) {
+        if (current.termType !== 'NamedNode' && current.termType !== 'BlankNode') {
+            break;
+        }
+        const firsts = store.getQuads(current, rdfFirst, null, null);
+        if (firsts.length === 0)
+            break;
+        branches.push(firsts[0].object);
+        const rests = store.getQuads(current, rdfRest, null, null);
+        if (rests.length === 0)
+            break;
+        current = rests[0].object;
+    }
+    return branches;
+}
+async function loadShapes(shapesFile) {
+    const { data } = await rdfDereferencer.dereference(shapesFile, {
+        localFiles: true,
+    });
+    const store = new Store();
+    await new Promise((resolve, reject) => store
+        .import(data)
+        .on('end', () => resolve())
+        .on('error', reject));
+    return store;
+}
+function readPath(store, propShape) {
+    const pathQuads = store.getQuads(propShape, sh.path, null, null);
+    if (pathQuads.length !== 1) {
+        throw new Error(`Property shape ${termLabel(propShape)} must have exactly one sh:path; found ${pathQuads.length}`);
+    }
+    const pathTerm = pathQuads[0].object;
+    if (pathTerm.termType !== 'NamedNode') {
+        throw new Error(`Unsupported sh:path form on property shape ${termLabel(propShape)}: ` +
+            `only plain IRI paths are supported (sequence, alternative and inverse paths are not).`);
+    }
+    return pathTerm;
+}
+function addUnique(chains, seen, chain) {
+    const key = chainKey(chain);
+    if (seen.has(key))
+        return;
+    seen.add(key);
+    chains.push(chain);
+}
+function chainKey(chain) {
+    return chain.map((n) => n.value).join('/');
+}
+function termKey(term) {
+    return `${term.termType}:${term.value}`;
+}
+function termLabel(term) {
+    return term.termType === 'BlankNode' ? `_:${term.value}` : `<${term.value}>`;
+}

package/dist/sampleStages.d.ts

@@ -0,0 +1,61 @@
+import { Stage, type StageOptions, type Validator } from '@lde/pipeline';
+import type { NamedNode } from '@rdfjs/types';
+import { type TargetShape } from './pathExtractor.js';
+type OnInvalid = NonNullable<StageOptions['validation']>['onInvalid'];
+/** Options for {@link shaclSampleStages}. */
+export interface ShaclSampleStagesOptions {
+    /** URL or local path to the SHACL shapes file. */
+    shapesFile: string;
+    /**
+     * Number of top-level resources to sample per `sh:targetClass`.
+     * @default 50
+     */
+    samplesPerClass?: number;
+    /**
+     * SPARQL query timeout in milliseconds.
+     * @default 60000
+     */
+    timeout?: number;
+    /**
+     * Maximum number of sampled subjects per executor call. Defaults to
+     * {@link samplesPerClass} so the whole sample fits in one CONSTRUCT
+     * round-trip; lower to spread work across multiple parallel queries.
+     */
+    batchSize?: number;
+    /**
+     * Maximum concurrent in-flight executor batches per stage. @default 10
+     */
+    maxConcurrency?: number;
+    /**
+     * Validator attached to every generated stage. Typically a
+     * {@link https://www.npmjs.com/package/@lde/pipeline-shacl-validator ShaclValidator}
+     * configured with the same {@link shapesFile}.
+     */
+    validator?: Validator;
+    /**
+     * Behaviour when a sampled batch fails validation. Only used when
+     * {@link validator} is set.
+     * @default 'write'
+     */
+    onInvalid?: OnInvalid;
+}
+/**
+ * Build one sampling {@link Stage} per `sh:targetClass` declared in the SHACL
+ * shapes file. Each stage pairs a SELECT-based {@link ItemSelector} that picks
+ * N instances of its target class with a CONSTRUCT executor that, for every
+ * path chain the SHACL declares (recursively, stopping at leaf constraints
+ * or cycles), pulls in the triples reachable along that chain’s terminal
+ * node.
+ *
+ * Pass a {@link Validator} to attach it to every generated stage:
+ *
+ * ```ts
+ * const validator = new ShaclValidator({ shapesFile, reportDir });
+ * const stages = await shaclSampleStages({ shapesFile, validator });
+ * ```
+ */
+export declare function shaclSampleStages(options: ShaclSampleStagesOptions): Promise<Stage[]>;
+export declare function buildSubjectSelectorQuery(targetClass: NamedNode, limit: number, subjectFilter?: string, namedGraph?: string): string;
+export declare function buildSampleQuery(shape: TargetShape): string;
+export {};
+//# sourceMappingURL=sampleStages.d.ts.map

package/dist/sampleStages.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sampleStages.d.ts","sourceRoot":"","sources":["../src/sampleStages.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EAIL,KAAK,YAAY,EACjB,KAAK,SAAS,EACf,MAAM,eAAe,CAAC;AAEvB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAuB,KAAK,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE3E,KAAK,SAAS,GAAG,WAAW,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;AAEtE,6CAA6C;AAC7C,MAAM,WAAW,wBAAwB;IACvC,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB;;;;OAIG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,KAAK,EAAE,CAAC,CAwBlB;AAiBD,wBAAgB,yBAAyB,CACvC,WAAW,EAAE,SAAS,EACtB,KAAK,EAAE,MAAM,EACb,aAAa,CAAC,EAAE,MAAM,EACtB,UAAU,CAAC,EAAE,MAAM,GAClB,MAAM,CAYR;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CAuB3D"}

package/dist/sampleStages.js

@@ -0,0 +1,86 @@
+import { Stage, SparqlConstructExecutor, SparqlItemSelector, } from '@lde/pipeline';
+import { assertSafeIri } from '@lde/dataset';
+import { extractTargetShapes } from './pathExtractor.js';
+/**
+ * Build one sampling {@link Stage} per `sh:targetClass` declared in the SHACL
+ * shapes file. Each stage pairs a SELECT-based {@link ItemSelector} that picks
+ * N instances of its target class with a CONSTRUCT executor that, for every
+ * path chain the SHACL declares (recursively, stopping at leaf constraints
+ * or cycles), pulls in the triples reachable along that chain’s terminal
+ * node.
+ *
+ * Pass a {@link Validator} to attach it to every generated stage:
+ *
+ * ```ts
+ * const validator = new ShaclValidator({ shapesFile, reportDir });
+ * const stages = await shaclSampleStages({ shapesFile, validator });
+ * ```
+ */
+export async function shaclSampleStages(options) {
+    const samplesPerClass = options.samplesPerClass ?? 50;
+    const timeout = options.timeout ?? 60_000;
+    const batchSize = options.batchSize ?? samplesPerClass;
+    const maxConcurrency = options.maxConcurrency;
+    const validation = options.validator
+        ? { validator: options.validator, onInvalid: options.onInvalid ?? 'write' }
+        : undefined;
+    const shapes = await extractTargetShapes(options.shapesFile);
+    return shapes.map((shape) => new Stage({
+        name: `shacl-sample-${localName(shape.targetClass.value)}`,
+        itemSelector: subjectSelector(shape.targetClass, samplesPerClass),
+        executors: new SparqlConstructExecutor({
+            query: buildSampleQuery(shape),
+            timeout,
+        }),
+        batchSize,
+        maxConcurrency,
+        validation,
+    }));
+}
+function subjectSelector(targetClass, limit) {
+    assertSafeIri(targetClass.value);
+    return {
+        select(distribution, batchSize) {
+            const query = buildSubjectSelectorQuery(targetClass, limit, distribution.subjectFilter, distribution.namedGraph);
+            return new SparqlItemSelector({ query }).select(distribution, batchSize);
+        },
+    };
+}
+export function buildSubjectSelectorQuery(targetClass, limit, subjectFilter, namedGraph) {
+    let fromClause = '';
+    if (namedGraph) {
+        assertSafeIri(namedGraph);
+        fromClause = `FROM <${namedGraph}>`;
+    }
+    return [
+        'SELECT DISTINCT ?s',
+        fromClause,
+        `WHERE { ${subjectFilter ?? ''} ?s a <${targetClass.value}> . }`,
+        `LIMIT ${limit}`,
+    ].join('\n');
+}
+export function buildSampleQuery(shape) {
+    for (const chain of shape.pathChains) {
+        for (const path of chain)
+            assertSafeIri(path.value);
+    }
+    const chainBranches = shape.pathChains
+        .map((chain) => ` UNION {
+    ?s ${chain.map((p) => `<${p.value}>`).join('/')} ?neighbour .
+    ?neighbour ?np ?nv .
+  }`)
+        .join('');
+    return `CONSTRUCT {
+  ?s ?p ?o .
+  ?neighbour ?np ?nv .
+}
+WHERE {
+  {
+    ?s ?p ?o .
+  }${chainBranches}
+}`;
+}
+function localName(iri) {
+    const match = /[#/]([^#/]+)$/.exec(iri);
+    return (match?.[1] ?? iri).replace(/[^A-Za-z0-9_-]/g, '_');
+}

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@lde/pipeline-shacl-sampler",
+  "version": "0.2.0",
+  "description": "Per-class sampling stages for @lde/pipeline, derived from SHACL shapes",
+  "repository": {
+    "url": "git+https://github.com/ldelements/lde.git",
+    "directory": "packages/pipeline-shacl-sampler"
+  },
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "development": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo"
+  ],
+  "dependencies": {
+    "@rdfjs/types": "^2.0.1",
+    "n3": "^2.0.3",
+    "rdf-dereference": "^5.0.0",
+    "tslib": "^2.3.0"
+  },
+  "peerDependencies": {
+    "@lde/dataset": "0.7.3",
+    "@lde/pipeline": "0.28.14"
+  }
+}