@lde/pipeline 0.34.0 → 0.34.1

package/README.md

@@ -298,7 +298,7 @@ After all stages for a dataset have run, the pipeline calls `validator.report(da
 Writes generated quads to a destination. A `Writer` is transactional: each pipeline run opens one run on it (`openRun(context)`), writes every dataset through the resulting `RunWriter`, and ends with exactly one `commit()` (on success) or `abort(error)` (on failure). The run lifecycle is the home of destination-level concerns such as atomic swaps, deletion sweeps and cross-pod locks; the pipeline drives `openRun → write* → commit/abort` uniformly and never branches on the writer’s update mode. After a dataset’s stages complete, its `flush(dataset, outcome)` says whether the dataset succeeded, so a writer can gate destructive finalization (e.g. an In-place stale-document sweep) on `'success'`.
 
 - `SparqlUpdateWriter` — writes to a SPARQL endpoint via UPDATE queries; writes are visible as they land, so `commit`/`abort` are no-ops
-- `FileWriter` — writes to local files; each file is streamed to a sibling temp file and atomically renamed on flush, `commit` finalizes any files still open, and `abort` discards temp output
+- `FileWriter` — writes to local files; each file is streamed to a sibling temp file and atomically renamed on flush, `commit` finalizes any files still open, and `abort` discards temp output. In `n-quads` format it skolemises blank nodes to dataset-scoped IRIs, because such files are typically `cat`-indexed into one served graph where document-scoped blank-node labels from different datasets would otherwise fuse (see `blankNodes`/`failOnBlankNodes` and ldelements/lde#474); Turtle/N-Triples output is left unchanged
 
 The `RunContext` handed to `openRun` carries the run’s identity (`runId`, `startedAt`), the full selection (`selectedSources()`, complete by commit time – including datasets skipped as unchanged) and the pipeline’s provenance store, when configured.
 

package/dist/guard/blankNodes.d.ts

@@ -16,6 +16,13 @@ import type { QuadTransform } from '../stage.js';
  * NO blank nodes. Mint stable (skolem) IRIs instead — see `skolemIri` in
  * `@lde/dataset`. These helpers make that invariant testable and enforceable.
  *
+ * `FileWriter` skolemises blank nodes automatically for `n-quads` output as a
+ * safety net, so a producer that slips a blank node through cannot fuse it into
+ * another dataset there. These helpers still matter for producers that must hold
+ * the invariant regardless of writer – e.g. output bound for a `SparqlUpdateWriter`
+ * or a `cat`-indexed Turtle sink, which do not skolemise – and for turning a slip
+ * into a loud failure rather than a silently rewritten IRI.
+ *
  * See ldelements/lde#474 and netwerk-digitaal-erfgoed/dataset-knowledge-graph#352.
  */
 /**

package/dist/guard/blankNodes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"blankNodes.d.ts","sourceRoot":"","sources":["../../src/guard/blankNodes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD;;;;;;;;;;;;;;;;;GAiBG;AAEH;;;GAGG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAU1D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAU9D;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,KAAK,aAAa,CAAC,OAAO,CAAC,CAelE"}
+{"version":3,"file":"blankNodes.d.ts","sourceRoot":"","sources":["../../src/guard/blankNodes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH;;;GAGG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAU1D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAU9D;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,KAAK,aAAa,CAAC,OAAO,CAAC,CAelE"}

package/dist/guard/blankNodes.js

@@ -14,6 +14,13 @@
  * NO blank nodes. Mint stable (skolem) IRIs instead — see `skolemIri` in
  * `@lde/dataset`. These helpers make that invariant testable and enforceable.
  *
+ * `FileWriter` skolemises blank nodes automatically for `n-quads` output as a
+ * safety net, so a producer that slips a blank node through cannot fuse it into
+ * another dataset there. These helpers still matter for producers that must hold
+ * the invariant regardless of writer – e.g. output bound for a `SparqlUpdateWriter`
+ * or a `cat`-indexed Turtle sink, which do not skolemise – and for turning a slip
+ * into a loud failure rather than a silently rewritten IRI.
+ *
  * See ldelements/lde#474 and netwerk-digitaal-erfgoed/dataset-knowledge-graph#352.
  */
 /**

package/dist/writer/fileWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAOvC,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE5E,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,GAAG,CAAC;CACtC;AAgBD;;;;;GAKG;AACH,MAAM,WAAW,aAAc,SAAQ,SAAS;IAC9C,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjE,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACxC;AAED;;;;;;;;;;GAUG;AACH,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAyB;IACnD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA4B;gBAE1C,OAAO,EAAE,iBAAiB;IAQhC,OAAO,CAAC,QAAQ,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC;YAwB9C,UAAU;YAqCV,SAAS;YAOT,SAAS;YAgCT,WAAW;IAczB;;;;OAIG;YACW,cAAc;IAQ5B,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,WAAW;YAIL,eAAe;IA6B7B,OAAO,CAAC,YAAY;CAUrB"}
+{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAOvC,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG5E,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,GAAG,CAAC;CACtC;AAgBD;;;;;GAKG;AACH,MAAM,WAAW,aAAc,SAAQ,SAAS;IAC9C,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjE,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACxC;AAED;;;;;;;;;;GAUG;AACH,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAyB;IACnD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA4B;gBAE1C,OAAO,EAAE,iBAAiB;IAQhC,OAAO,CAAC,QAAQ,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC;YAwB9C,UAAU;YAsDV,SAAS;YAOT,SAAS;YAgCT,WAAW;IAczB;;;;OAIG;YACW,cAAc;IAQ5B,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,WAAW;YAIL,eAAe;IA6B7B,OAAO,CAAC,YAAY;CAUrB"}

package/dist/writer/fileWriter.js

@@ -3,6 +3,7 @@ import { mkdir, rename, rm } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
 import filenamifyUrl from 'filenamify-url';
 import { DataFactory, Writer as N3Writer } from 'n3';
+import { skolemizeBlankNodes } from './skolemizeBlankNodes.js';
 const formatMap = {
     turtle: 'Turtle',
     'n-triples': 'N-Triples',
@@ -71,6 +72,22 @@ export class FileWriter {
         const addQuad = (quad) => n3Writer.addQuad(graphNode
             ? DataFactory.quad(quad.subject, quad.predicate, quad.object, graphNode)
             : quad);
+        // n-quads files are cat-ed into one served graph, where document-scoped
+        // blank-node labels from different datasets collapse into one node
+        // (ldelements/lde#478, dataset-knowledge-graph#352/#420). Skolemise blank
+        // nodes to dataset-scoped IRIs so they stay distinct. Hashing the write to
+        // scope the IRIs needs it buffered, so only n-quads pays that cost – Turtle
+        // and N-Triples (single-document, offline) stream through unchanged.
+        if (this.format === 'n-quads') {
+            const batch = [first.value];
+            for await (const quad of { [Symbol.asyncIterator]: () => iterator }) {
+                batch.push(quad);
+            }
+            for (const quad of skolemizeBlankNodes(batch, dataset.iri.toString())) {
+                addQuad(quad);
+            }
+            return;
+        }
         addQuad(first.value);
         for await (const quad of { [Symbol.asyncIterator]: () => iterator }) {
             addQuad(quad);

package/dist/writer/skolemizeBlankNodes.d.ts

@@ -0,0 +1,27 @@
+import type { Quad } from '@rdfjs/types';
+/**
+ * Rewrite every blank node in `quads` to a deterministic, dataset-scoped IRI,
+ * leaving the quads otherwise unchanged.
+ *
+ * A served store such as the Dataset Knowledge Graph assembles one graph per
+ * dataset by concatenating every per-dataset n-quads file into a single index
+ * (`qlever index` over the `cat` of all files). Document-scoped blank-node
+ * labels are not preserved across that concatenation: two datasets' `_:b0`
+ * collapse into one node, silently fusing unrelated measurements, activities
+ * and linksets across datasets (see ldelements/lde#478 and
+ * netwerk-digitaal-erfgoed/dataset-knowledge-graph#352 / #420). IRIs are never
+ * relabelled, so skolemising each blank node keeps distinct nodes distinct.
+ *
+ * Each blank node becomes `<dataset>/.well-known/skolem#<batch>-<label>`. The
+ * dataset IRI rules out fusion across datasets; `<batch>`, a content hash of
+ * this write's quads, rules out fusion across the separate writes that land in
+ * one dataset's file – their labels both restart at `b0`, but writes carrying
+ * different content hash differently. Two byte-identical writes collapse onto
+ * the same IRIs, which is correct: they carry the same nodes.
+ *
+ * @param quads - The quads to rewrite, possibly containing blank nodes.
+ * @param datasetIri - The dataset the quads belong to; scopes every minted IRI.
+ * @returns The same quads with every blank node replaced by a skolem IRI.
+ */
+export declare function skolemizeBlankNodes(quads: Quad[], datasetIri: string): Quad[];
+//# sourceMappingURL=skolemizeBlankNodes.d.ts.map

package/dist/writer/skolemizeBlankNodes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"skolemizeBlankNodes.d.ts","sourceRoot":"","sources":["../../src/writer/skolemizeBlankNodes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,IAAI,EAKL,MAAM,cAAc,CAAC;AAMtB;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,EAAE,CAgB7E"}

package/dist/writer/skolemizeBlankNodes.js

@@ -0,0 +1,53 @@
+import { DataFactory } from 'n3';
+import { hashSuffix, skolemIri } from '@lde/dataset';
+const { namedNode, quad: makeQuad } = DataFactory;
+/**
+ * Rewrite every blank node in `quads` to a deterministic, dataset-scoped IRI,
+ * leaving the quads otherwise unchanged.
+ *
+ * A served store such as the Dataset Knowledge Graph assembles one graph per
+ * dataset by concatenating every per-dataset n-quads file into a single index
+ * (`qlever index` over the `cat` of all files). Document-scoped blank-node
+ * labels are not preserved across that concatenation: two datasets' `_:b0`
+ * collapse into one node, silently fusing unrelated measurements, activities
+ * and linksets across datasets (see ldelements/lde#478 and
+ * netwerk-digitaal-erfgoed/dataset-knowledge-graph#352 / #420). IRIs are never
+ * relabelled, so skolemising each blank node keeps distinct nodes distinct.
+ *
+ * Each blank node becomes `<dataset>/.well-known/skolem#<batch>-<label>`. The
+ * dataset IRI rules out fusion across datasets; `<batch>`, a content hash of
+ * this write's quads, rules out fusion across the separate writes that land in
+ * one dataset's file – their labels both restart at `b0`, but writes carrying
+ * different content hash differently. Two byte-identical writes collapse onto
+ * the same IRIs, which is correct: they carry the same nodes.
+ *
+ * @param quads - The quads to rewrite, possibly containing blank nodes.
+ * @param datasetIri - The dataset the quads belong to; scopes every minted IRI.
+ * @returns The same quads with every blank node replaced by a skolem IRI.
+ */
+export function skolemizeBlankNodes(quads, datasetIri) {
+    const base = `${datasetIri}/.well-known/skolem#${hashSuffix(fingerprint(quads))}`;
+    const skolemize = (term) => term.termType === 'BlankNode'
+        ? namedNode(skolemIri(base, term.value))
+        : term;
+    // A skolemised blank node is a NamedNode, valid in every term position, so the
+    // casts hold: skolemize only rewrites blank nodes and leaves other terms as-is.
+    return quads.map((quad) => makeQuad(skolemize(quad.subject), quad.predicate, skolemize(quad.object), skolemize(quad.graph)));
+}
+/** A deterministic string identifying a write's quads, for a per-write IRI segment. */
+function fingerprint(quads) {
+    return quads
+        .map((quad) => `${termKey(quad.subject)} ${quad.predicate.value} ${termKey(quad.object)}`)
+        .sort()
+        .join('\n');
+}
+function termKey(term) {
+    switch (term.termType) {
+        case 'BlankNode':
+            return `_:${term.value}`;
+        case 'Literal':
+            return `"${term.value}"^^<${term.datatype.value}>@${term.language}`;
+        default:
+            return `<${term.value}>`;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.34.0",
+  "version": "0.34.1",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -39,7 +39,7 @@
     "fetch-sparql-endpoint": "^7.1.1",
     "filenamify-url": "^4.0.0",
     "is-network-error": "^1.3.2",
-    "n3": "^2.1.0",
+    "n3": "^2.1.1",
     "p-retry": "^8.0.0",
     "rdf-string": "^2.0.1",
     "tslib": "^2.3.0"