@lde/pipeline 0.30.8 → 0.30.10

package/dist/writer/fileWriter.d.ts

@@ -21,12 +21,23 @@ export interface FileWriterOptions {
      * Only used when format is 'turtle'.
      */
     prefixes?: Record<string, string>;
+    /**
+     * Derive the named-graph IRI each quad is written into. Only meaningful for
+     * format `'n-quads'`; ignored for `'turtle'` and `'n-triples'`, which have no
+     * graph slot. When set, every quad is re-emitted with this graph term,
+     * regardless of the quad's own graph — mirroring
+     * {@link SparqlUpdateWriter}'s `graphIri`, so the same callback produces the
+     * same named-graph structure whether you write to a SPARQL store or to files.
+     * Defaults to undefined (quads written as-is, i.e. the default graph).
+     */
+    graphIri?: (dataset: Dataset) => URL;
 }
 export declare class FileWriter implements Writer {
     private readonly outputDir;
     readonly format: 'turtle' | 'n-triples' | 'n-quads';
     private readonly replacementCharacter;
     private readonly prefixes?;
+    private readonly graphIri?;
     private readonly activeWriters;
     constructor(options: FileWriterOptions);
     write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;

package/dist/writer/fileWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAiBD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAyB;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAG1B;gBAEQ,OAAO,EAAE,iBAAiB;IAOhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAclE,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB5C,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,WAAW;YAIL,iBAAiB;IAwB/B,OAAO,CAAC,YAAY;CAUrB"}
+{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,GAAG,CAAC;CACtC;AAiBD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAyB;IACnD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA4B;IACtD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAG1B;gBAEQ,OAAO,EAAE,iBAAiB;IAQhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAiClE,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IA8B5C,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,WAAW;YAIL,iBAAiB;IA4B/B,OAAO,CAAC,YAAY;CAUrB"}

package/dist/writer/fileWriter.js

@@ -1,8 +1,8 @@
 import { createWriteStream } from 'node:fs';
-import { mkdir } from 'node:fs/promises';
+import { mkdir, rename, rm } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
 import filenamifyUrl from 'filenamify-url';
-import { Writer as N3Writer } from 'n3';
+import { DataFactory, Writer as N3Writer } from 'n3';
 /**
  * Streams RDF quads to files on disk using N3 Writer.
  *
@@ -22,12 +22,14 @@ export class FileWriter {
     format;
     replacementCharacter;
     prefixes;
+    graphIri;
     activeWriters = new Map();
     constructor(options) {
         this.outputDir = options.outputDir;
         this.format = options.format ?? 'n-triples';
         this.replacementCharacter = options.replacementCharacter ?? '-';
         this.prefixes = options.prefixes;
+        this.graphIri = options.graphIri;
     }
     async write(dataset, quads) {
         // Peek at the first quad to avoid creating empty files.
@@ -36,9 +38,18 @@ export class FileWriter {
         if (first.done)
             return;
         const { n3Writer } = await this.getOrCreateWriter(dataset);
-        n3Writer.addQuad(first.value);
+        // Re-emit each quad into the configured named graph (n-quads only). The
+        // pipeline's quads carry no graph context, so the graph is supplied here
+        // exactly as SparqlUpdateWriter supplies it via INSERT DATA { GRAPH … }.
+        const graphNode = this.format === 'n-quads' && this.graphIri
+            ? DataFactory.namedNode(this.graphIri(dataset).toString())
+            : undefined;
+        const addQuad = (quad) => n3Writer.addQuad(graphNode
+            ? DataFactory.quad(quad.subject, quad.predicate, quad.object, graphNode)
+            : quad);
+        addQuad(first.value);
         for await (const quad of { [Symbol.asyncIterator]: () => iterator }) {
-            n3Writer.addQuad(quad);
+            addQuad(quad);
         }
     }
     async flush(dataset) {
@@ -47,18 +58,29 @@ export class FileWriter {
         if (!entry)
             return;
         this.activeWriters.delete(key);
-        await new Promise((resolve, reject) => {
-            if (entry.stream.errored) {
-                reject(entry.stream.errored);
-                return;
-            }
-            entry.n3Writer.end((error) => {
-                if (error)
-                    reject(error);
-                else
-                    resolve();
+        // Quads are streamed to a sibling temp file; only on a clean flush is it
+        // atomically renamed onto the final path. A crash therefore leaves at most
+        // a stale `*.tmp` — never a truncated final file — so a downstream index
+        // rebuild that globs the final extension never reads a half-written file.
+        try {
+            await new Promise((resolve, reject) => {
+                if (entry.stream.errored) {
+                    reject(entry.stream.errored);
+                    return;
+                }
+                entry.n3Writer.end((error) => {
+                    if (error)
+                        reject(error);
+                    else
+                        resolve();
+                });
             });
-        });
+        }
+        catch (error) {
+            await rm(entry.tempPath, { force: true, recursive: true });
+            throw error;
+        }
+        await rename(entry.tempPath, key);
     }
     getOutputPath(dataset) {
         return this.getFilePath(dataset);
@@ -79,7 +101,11 @@ export class FileWriter {
         if (existing)
             return existing;
         await mkdir(dirname(key), { recursive: true });
-        const stream = createWriteStream(key, { flags: 'w' });
+        // Write to a sibling temp file (same directory, so the flush rename stays on
+        // one filesystem and is atomic). The `.tmp` suffix keeps it out of any glob
+        // on the final extension.
+        const tempPath = `${key}.tmp`;
+        const stream = createWriteStream(tempPath, { flags: 'w' });
         stream.on('error', (error) => {
             // Surface stream errors when flushing; prevents 'unhandled error' crashes.
             stream.destroy(error);
@@ -88,7 +114,7 @@ export class FileWriter {
             format: formatMap[this.format],
             prefixes: this.prefixes,
         });
-        const entry = { n3Writer, stream };
+        const entry = { n3Writer, stream, tempPath };
         this.activeWriters.set(key, entry);
         return entry;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.30.8",
+  "version": "0.30.10",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -24,10 +24,10 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "0.7.4",
-    "@lde/dataset-registry-client": "0.8.0",
-    "@lde/distribution-probe": "0.1.7",
-    "@lde/sparql-importer": "0.6.2",
+    "@lde/dataset": "0.7.5",
+    "@lde/dataset-registry-client": "0.8.1",
+    "@lde/distribution-probe": "0.1.8",
+    "@lde/sparql-importer": "0.6.3",
     "@lde/sparql-server": "0.4.11",
     "@rdfjs/types": "^2.0.1",
     "@traqula/generator-sparql-1-1": "^1.1.1",