@lde/pipeline 0.7.0 → 0.7.1

package/dist/writer/fileWriter.d.ts

@@ -15,6 +15,7 @@ export interface FileWriterOptions {
 export declare class FileWriter implements Writer {
     private readonly outputDir;
     readonly format: 'turtle' | 'n-triples' | 'n-quads';
+    private readonly writtenFiles;
     constructor(options: FileWriterOptions);
     write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
     getOutputPath(dataset: Dataset): string;

package/dist/writer/fileWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;CAC7C;AAaD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;gBAExC,OAAO,EAAE,iBAAiB;IAKhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBxE,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,YAAY;CAUrB"}
+{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;CAC7C;AAsBD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAqB;gBAEtC,OAAO,EAAE,iBAAiB;IAKhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BxE,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,YAAY;CAUrB"}

package/dist/writer/fileWriter.js

@@ -7,6 +7,15 @@ import { Writer as N3Writer } from 'n3';
  * Streams RDF quads to files on disk using N3 Writer.
  *
  * Files are named based on the dataset IRI using filenamify-url.
+ *
+ * The first {@link write} call for a given dataset creates (or overwrites) the file.
+ * Subsequent calls for the same dataset append to it, so that multiple pipeline stages
+ * can each contribute quads to a single output file.
+ *
+ * **Note:** With `format: 'turtle'` (the default) each append will repeat the prefix
+ * declarations at the start of each chunk. For multi-stage pipelines, prefer
+ * `format: 'n-triples'` or `format: 'n-quads'`, which produce clean line-oriented
+ * output without repeated headers.
  */
 const formatMap = {
     turtle: 'Turtle',
@@ -16,6 +25,7 @@ const formatMap = {
 export class FileWriter {
     outputDir;
     format;
+    writtenFiles = new Set();
     constructor(options) {
         this.outputDir = options.outputDir;
         this.format = options.format ?? 'turtle';
@@ -28,7 +38,9 @@ export class FileWriter {
             return;
         const filePath = join(this.outputDir, this.getFilename(dataset));
         await mkdir(dirname(filePath), { recursive: true });
-        const stream = createWriteStream(filePath);
+        const flags = this.writtenFiles.has(filePath) ? 'a' : 'w';
+        this.writtenFiles.add(filePath);
+        const stream = createWriteStream(filePath, { flags });
         const writer = new N3Writer(stream, { format: formatMap[this.format] });
         writer.addQuad(first.value);
         for await (const quad of { [Symbol.asyncIterator]: () => iterator }) {

package/dist/writer/sparqlUpdateWriter.d.ts

@@ -26,14 +26,16 @@ export interface SparqlWriterOptions {
 /**
  * Writes RDF data to a SPARQL endpoint using SPARQL UPDATE INSERT DATA queries.
  *
- * Clears the named graph before writing, then streams quads in batches
- * to avoid accumulating the entire dataset in memory.
+ * Clears the named graph before the first write per dataset per instance, then
+ * streams quads in batches to avoid accumulating the entire dataset in memory.
+ * Subsequent calls to {@link write} for the same dataset append rather than replace.
  */
 export declare class SparqlUpdateWriter implements Writer {
     private readonly endpoint;
     private readonly auth?;
     private readonly fetch;
     private readonly batchSize;
+    private readonly clearedGraphs;
     constructor(options: SparqlWriterOptions);
     write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
     private clearGraph;

package/dist/writer/sparqlUpdateWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sparqlUpdateWriter.d.ts","sourceRoot":"","sources":["../../src/writer/sparqlUpdateWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,mBAAmB;IAOlC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAS1D,UAAU;YAIV,WAAW;YAOX,aAAa;CAqB5B"}
+{"version":3,"file":"sparqlUpdateWriter.d.ts","sourceRoot":"","sources":["../../src/writer/sparqlUpdateWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;GAMG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;gBAEvC,OAAO,EAAE,mBAAmB;IAOlC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAa1D,UAAU;YAIV,WAAW;YAOX,aAAa;CAqB5B"}

package/dist/writer/sparqlUpdateWriter.js

@@ -3,14 +3,16 @@ import { serializeQuads } from './serialize.js';
 /**
  * Writes RDF data to a SPARQL endpoint using SPARQL UPDATE INSERT DATA queries.
  *
- * Clears the named graph before writing, then streams quads in batches
- * to avoid accumulating the entire dataset in memory.
+ * Clears the named graph before the first write per dataset per instance, then
+ * streams quads in batches to avoid accumulating the entire dataset in memory.
+ * Subsequent calls to {@link write} for the same dataset append rather than replace.
  */
 export class SparqlUpdateWriter {
     endpoint;
     auth;
     fetch;
     batchSize;
+    clearedGraphs = new Set();
     constructor(options) {
         this.endpoint = options.endpoint;
         this.auth = options.auth;
@@ -19,7 +21,10 @@ export class SparqlUpdateWriter {
     }
     async write(dataset, quads) {
         const graphUri = dataset.iri.toString();
-        await this.clearGraph(graphUri);
+        if (!this.clearedGraphs.has(graphUri)) {
+            await this.clearGraph(graphUri);
+            this.clearedGraphs.add(graphUri);
+        }
         for await (const chunk of batch(quads, this.batchSize)) {
             await this.insertBatch(graphUri, chunk);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"