@lde/pipeline 0.13.0 → 0.14.0

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AA8BD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;YA4Cd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAsCD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;YA6Cd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}

package/dist/pipeline.js

@@ -20,6 +20,10 @@ class FanOutWriter {
             })());
         }
     }
+    async flush(dataset) {
+        for (const w of this.writers)
+            await w.flush?.(dataset);
+    }
 }
 class TransformWriter {
     inner;
@@ -31,6 +35,9 @@ class TransformWriter {
     async write(dataset, quads) {
         await this.inner.write(dataset, this.transform(quads, dataset));
     }
+    async flush(dataset) {
+        await this.inner.flush?.(dataset);
+    }
 }
 export class Pipeline {
     name;
@@ -100,6 +107,7 @@ export class Pipeline {
         finally {
             await this.distributionResolver.cleanup?.();
         }
+        await this.writer.flush?.(dataset);
         this.reporter?.datasetComplete?.(dataset);
     }
     async runStage(dataset, distribution, stage) {

package/dist/writer/fileWriter.d.ts

@@ -16,16 +16,25 @@ export interface FileWriterOptions {
      * @default '-'
      */
     replacementCharacter?: string;
+    /**
+     * Turtle prefix declarations. Keys are prefix names, values are namespace IRIs.
+     * Only used when format is 'turtle'.
+     */
+    prefixes?: Record<string, string>;
 }
 export declare class FileWriter implements Writer {
     private readonly outputDir;
     readonly format: 'turtle' | 'n-triples' | 'n-quads';
     private readonly replacementCharacter;
-    private readonly writtenFiles;
+    private readonly prefixes?;
+    private readonly activeWriters;
     constructor(options: FileWriterOptions);
     write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
+    flush(dataset: Dataset): Promise<void>;
     getOutputPath(dataset: Dataset): string;
     getFilename(dataset: Dataset): string;
+    private getFilePath;
+    private getOrCreateWriter;
     private getExtension;
 }
 //# sourceMappingURL=fileWriter.d.ts.map

package/dist/writer/fileWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAqBD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAqB;gBAEtC,OAAO,EAAE,iBAAiB;IAMhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BxE,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,YAAY;CAUrB"}
+{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAiBD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAyB;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAG1B;gBAEQ,OAAO,EAAE,iBAAiB;IAOhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAclE,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAc5C,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,WAAW;YAIL,iBAAiB;IAoB/B,OAAO,CAAC,YAAY;CAUrB"}

package/dist/writer/fileWriter.js

@@ -8,13 +8,9 @@ import { Writer as N3Writer } from 'n3';
  *
  * Files are named based on the dataset IRI using filenamify-url.
  *
- * The first {@link write} call for a given dataset creates (or overwrites) the file.
- * Subsequent calls for the same dataset append to it, so that multiple pipeline stages
- * can each contribute quads to a single output file.
- *
- * **Note:** With `format: 'turtle'` each append will repeat the prefix declarations
- * at the start of each chunk. The default `format: 'n-triples'` produces clean
- * line-oriented output without repeated headers.
+ * A single N3Writer is kept open per dataset across all {@link write} calls,
+ * so Turtle prefix declarations are written once and triples can be grouped
+ * by subject. Call {@link flush} after all stages complete to finalize the file.
  */
 const formatMap = {
     turtle: 'Turtle',
@@ -25,11 +21,13 @@ export class FileWriter {
     outputDir;
     format;
     replacementCharacter;
-    writtenFiles = new Set();
+    prefixes;
+    activeWriters = new Map();
     constructor(options) {
         this.outputDir = options.outputDir;
         this.format = options.format ?? 'n-triples';
         this.replacementCharacter = options.replacementCharacter ?? '-';
+        this.prefixes = options.prefixes;
     }
     async write(dataset, quads) {
         // Peek at the first quad to avoid creating empty files.
@@ -37,18 +35,20 @@ export class FileWriter {
         const first = await iterator.next();
         if (first.done)
             return;
-        const filePath = join(this.outputDir, this.getFilename(dataset));
-        await mkdir(dirname(filePath), { recursive: true });
-        const flags = this.writtenFiles.has(filePath) ? 'a' : 'w';
-        this.writtenFiles.add(filePath);
-        const stream = createWriteStream(filePath, { flags });
-        const writer = new N3Writer(stream, { format: formatMap[this.format] });
-        writer.addQuad(first.value);
+        const { n3Writer } = await this.getOrCreateWriter(dataset);
+        n3Writer.addQuad(first.value);
         for await (const quad of { [Symbol.asyncIterator]: () => iterator }) {
-            writer.addQuad(quad);
+            n3Writer.addQuad(quad);
         }
+    }
+    async flush(dataset) {
+        const key = this.getFilePath(dataset);
+        const entry = this.activeWriters.get(key);
+        if (!entry)
+            return;
+        this.activeWriters.delete(key);
         await new Promise((resolve, reject) => {
-            writer.end((error) => {
+            entry.n3Writer.end((error) => {
                 if (error)
                     reject(error);
                 else
@@ -57,7 +57,7 @@ export class FileWriter {
         });
     }
     getOutputPath(dataset) {
-        return join(this.outputDir, this.getFilename(dataset));
+        return this.getFilePath(dataset);
     }
     getFilename(dataset) {
         const extension = this.getExtension();
@@ -66,6 +66,24 @@ export class FileWriter {
         });
         return `${baseName}.${extension}`;
     }
+    getFilePath(dataset) {
+        return join(this.outputDir, this.getFilename(dataset));
+    }
+    async getOrCreateWriter(dataset) {
+        const key = this.getFilePath(dataset);
+        const existing = this.activeWriters.get(key);
+        if (existing)
+            return existing;
+        await mkdir(dirname(key), { recursive: true });
+        const stream = createWriteStream(key, { flags: 'w' });
+        const n3Writer = new N3Writer(stream, {
+            format: formatMap[this.format],
+            prefixes: this.prefixes,
+        });
+        const entry = { n3Writer, stream };
+        this.activeWriters.set(key, entry);
+        return entry;
+    }
     getExtension() {
         switch (this.format) {
             case 'turtle':

package/dist/writer/writer.d.ts

@@ -11,5 +11,13 @@ export interface Writer {
      * @param quads The RDF quads to write
      */
     write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
+    /**
+     * Finalize writing for a dataset. Called after all stages complete.
+     *
+     * Writers that buffer output across multiple {@link write} calls (e.g. to
+     * share Turtle prefix declarations) should implement this to flush remaining
+     * data and release resources.
+     */
+    flush?(dataset: Dataset): Promise<void>;
 }
 //# sourceMappingURL=writer.d.ts.map

package/dist/writer/writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB;;;;;OAKG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE"}
+{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB;;;;;OAKG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnE;;;;;;OAMG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzC"}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "repository": {
-    "url": "https://github.com/ldengine/lde",
+    "url": "git+https://github.com/ldengine/lde.git",
     "directory": "packages/pipeline"
   },
   "type": "module",