@lde/pipeline 0.6.19 → 0.6.21

package/dist/stage.d.ts

@@ -1,7 +1,7 @@
 import { Dataset, Distribution } from '@lde/dataset';
-import type { Quad } from '@rdfjs/types';
 import type { Executor, VariableBindings } from './sparql/executor.js';
 import { NotSupported } from './sparql/executor.js';
+import type { Writer } from './writer/writer.js';
 export interface StageOptions {
     name: string;
     executors: Executor | Executor[];
@@ -15,7 +15,8 @@ export declare class Stage {
     private readonly selector?;
     private readonly batchSize;
     constructor(options: StageOptions);
-    run(dataset: Dataset, distribution: Distribution): Promise<AsyncIterable<Quad> | NotSupported>;
+    run(dataset: Dataset, distribution: Distribution, writer: Writer): Promise<NotSupported | void>;
+    private executeWithSelector;
     private executeAll;
 }
 /** Stage-level selector that yields variable bindings for use in executor queries. Pagination is an implementation detail. */

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGpD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,YAAY;IAS3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;YAwBhC,UAAU;CAkBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,YAAY;IAS3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAYjB,mBAAmB;YAuBnB,UAAU;CAkBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}

package/dist/stage.js

@@ -13,10 +13,16 @@ export class Stage {
         this.selector = options.selector;
         this.batchSize = options.batchSize ?? 10;
     }
-    async run(dataset, distribution) {
-        if (!this.selector) {
-            return this.executeAll(dataset, distribution);
+    async run(dataset, distribution, writer) {
+        const streams = this.selector
+            ? await this.executeWithSelector(dataset, distribution)
+            : await this.executeAll(dataset, distribution);
+        if (streams instanceof NotSupported) {
+            return streams;
         }
+        await writer.write(dataset, mergeStreams(streams));
+    }
+    async executeWithSelector(dataset, distribution) {
         const streams = [];
         for await (const bindings of batch(this.selector, this.batchSize)) {
             for (const executor of this.executors) {
@@ -31,7 +37,7 @@ export class Stage {
         if (streams.length === 0) {
             return new NotSupported('All executors returned NotSupported');
         }
-        return mergeStreams(streams);
+        return streams;
     }
     async executeAll(dataset, distribution) {
         const streams = [];
@@ -44,7 +50,7 @@ export class Stage {
         if (streams.length === 0) {
             return new NotSupported('All executors returned NotSupported');
         }
-        return mergeStreams(streams);
+        return streams;
     }
 }
 async function* mergeStreams(streams) {

package/dist/writer/fileWriter.d.ts

@@ -1,5 +1,5 @@
 import { Dataset } from '@lde/dataset';
-import type { DatasetCore } from '@rdfjs/types';
+import type { Quad } from '@rdfjs/types';
 import { Writer } from './writer.js';
 export interface FileWriterOptions {
     /**
@@ -16,7 +16,7 @@ export declare class FileWriter implements Writer {
     private readonly outputDir;
     private readonly format;
     constructor(options: FileWriterOptions);
-    write(dataset: Dataset, data: DatasetCore): Promise<void>;
+    write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
     private getFilename;
     private getExtension;
 }

package/dist/writer/fileWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAIhD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;CAC7C;AAaD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqC;gBAEhD,OAAO,EAAE,iBAAiB;IAKhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB/D,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,YAAY;CAUrB"}
+{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;CAC7C;AAaD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqC;gBAEhD,OAAO,EAAE,iBAAiB;IAKhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBxE,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,YAAY;CAUrB"}

package/dist/writer/fileWriter.js

@@ -1,9 +1,10 @@
-import { mkdir, writeFile } from 'node:fs/promises';
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
 import filenamifyUrl from 'filenamify-url';
-import { serializeQuads } from './serialize.js';
+import { Writer as N3Writer } from 'n3';
 /**
- * Writes RDF data to files on disk.
+ * Streams RDF quads to files on disk using N3 Writer.
  *
  * Files are named based on the dataset IRI using filenamify-url.
  */
@@ -19,17 +20,28 @@ export class FileWriter {
         this.outputDir = options.outputDir;
         this.format = options.format ?? 'turtle';
     }
-    async write(dataset, data) {
-        const quads = [...data];
-        if (quads.length === 0) {
+    async write(dataset, quads) {
+        // Peek at the first quad to avoid creating empty files.
+        const iterator = quads[Symbol.asyncIterator]();
+        const first = await iterator.next();
+        if (first.done)
             return;
-        }
-        const filename = this.getFilename(dataset);
-        const filePath = join(this.outputDir, filename);
-        // Ensure the output directory exists.
+        const filePath = join(this.outputDir, this.getFilename(dataset));
         await mkdir(dirname(filePath), { recursive: true });
-        const content = await serializeQuads(quads, formatMap[this.format]);
-        await writeFile(filePath, content, 'utf-8');
+        const stream = createWriteStream(filePath);
+        const writer = new N3Writer(stream, { format: formatMap[this.format] });
+        writer.addQuad(first.value);
+        for await (const quad of { [Symbol.asyncIterator]: () => iterator }) {
+            writer.addQuad(quad);
+        }
+        await new Promise((resolve, reject) => {
+            writer.end((error) => {
+                if (error)
+                    reject(error);
+                else
+                    resolve();
+            });
+        });
     }
     getFilename(dataset) {
         const extension = this.getExtension();

package/dist/writer/sparqlUpdateWriter.d.ts

@@ -1,5 +1,5 @@
 import { Dataset } from '@lde/dataset';
-import type { DatasetCore } from '@rdfjs/types';
+import type { Quad } from '@rdfjs/types';
 import { Writer } from './writer.js';
 export interface SparqlWriterOptions {
     /**
@@ -21,14 +21,17 @@ export interface SparqlWriterOptions {
 /**
  * Writes RDF data to a SPARQL endpoint using SPARQL UPDATE INSERT DATA queries.
  *
- * Each dataset's data is written to a named graph based on the dataset IRI.
+ * Clears the named graph before writing, then streams quads in batches
+ * to avoid accumulating the entire dataset in memory.
  */
 export declare class SparqlUpdateWriter implements Writer {
     private readonly endpoint;
     private readonly fetch;
     private readonly batchSize;
     constructor(options: SparqlWriterOptions);
-    write(dataset: Dataset, data: DatasetCore): Promise<void>;
+    write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
+    private clearGraph;
     private insertBatch;
+    private executeUpdate;
 }
 //# sourceMappingURL=sparqlUpdateWriter.d.ts.map

package/dist/writer/sparqlUpdateWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sparqlUpdateWriter.d.ts","sourceRoot":"","sources":["../../src/writer/sparqlUpdateWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,WAAW,EAAQ,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,mBAAmB;IAMlC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;YAejD,WAAW;CAmB1B"}
+{"version":3,"file":"sparqlUpdateWriter.d.ts","sourceRoot":"","sources":["../../src/writer/sparqlUpdateWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,mBAAmB;IAMlC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAS1D,UAAU;YAIV,WAAW;YAOX,aAAa;CAgB5B"}

package/dist/writer/sparqlUpdateWriter.js

@@ -1,8 +1,10 @@
+import { batch } from '../batch.js';
 import { serializeQuads } from './serialize.js';
 /**
  * Writes RDF data to a SPARQL endpoint using SPARQL UPDATE INSERT DATA queries.
  *
- * Each dataset's data is written to a named graph based on the dataset IRI.
+ * Clears the named graph before writing, then streams quads in batches
+ * to avoid accumulating the entire dataset in memory.
  */
 export class SparqlUpdateWriter {
     endpoint;
@@ -13,21 +15,21 @@ export class SparqlUpdateWriter {
         this.fetch = options.fetch ?? globalThis.fetch;
         this.batchSize = options.batchSize ?? 10000;
     }
-    async write(dataset, data) {
+    async write(dataset, quads) {
         const graphUri = dataset.iri.toString();
-        const quads = [...data];
-        if (quads.length === 0) {
-            return;
-        }
-        // Process in batches to avoid hitting endpoint size limits.
-        for (let i = 0; i < quads.length; i += this.batchSize) {
-            const batch = quads.slice(i, i + this.batchSize);
-            await this.insertBatch(graphUri, batch);
+        await this.clearGraph(graphUri);
+        for await (const chunk of batch(quads, this.batchSize)) {
+            await this.insertBatch(graphUri, chunk);
         }
     }
+    async clearGraph(graphUri) {
+        await this.executeUpdate(`CLEAR GRAPH <${graphUri}>`);
+    }
     async insertBatch(graphUri, quads) {
         const turtleData = await serializeQuads(quads, 'N-Triples');
-        const query = `INSERT DATA { GRAPH <${graphUri}> { ${turtleData} } }`;
+        await this.executeUpdate(`INSERT DATA { GRAPH <${graphUri}> { ${turtleData} } }`);
+    }
+    async executeUpdate(query) {
         const response = await this.fetch(this.endpoint.toString(), {
             method: 'POST',
             headers: {

package/dist/writer/writer.d.ts

@@ -1,5 +1,5 @@
 import { Dataset } from '@lde/dataset';
-import type { DatasetCore } from '@rdfjs/types';
+import type { Quad } from '@rdfjs/types';
 /**
  * Interface for writing RDF data to a destination.
  */
@@ -8,8 +8,8 @@ export interface Writer {
      * Write RDF data for a dataset to the destination.
      *
      * @param dataset The dataset metadata
-     * @param data The RDF data to write
+     * @param quads The RDF quads to write
      */
-    write(dataset: Dataset, data: DatasetCore): Promise<void>;
+    write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
 }
 //# sourceMappingURL=writer.d.ts.map

package/dist/writer/writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB;;;;;OAKG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3D"}
+{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB;;;;;OAKG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.19",
+  "version": "0.6.21",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"