@lde/pipeline 0.6.20 → 0.6.21

package/dist/writer/sparqlUpdateWriter.d.ts

@@ -21,7 +21,8 @@ export interface SparqlWriterOptions {
 /**
  * Writes RDF data to a SPARQL endpoint using SPARQL UPDATE INSERT DATA queries.
  *
- * Each dataset's data is written to a named graph based on the dataset IRI.
+ * Clears the named graph before writing, then streams quads in batches
+ * to avoid accumulating the entire dataset in memory.
  */
 export declare class SparqlUpdateWriter implements Writer {
     private readonly endpoint;
@@ -29,6 +30,8 @@ export declare class SparqlUpdateWriter implements Writer {
     private readonly batchSize;
     constructor(options: SparqlWriterOptions);
     write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
+    private clearGraph;
     private insertBatch;
+    private executeUpdate;
 }
 //# sourceMappingURL=sparqlUpdateWriter.d.ts.map

package/dist/writer/sparqlUpdateWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sparqlUpdateWriter.d.ts","sourceRoot":"","sources":["../../src/writer/sparqlUpdateWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,mBAAmB;IAMlC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAkB1D,WAAW;CAmB1B"}
+{"version":3,"file":"sparqlUpdateWriter.d.ts","sourceRoot":"","sources":["../../src/writer/sparqlUpdateWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,mBAAmB;IAMlC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAS1D,UAAU;YAIV,WAAW;YAOX,aAAa;CAgB5B"}

package/dist/writer/sparqlUpdateWriter.js

@@ -1,8 +1,10 @@
+import { batch } from '../batch.js';
 import { serializeQuads } from './serialize.js';
 /**
  * Writes RDF data to a SPARQL endpoint using SPARQL UPDATE INSERT DATA queries.
  *
- * Each dataset's data is written to a named graph based on the dataset IRI.
+ * Clears the named graph before writing, then streams quads in batches
+ * to avoid accumulating the entire dataset in memory.
  */
 export class SparqlUpdateWriter {
     endpoint;
@@ -15,22 +17,19 @@ export class SparqlUpdateWriter {
     }
     async write(dataset, quads) {
         const graphUri = dataset.iri.toString();
-        const collected = [];
-        for await (const quad of quads) {
-            collected.push(quad);
-        }
-        if (collected.length === 0) {
-            return;
-        }
-        // Process in batches to avoid hitting endpoint size limits.
-        for (let i = 0; i < collected.length; i += this.batchSize) {
-            const batch = collected.slice(i, i + this.batchSize);
-            await this.insertBatch(graphUri, batch);
+        await this.clearGraph(graphUri);
+        for await (const chunk of batch(quads, this.batchSize)) {
+            await this.insertBatch(graphUri, chunk);
         }
     }
+    async clearGraph(graphUri) {
+        await this.executeUpdate(`CLEAR GRAPH <${graphUri}>`);
+    }
     async insertBatch(graphUri, quads) {
         const turtleData = await serializeQuads(quads, 'N-Triples');
-        const query = `INSERT DATA { GRAPH <${graphUri}> { ${turtleData} } }`;
+        await this.executeUpdate(`INSERT DATA { GRAPH <${graphUri}> { ${turtleData} } }`);
+    }
+    async executeUpdate(query) {
         const response = await this.fetch(this.endpoint.toString(), {
             method: 'POST',
             headers: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.20",
+  "version": "0.6.21",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"