@rdfc/sparql-ingest-processor-ts 2.1.0 → 2.1.2

package/README.md

@@ -46,6 +46,8 @@ npm install @rdfc/sparql-ingest-processor-ts
 <ingester> a rdfc:SPARQLIngest;
     rdfc:memberStream <in>;
     rdfc:ingestConfig [
+        rdfc:operationMode "Replication";
+        rdfc:memberBatchSize 500;
         rdfc:memberShape "http://ex.org/Shape";
         rdfc:changeSemantics [
             rdfc:changeTypePath "http://ex.org/changeType";
@@ -83,6 +85,8 @@ npm install @rdfc/sparql-ingest-processor-ts
 ---
 
 ### Parameters of `rdfc:IngestConfig`:
+- `rdfc:operationMode` (**string**, optional): Operation mode of the processor. Can be `Replication` or `Sync`. `Replication` mode uses the [SPARQL Graph Store Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/) to ingest data directly into a triple store, which assumes that all received data is meant to be added. `Sync` mode uses the [SPARQL UPDATE specification](https://www.w3.org/TR/sparql11-update/) to ingest data via SPARQL queries, which allows for all CRUD operations.
+- `rdfc:memberBatchSize` (**integer**, optional): Number of input records to batch together for ingestion when running in `Replication` mode.
 - `rdfc:memberShape` (**string**, optional): SHACL shape used to guide query construction when payloads are incomplete.
 - `rdfc:changeSemantics` (**rdfc:ChangeSemantics**, optional): Configures mapping between change types (create/update/delete) and SPARQL operations.
 - `rdfc:targetNamedGraph` (**string**, optional): Force all operations into a specific named graph.
@@ -132,7 +136,6 @@ npm install @rdfc/sparql-ingest-processor-ts
 
 ## Notes
 
-- Delete operations can be handled differently depending on how complete the input record is.
+- Delete operations can be handled differently depending on how complete the input record is. A SHACL shape (`rdfc:memberShape`) can be provided to help identify deletion targets when payloads are incomplete.
 - Transactions can buffer multiple input records and commit them together using `rdfc:transactionConfig`.
-- A SHACL shape (`rdfc:memberShape`) can be provided to help identify deletion targets when payloads are incomplete.
 

package/lib/SPARQLIngest.js

@@ -34,6 +34,7 @@ export class SPARQLIngest extends Processor {
     }
     async transform() {
         for await (const rawQuads of this.memberStream.strings()) {
+            this.logger.debug(`Raw member data received: \n${rawQuads}`);
             const quads = new Parser().parse(rawQuads);
             this.logger.verbose(`Parsed ${quads.length} quads from received member data`);
             const store = RdfStore.createDefault();
@@ -45,6 +46,7 @@ export class SPARQLIngest extends Processor {
                     store.addQuad(q);
                 }
             });
+            sanitizeQuads(store);
             let query;
             const memberIRI = getObjects(store, null, SDS.terms.payload, SDS.terms.custom("DataDescription"))[0];
             if (memberIRI) {
@@ -99,7 +101,6 @@ export class SPARQLIngest extends Processor {
                     }
                     else {
                         const ctv = store.getQuads(null, df.namedNode(this.config.changeSemantics.changeTypePath))[0];
-                        sanitizeQuads(store);
                         if (ctv.object.value === this.config.changeSemantics.createValue) {
                             this.logger.info(`Preparing 'INSERT DATA {}' SPARQL query for member ${memberIRI.value}`);
                             query = CREATE(store, this.config.forVirtuoso);
@@ -128,12 +129,7 @@ export class SPARQLIngest extends Processor {
                     }
                     else {
                         if (this.config.operationMode === OperationMode.REPLICATION) {
-                            this.memberBatch.push(...store.getQuads().map(q => {
-                                if (q.graph.equals(df.defaultGraph()) && this.config.targetNamedGraph) {
-                                    q.graph = df.namedNode(this.config.targetNamedGraph);
-                                }
-                                return q;
-                            }));
+                            this.memberBatch.push(...store.getQuads());
                             this.batchCount++;
                             if (this.batchCount < this.config.memberBatchSize) {
                                 continue;

package/lib/Utils.js

@@ -81,13 +81,14 @@ export async function doSPARQLRequest(query, config, logger) {
         const timeout = config.measurePerformance?.queryTimeout || 1800;
         if (query.length > 0 && typeof query[0] !== 'string') {
             const quads = query;
-            const writer = new N3Writer();
+            const writer = new N3Writer({ format: "N-Quads" });
             const serialized = writer.quadsToString(quads);
             const url = new URL(config.graphStoreUrl);
             if (config.accessToken) {
                 url.searchParams.append("access-token", config.accessToken);
             }
-            logger.debug(`Executing SPARQL Graph Store request (POST) with ${quads.length} quads.`);
+            logger.verbose(`[doSPARQLRequest] Executing SPARQL Graph Store request (POST) with ${quads.length} quads.`);
+            logger.debug(`[doSPARQLRequest] POSTing the following RDF quads:\n${serialized}`);
             const res = await fetch(url.toString(), {
                 method: "POST",
                 headers: {
@@ -113,6 +114,7 @@ export async function doSPARQLRequest(query, config, logger) {
             queries.push(jointQuery);
         }
         for (const q of queries) {
+            logger.debug(`[doSPARQLRequest] Executing SPARQL query: \n${q}`);
             const res = await fetch(config.graphStoreUrl, {
                 method: "POST",
                 headers: {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@rdfc/sparql-ingest-processor-ts",
-    "version": "2.1.0",
+    "version": "2.1.2",
     "description": "SPARQL Update function to be within RDF-Connect pipelines",
     "author": "Julián Rojas",
     "contributors": [
@@ -28,7 +28,7 @@
     },
     "dependencies": {
         "@treecg/types": "^0.4.6",
-        "n3": "^2.0.1",
+        "n3": "^1.26.0",
         "rdf-data-factory": "^2.0.2",
         "rdf-stores": "^2.1.1",
         "undici": "^7.21.0",
@@ -39,7 +39,7 @@
         "@rdfc/js-runner": "^3.0.2",
         "@rdfjs/types": "^2.0.1",
         "@types/n3": "^1.26.1",
-        "@types/node": "^25.2.1",
+        "@types/node": "^25.2.2",
         "@typescript-eslint/eslint-plugin": "^8.54.0",
         "@typescript-eslint/parser": "^8.54.0",
         "@vitest/coverage-v8": "^4.0.18",