@rdfc/sparql-ingest-processor-ts 2.0.3 → 2.1.0

package/README.md

@@ -2,10 +2,10 @@
 
 [![Build and tests with Node.js](https://github.com/rdf-connect/sparql-ingest-processor-ts/actions/workflows/build-test.yml/badge.svg)](https://github.com/rdf-connect/sparql-ingest-processor-ts/actions/workflows/build-test.yml) [![npm](https://img.shields.io/npm/v/@rdfc/sparql-ingest-processor-ts.svg?style=popout)](https://npmjs.com/package/@rdfc/sparql-ingest-processor-ts)
 
-TypeScript [RDF-Connect](https://rdf-connect.github.io/rdfc.github.io/) processor for ingesting [SDS records](https://treecg.github.io/SmartDataStreams-Spec/) into a SPARQL endpoint.
+TypeScript [RDF-Connect](https://rdf-connect.github.io/rdfc.github.io/) processor for ingesting [SDS records](https://treecg.github.io/SmartDataStreams-Spec/) or in general, a stream of RDF quads into a SPARQL endpoint.
 
-This processor takes a stream of RDF records, transforms them into [SPARQL Update](https://www.w3.org/TR/sparql11-update/) queries, and executes them against a SPARQL Graph Store via the [SPARQL Protocol](https://www.w3.org/TR/sparql11-protocol/).  
-It supports `INSERT DATA`, `DELETE INSERT WHERE`, and `DELETE WHERE` queries, configurable through change semantics or SDS record content.
+This processor takes a stream of RDF records, transforms them into corresponding [SPARQL Update](https://www.w3.org/TR/sparql11-update/) queries, and executes them against a SPARQL Graph Store via the [SPARQL Protocol](https://www.w3.org/TR/sparql11-protocol/).  
+It supports `INSERT DATA`, `DELETE INSERT WHERE`, and `DELETE WHERE` queries, configurable through change semantics or SDS record content. It also supports direct quad ingestion via the SPARQL Graph Store Protocol.
 
 ---
 
@@ -46,8 +46,7 @@ npm install @rdfc/sparql-ingest-processor-ts
 <ingester> a rdfc:SPARQLIngest;
     rdfc:memberStream <in>;
     rdfc:ingestConfig [
-        rdfc:memberIsGraph false;
-        rdfc:memberShape "http://ex.org/Shape1", "http://ex.org/Shape2";
+        rdfc:memberShape "http://ex.org/Shape";
         rdfc:changeSemantics [
             rdfc:changeTypePath "http://ex.org/changeType";
             rdfc:createValue "http://ex.org/Create";
@@ -60,7 +59,14 @@ npm install @rdfc/sparql-ingest-processor-ts
             rdfc:transactionEndPath "http://ex.org/transactionEnd"
         ];
         rdfc:graphStoreUrl "http://example.org/sparql";
-        rdfc:forVirtuoso false
+        rdfc:forVirtuoso false;
+        rdfc:accessToken "myAccessToken";
+        rdfc:measurePerformance [
+            rdfc:name "myPerformanceMeasurement";
+            rdfc:outputPath "/path/to/output.json";
+            rdfc:failureIsFatal true;
+            rdfc:queryTimeout 30000
+        ]
     ];
     rdfc:sparqlWriter <out>.
 ```
@@ -77,13 +83,12 @@ npm install @rdfc/sparql-ingest-processor-ts
 ---
 
 ### Parameters of `rdfc:IngestConfig`:
-- `rdfc:memberIsGraph` (**boolean**, required): Whether each SDS record represents a named graph.
-- `rdfc:memberShape` (**string**, optional, repeatable): SHACL shape identifiers used to guide query construction when payloads are incomplete.
+- `rdfc:memberShape` (**string**, optional): SHACL shape used to guide query construction when payloads are incomplete.
 - `rdfc:changeSemantics` (**rdfc:ChangeSemantics**, optional): Configures mapping between change types (create/update/delete) and SPARQL operations.
-- `rdfc:targetNamedGraph` (**string**, optional): Force all operations into a specific named graph (ignored if `memberIsGraph = true`).
+- `rdfc:targetNamedGraph` (**string**, optional): Force all operations into a specific named graph.
 - `rdfc:transactionConfig` (**rdfc:TransactionConfig**, optional): Groups records by transaction ID for atomic updates.
 - `rdfc:graphStoreUrl` (**string**, optional): SPARQL Graph Store endpoint URL.
-- `rdfc:forVirtuoso` (**boolean**, optional): Enables Virtuoso-specific handling.
+- `rdfc:forVirtuoso` (**boolean**, optional): Enables Virtuoso-specific handling to avoid query size limits.
 - `rdfc:accessToken` (**string**, optional): Access token for authenticated graph stores.
 - `rdfc:measurePerformance` (**rdfc:PerformanceConfig**, optional): Enables performance measurement of SPARQL queries.
 
@@ -117,7 +122,6 @@ npm install @rdfc/sparql-ingest-processor-ts
 <ingester> a rdfc:SPARQLIngest;
     rdfc:memberStream <in>;
     rdfc:ingestConfig [
-        rdfc:memberIsGraph true;
         rdfc:targetNamedGraph "http://example.org/targetGraph";
         rdfc:graphStoreUrl "http://example.org/sparql"
     ];
@@ -128,8 +132,7 @@ npm install @rdfc/sparql-ingest-processor-ts
 
 ## Notes
 
-- Delete operations can be handled differently depending on how complete the SDS record payload is.
-- When `memberIsGraph = true`, queries are wrapped with `GRAPH` and `WITH` clauses.
-- Transactions can buffer multiple SDS records and commit them together using `rdfc:transactionConfig`.
-- SHACL shapes (`rdfc:memberShape`) can be provided to help identify deletion targets when payloads are incomplete.
+- Delete operations can be handled differently depending on how complete the input record is.
+- Transactions can buffer multiple input records and commit them together using `rdfc:transactionConfig`.
+- A SHACL shape (`rdfc:memberShape`) can be provided to help identify deletion targets when payloads are incomplete.
 

package/lib/SPARQLIngest.d.ts

@@ -26,8 +26,7 @@ export declare enum OperationMode {
 export type IngestConfig = {
     operationMode?: OperationMode;
     memberBatchSize?: number;
-    memberIsGraph?: boolean;
-    memberShapes?: string[];
+    memberShape?: string;
     changeSemantics?: ChangeSemantics;
     targetNamedGraph?: string;
     transactionConfig?: TransactionConfig;
@@ -57,7 +56,6 @@ export declare class SPARQLIngest extends Processor<SPARQLIngestArgs> {
     transform(this: SPARQLIngestArgs & this): Promise<void>;
     produce(this: SPARQLIngestArgs & this): Promise<void>;
     verifyTransaction(stores: RdfStore[], transactionIdPath: string, transactionId: Term): void;
-    getNamedGraphIfAny(memberIRI: Term, memberIsGraph: boolean | undefined, targetNamedGraph?: string): string | undefined;
     createTransactionQueries(transactionMembers: TransactionMember[], config: IngestConfig): string;
 }
 export {};

package/lib/SPARQLIngest.js

@@ -34,11 +34,17 @@ export class SPARQLIngest extends Processor {
     }
     async transform() {
         for await (const rawQuads of this.memberStream.strings()) {
-            this.logger.debug(`Raw member data received: \n${rawQuads}`);
             const quads = new Parser().parse(rawQuads);
             this.logger.verbose(`Parsed ${quads.length} quads from received member data`);
             const store = RdfStore.createDefault();
-            quads.forEach(q => store.addQuad(q));
+            quads.forEach(q => {
+                if (q.graph.equals(df.defaultGraph()) && this.config.targetNamedGraph) {
+                    store.addQuad(df.quad(q.subject, q.predicate, q.object, df.namedNode(this.config.targetNamedGraph)));
+                }
+                else {
+                    store.addQuad(q);
+                }
+            });
             let query;
             const memberIRI = getObjects(store, null, SDS.terms.payload, SDS.terms.custom("DataDescription"))[0];
             if (memberIRI) {
@@ -92,21 +98,19 @@ export class SPARQLIngest extends Processor {
                         this.transactionMembers = [];
                     }
                     else {
-                        const ng = this.getNamedGraphIfAny(memberIRI, this.config.memberIsGraph, this.config.targetNamedGraph);
                         const ctv = store.getQuads(null, df.namedNode(this.config.changeSemantics.changeTypePath))[0];
-                        store.removeQuad(ctv);
                         sanitizeQuads(store);
                         if (ctv.object.value === this.config.changeSemantics.createValue) {
                             this.logger.info(`Preparing 'INSERT DATA {}' SPARQL query for member ${memberIRI.value}`);
-                            query = CREATE(store, this.config.forVirtuoso, ng);
+                            query = CREATE(store, this.config.forVirtuoso);
                         }
                         else if (ctv.object.value === this.config.changeSemantics.updateValue) {
                             this.logger.info(`Preparing 'DELETE {} INSERT {} WHERE {}' SPARQL query for member ${memberIRI.value}`);
-                            query = UPDATE(store, this.config.forVirtuoso, ng);
+                            query = UPDATE(store, this.config.forVirtuoso);
                         }
                         else if (ctv.object.value === this.config.changeSemantics.deleteValue) {
                             this.logger.info(`Preparing 'DELETE WHERE {}' SPARQL query for member ${memberIRI.value}`);
-                            query = [DELETE(store, [memberIRI.value], this.config.memberShapes, ng)];
+                            query = DELETE(store, memberIRI.value, this.config.memberShape);
                         }
                         else {
                             this.logger.error(`[sparqlIngest] Unrecognized change type value: ${ctv.object.value}`);
@@ -117,42 +121,45 @@ export class SPARQLIngest extends Processor {
                 else {
                     if (this.transactionMembers.length > 0) {
                         this.transactionMembers.forEach(ts => {
-                            ts.store.getQuads(null, null, null, null).forEach(q => store.addQuad(q));
+                            ts.store.getQuads().forEach(q => store.addQuad(q));
                         });
                         this.logger.info(`Preparing 'DELETE {} WHERE {} + INSERT DATA {}' SPARQL query for transaction member ${memberIRI.value}`);
-                        query = UPDATE(store, this.config.forVirtuoso, this.config.targetNamedGraph);
+                        query = UPDATE(store, this.config.forVirtuoso);
                     }
                     else {
                         if (this.config.operationMode === OperationMode.REPLICATION) {
-                            this.memberBatch.push(...store.getQuads(null, null, null, null));
+                            this.memberBatch.push(...store.getQuads().map(q => {
+                                if (q.graph.equals(df.defaultGraph()) && this.config.targetNamedGraph) {
+                                    q.graph = df.namedNode(this.config.targetNamedGraph);
+                                }
+                                return q;
+                            }));
                             this.batchCount++;
                             if (this.batchCount < this.config.memberBatchSize) {
                                 continue;
                             }
                         }
                         else {
-                            const ng = this.getNamedGraphIfAny(memberIRI, this.config.memberIsGraph, this.config.targetNamedGraph);
-                            this.logger.info(`Preparing 'DELETE {} WHERE {} + INSERT DATA {}' SPARQL query for member ${memberIRI.value}`);
-                            query = UPDATE(store, this.config.forVirtuoso, ng);
+                            this.logger.info(`Preparing 'DELETE {} WHERE {} + INSERT DATA {}' SPARQL queries for member ${memberIRI.value}`);
+                            query = UPDATE(store, this.config.forVirtuoso);
                         }
                     }
                 }
             }
             else {
                 if (this.config.operationMode === OperationMode.REPLICATION) {
-                    this.memberBatch.push(...store.getQuads(null, null, null, null));
+                    this.memberBatch.push(...store.getQuads());
                     this.batchCount++;
                     if (this.batchCount < this.config.memberBatchSize) {
                         continue;
                     }
                 }
                 else {
-                    this.logger.info(`Preparing 'DELETE {} WHERE {} + INSERT DATA {}' SPARQL query for received triples (${store.size})`);
-                    query = UPDATE(store, this.config.forVirtuoso, this.config.targetNamedGraph);
+                    this.logger.info(`Preparing 'DELETE {} WHERE {} + INSERT DATA {}' SPARQL queries for received quads (${store.size})`);
+                    query = UPDATE(store, this.config.forVirtuoso);
                 }
             }
             if (query && query.length > 0) {
-                this.logger.debug(`Complete SPARQL query generated for received member: \n${query.join("\n")}`);
                 if (this.config.graphStoreUrl) {
                     try {
                         const t0 = Date.now();
@@ -176,7 +183,14 @@ export class SPARQLIngest extends Processor {
                     }
                 }
                 if (this.sparqlWriter) {
-                    await this.sparqlWriter.string(query.join("\n"));
+                    if (this.config.forVirtuoso) {
+                        for (const q of query) {
+                            await this.sparqlWriter.string(q);
+                        }
+                    }
+                    else {
+                        await this.sparqlWriter.string(query.join("\n"));
+                    }
                 }
             }
             else {
@@ -256,16 +270,6 @@ export class SPARQLIngest extends Processor {
             }
         }
     }
-    getNamedGraphIfAny(memberIRI, memberIsGraph, targetNamedGraph) {
-        let ng;
-        if (memberIsGraph) {
-            ng = memberIRI.value;
-        }
-        else if (targetNamedGraph) {
-            ng = targetNamedGraph;
-        }
-        return ng;
-    }
     createTransactionQueries(transactionMembers, config) {
         this.createTransactionQueriesLogger.info(`Creating multi-operation SPARQL UPDATE query for ${transactionMembers.length}`
             + ` members of transaction ${transactionMembers[0].transactionId}`);
@@ -276,15 +280,14 @@ export class SPARQLIngest extends Processor {
         const transactionQueryBuilder = [];
         for (const tsm of transactionMembers) {
             const ctv = tsm.store.getQuads(null, df.namedNode(config.changeSemantics.changeTypePath))[0];
-            tsm.store.removeQuad(ctv);
             if (ctv.object.value === config.changeSemantics.createValue) {
-                tsm.store.getQuads(null, null, null, null).forEach(q => createStore.addQuad(q));
+                tsm.store.getQuads().forEach(q => createStore.addQuad(q));
             }
             else if (ctv.object.value === config.changeSemantics.updateValue) {
-                tsm.store.getQuads(null, null, null, null).forEach(q => updateStore.addQuad(q));
+                tsm.store.getQuads().forEach(q => updateStore.addQuad(q));
             }
             else if (ctv.object.value === config.changeSemantics.deleteValue) {
-                tsm.store.getQuads(null, null, null, null).forEach(q => deleteStore.addQuad(q));
+                tsm.store.getQuads().forEach(q => deleteStore.addQuad(q));
                 deleteMembers.push(tsm.memberId);
             }
             else {
@@ -293,14 +296,16 @@ export class SPARQLIngest extends Processor {
             }
         }
         if (createStore.size > 0) {
-            transactionQueryBuilder.push(CREATE(createStore, config.forVirtuoso, config.targetNamedGraph).join("\n"));
+            transactionQueryBuilder.push(CREATE(createStore, config.forVirtuoso).join("\n"));
         }
         if (updateStore.size > 0) {
-            transactionQueryBuilder.push(UPDATE(updateStore, config.forVirtuoso, config.targetNamedGraph).join("\n"));
+            transactionQueryBuilder.push(UPDATE(updateStore, config.forVirtuoso).join("\n"));
         }
         if (deleteStore.size > 0) {
-            transactionQueryBuilder.push(DELETE(deleteStore, deleteMembers, config.memberShapes, config.targetNamedGraph));
+            deleteMembers.forEach(dm => {
+                transactionQueryBuilder.push(DELETE(deleteStore, dm, config.memberShape).join("\n"));
+            });
         }
-        return transactionQueryBuilder.join(";\n");
+        return transactionQueryBuilder.join("\n");
     }
 }

package/lib/SPARQLQueries.d.ts

@@ -1,4 +1,4 @@
 import { RdfStore } from "rdf-stores";
-export declare const CREATE: (store: RdfStore, forVirtuoso?: boolean, namedGraph?: string, multipleNamedGraphs?: boolean) => string[];
-export declare const UPDATE: (store: RdfStore, forVirtuoso?: boolean, namedGraph?: string, multipleNamedGraphs?: boolean) => string[];
-export declare const DELETE: (store: RdfStore, memberIRIs: string[], memberShapes?: string[], namedGraph?: string, multipleNamedGraphs?: boolean) => string;
+export declare const CREATE: (store: RdfStore, forVirtuoso?: boolean) => string[];
+export declare const UPDATE: (store: RdfStore, forVirtuoso?: boolean) => string[];
+export declare const DELETE: (store: RdfStore, memberIRI: string, memberShape?: string) => string[];

package/lib/SPARQLQueries.js

@@ -2,73 +2,81 @@ import { RDF, SHACL } from "@treecg/types";
 import { Writer as N3Writer, Parser } from "n3";
 import { RdfStore } from "rdf-stores";
 import { DataFactory } from "rdf-data-factory";
-import { getObjects, getSubjects, splitStore } from "./Utils.js";
+import { getObjects, getSubjects, splitStoreOnSize, splitStorePerNamedGraph } from "./Utils.js";
 const df = new DataFactory();
-export const CREATE = (store, forVirtuoso, namedGraph, multipleNamedGraphs) => {
-    const stores = splitStore(store, forVirtuoso ? 500 : 50000);
-    return stores.map((subStore, i) => {
-        return `
-            INSERT DATA {
-                ${namedGraph ? `GRAPH <${namedGraph}> {` : ""}
-                    ${new N3Writer().quadsToString(subStore.getQuads())}
-                ${namedGraph ? `}` : ""}
-            }
-            ${i === stores.length - 1 ? "" : ";"}
-        `;
-    });
+export const CREATE = (store, forVirtuoso) => {
+    const queries = [];
+    const storesPerGraph = splitStorePerNamedGraph(store);
+    for (const { graph, store } of storesPerGraph) {
+        const subStores = splitStoreOnSize(store, forVirtuoso ? 500 : 50000);
+        subStores.forEach((s, i) => {
+            queries.push(`
+                INSERT DATA {
+                    ${graph.equals(df.defaultGraph()) ? "" : `GRAPH <${graph.value}> {`}
+                        ${new N3Writer().quadsToString(s.getQuads().map(q => {
+                return df.quad(q.subject, q.predicate, q.object, df.defaultGraph());
+            }))}
+                    ${graph.equals(df.defaultGraph()) ? "" : `}`}
+                };
+            `);
+        });
+    }
+    return queries;
 };
-export const UPDATE = (store, forVirtuoso, namedGraph, multipleNamedGraphs) => {
-    const formattedQuery = formatQuery(store);
-    const stores = splitStore(store, forVirtuoso ? 500 : 50000);
-    const queries = [
-        `
-            ${namedGraph ? `WITH <${namedGraph}>` : ""}
+export const UPDATE = (store, forVirtuoso) => {
+    const queries = [];
+    const storesPerGraph = splitStorePerNamedGraph(store);
+    for (const { graph, store } of storesPerGraph) {
+        const subStores = splitStoreOnSize(store, forVirtuoso ? 500 : 50000);
+        const formattedQuery = formatQuery(store);
+        const deleteInsertQuery = [`
+            ${graph.equals(df.defaultGraph()) ? "" : `WITH <${graph.value}>`}
             DELETE { 
                 ${formattedQuery[0]} 
             }
             WHERE { 
                 ${formattedQuery[0]} 
             };
-        `
-    ];
-    stores.forEach((subStore, i) => {
-        queries.push(`
-            INSERT DATA {
-                ${namedGraph ? `GRAPH <${namedGraph}> {` : ""}
-                    ${new N3Writer().quadsToString(subStore.getQuads())}
-                ${namedGraph ? `}` : ""}
-            }
-            ${i === stores.length - 1 ? "" : ";"}
-        `);
-    });
+        `];
+        subStores.forEach((s, i) => {
+            deleteInsertQuery.push(`
+                INSERT DATA {
+                    ${graph.equals(df.defaultGraph()) ? "" : `GRAPH <${graph.value}> {`}
+                        ${new N3Writer().quadsToString(s.getQuads().map(q => {
+                return df.quad(q.subject, q.predicate, q.object, df.defaultGraph());
+            }))}
+                    ${graph.equals(df.defaultGraph()) ? "" : `}`}
+                };
+            `);
+        });
+        queries.push(...deleteInsertQuery);
+    }
     return queries;
 };
-export const DELETE = (store, memberIRIs, memberShapes, namedGraph, multipleNamedGraphs) => {
-    const deleteBuilder = [];
-    const whereBuilder = [];
-    let indexStart = 0;
-    for (const memberIRI of memberIRIs) {
-        const formatted = formatQuery(store, memberIRI, memberShapes, indexStart);
-        deleteBuilder.push(formatted.length > 1 ? formatted[1] : formatted[0]);
-        whereBuilder.push(formatted[0]);
-        indexStart++;
+export const DELETE = (store, memberIRI, memberShape) => {
+    const queries = [];
+    const storesPerGraph = splitStorePerNamedGraph(store);
+    for (const { graph, store } of storesPerGraph) {
+        const formatted = formatQuery(store, memberIRI, memberShape);
+        const deleteBuilder = formatted.length > 1 ? formatted[1] : formatted[0];
+        const whereBuilder = formatted[0];
+        queries.push(`
+            ${graph.equals(df.defaultGraph()) ? "" : `WITH <${graph.value}>`}
+            DELETE {
+                ${deleteBuilder}
+            } WHERE {
+                ${whereBuilder}
+            };
+        `);
     }
-    return `
-        ${namedGraph ? `WITH <${namedGraph}>` : ""}
-        DELETE {
-            ${deleteBuilder.join("\n")}
-        } WHERE {
-            ${whereBuilder.join("\n")}
-        }
-    `;
+    return queries;
 };
-function formatQuery(memberStore, memberIRI, memberShapes, indexStart = 0) {
+function formatQuery(memberStore, memberIRI, memberShape, indexStart = 0) {
     const subjectSet = new Set();
     const blankNodeMap = new Map();
     const queryBuilder = [];
-    const formattedQueries = [];
     let i = indexStart;
-    if (!memberShapes || memberShapes.length === 0) {
+    if (!memberShape) {
         for (const quad of memberStore.getQuads()) {
             if (!subjectSet.has(quad.subject.value)) {
                 subjectSet.add(quad.subject.value);
@@ -82,7 +90,8 @@ function formatQuery(memberStore, memberIRI, memberShapes, indexStart = 0) {
                     if (quad.object.termType === "BlankNode") {
                         blankNodeMap.set(quad.object.value, `?bn_ref_${i}`);
                     }
-                    queryBuilder.push(`${blankNodeMap.get(quad.subject.value)} <${quad.predicate.value}> ${quad.object.termType === "Literal" ? `"${quad.object.value}"^^<${quad.object.datatype.value}>`
+                    queryBuilder.push(`${blankNodeMap.get(quad.subject.value)} <${quad.predicate.value}> ${quad.object.termType === "Literal"
+                        ? `"${quad.object.value}"^^<${quad.object.datatype.value}>`
                         : quad.object.termType === "BlankNode" ? `${blankNodeMap.get(quad.object.value)} `
                             : `<${quad.object.value}>`}.`);
                     queryBuilder.push(`${blankNodeMap.get(quad.subject.value)} ?p_${i} ?o_${i}.`);
@@ -91,51 +100,28 @@ function formatQuery(memberStore, memberIRI, memberShapes, indexStart = 0) {
                 i++;
             }
         }
-        formattedQueries.push(queryBuilder.join("\n"));
+        return [queryBuilder.join("\n")];
     }
     else {
-        const shapeIndex = new Map();
-        memberShapes.forEach(msh => {
-            const shapeStore = RdfStore.createDefault();
-            new Parser().parse(msh).forEach(quad => shapeStore.addQuad(quad));
-            shapeIndex.set(extractMainTargetClass(shapeStore).value, shapeStore);
-        });
+        const shapeStore = RdfStore.createDefault();
+        new Parser().parse(memberShape).forEach(quad => shapeStore.addQuad(quad));
         queryBuilder.push(`<${memberIRI}> ?p_${i} ?o_${i}.`);
-        const memberType = getObjects(memberStore, df.namedNode(memberIRI), RDF.terms.type)[0];
-        if (memberType) {
-            i++;
-            const mshStore = shapeIndex.get(memberType.value);
-            const propShapes = getObjects(mshStore, null, SHACL.terms.property, null);
-            for (const propSh of propShapes) {
-                const pred = getObjects(mshStore, propSh, SHACL.terms.path, null)[0];
-                queryBuilder.push(`<${memberIRI}> <${pred.value}> ?subEnt_${i}.`);
-                queryBuilder.push(`?subEnt_${i} ?p_${i} ?o_${i}.`);
-                i++;
-            }
-            formattedQueries.push(queryBuilder.join("\n"));
-        }
-        else {
-            const deleteQueryBuilder = [];
-            deleteQueryBuilder.push(`<${memberIRI}> ?p_${i} ?o_${i}.`);
+        const deleteQueryBuilder = [];
+        deleteQueryBuilder.push(`<${memberIRI}> ?p_${i} ?o_${i}.`);
+        i++;
+        const propShapes = getObjects(shapeStore, null, SHACL.terms.property, null);
+        queryBuilder.push(" OPTIONAL { ");
+        for (const propSh of propShapes) {
+            const pred = getObjects(shapeStore, propSh, SHACL.terms.path, null)[0];
+            queryBuilder.push(`<${memberIRI}> <${pred.value}> ?subEnt_${i}.`);
+            deleteQueryBuilder.push(`<${memberIRI}> <${pred.value}> ?subEnt_${i}.`);
+            queryBuilder.push(`?subEnt_${i} ?p_${i} ?o_${i}.`);
+            deleteQueryBuilder.push(`?subEnt_${i} ?p_${i} ?o_${i}.`);
             i++;
-            shapeIndex.forEach(mshStore => {
-                const propShapes = getObjects(mshStore, null, SHACL.terms.property, null);
-                queryBuilder.push(" OPTIONAL { ");
-                for (const propSh of propShapes) {
-                    const pred = getObjects(mshStore, propSh, SHACL.terms.path, null)[0];
-                    queryBuilder.push(`<${memberIRI}> <${pred.value}> ?subEnt_${i}.`);
-                    deleteQueryBuilder.push(`<${memberIRI}> <${pred.value}> ?subEnt_${i}.`);
-                    queryBuilder.push(`?subEnt_${i} ?p_${i} ?o_${i}.`);
-                    deleteQueryBuilder.push(`?subEnt_${i} ?p_${i} ?o_${i}.`);
-                    i++;
-                }
-                queryBuilder.push(" }");
-            });
-            formattedQueries.push(queryBuilder.join("\n"));
-            formattedQueries.push(deleteQueryBuilder.join("\n"));
         }
+        queryBuilder.push(" }");
+        return [queryBuilder.join("\n"), deleteQueryBuilder.join("\n")];
     }
-    return formattedQueries;
 }
 function extractMainTargetClass(store) {
     const nodeShapes = getSubjects(store, RDF.terms.type, SHACL.terms.NodeShape, null);

package/lib/Utils.d.ts

@@ -1,9 +1,13 @@
 import { RdfStore } from "rdf-stores";
-import type { Term, Quad_Subject, Quad_Object, Quad } from "@rdfjs/types";
+import type { Term, Quad_Subject, Quad_Object, Quad, Quad_Graph } from "@rdfjs/types";
 import type { IngestConfig } from "./SPARQLIngest.js";
 import { Logger } from "winston";
 export declare function getSubjects(store: RdfStore, predicate: Term | null, object: Term | null, graph?: Term | null): Quad_Subject[];
 export declare function getObjects(store: RdfStore, subject: Term | null, predicate: Term | null, graph?: Term | null): Quad_Object[];
-export declare function splitStore(store: RdfStore, threshold: number): RdfStore[];
+export declare function splitStorePerNamedGraph(store: RdfStore): {
+    graph: Quad_Graph;
+    store: RdfStore;
+}[];
+export declare function splitStoreOnSize(store: RdfStore, threshold: number): RdfStore[];
 export declare function sanitizeQuads(store: RdfStore): void;
 export declare function doSPARQLRequest(query: string[] | Quad[], config: IngestConfig, logger: Logger): Promise<void>;

package/lib/Utils.js

@@ -14,7 +14,20 @@ export function getObjects(store, subject, predicate, graph) {
         return quad.object;
     });
 }
-export function splitStore(store, threshold) {
+export function splitStorePerNamedGraph(store) {
+    const stores = [];
+    const namedGraphs = new Set();
+    store.getQuads(null, null, null, null)
+        .forEach(q => namedGraphs.add(q.graph));
+    namedGraphs.forEach(ng => {
+        const subStore = RdfStore.createDefault();
+        const quads = store.getQuads(null, null, null, ng);
+        quads.forEach(q => subStore.addQuad(q));
+        stores.push({ graph: ng, store: subStore });
+    });
+    return stores;
+}
+export function splitStoreOnSize(store, threshold) {
     const stores = [];
     if (store.size < threshold) {
         stores.push(store);
@@ -92,7 +105,7 @@ export async function doSPARQLRequest(query, config, logger) {
             return;
         }
         let queries = [];
-        const jointQuery = query.join("\n");
+        const jointQuery = query.join(";\n");
         if (config.forVirtuoso && Buffer.byteLength(jointQuery, 'utf8') > 1e6) {
             queries = query;
         }
@@ -100,7 +113,6 @@ export async function doSPARQLRequest(query, config, logger) {
             queries.push(jointQuery);
         }
         for (const q of queries) {
-            logger.debug(`Executing SPARQL query: \n${q}`);
             const res = await fetch(config.graphStoreUrl, {
                 method: "POST",
                 headers: {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@rdfc/sparql-ingest-processor-ts",
-    "version": "2.0.3",
+    "version": "2.1.0",
     "description": "SPARQL Update function to be within RDF-Connect pipelines",
     "author": "Julián Rojas",
     "contributors": [
@@ -31,7 +31,7 @@
         "n3": "^2.0.1",
         "rdf-data-factory": "^2.0.2",
         "rdf-stores": "^2.1.1",
-        "undici": "^7.20.0",
+        "undici": "^7.21.0",
         "winston": "^3.19.0"
     },
     "devDependencies": {
@@ -50,7 +50,7 @@
         "ts-patch": "^3.3.0",
         "tsc-alias": "^1.8.16",
         "typescript": "^5.9.3",
-        "vite-tsconfig-paths": "^6.0.5",
+        "vite-tsconfig-paths": "^6.1.0",
         "vitest": "^4.0.18"
     }
 }

package/processors.ttl

@@ -43,16 +43,11 @@ rdfc:SPARQLIngest rdfc:jsImplementationOf rdfc:Processor;
     sh:datatype xsd:integer;
     sh:name "memberBatchSize";
     sh:maxCount 1;
-  ], [
-    sh:path rdfc:memberIsGraph;
-    sh:datatype xsd:boolean;
-    sh:name "memberIsGraph";
-    sh:maxCount 1;
   ], [
     sh:path rdfc:memberShape;
     sh:datatype xsd:string;
-    sh:name "memberShapes";
-    sh:minCount 0;
+    sh:name "memberShape";
+    sh:maxCount 1;
   ], [
     sh:path rdfc:changeSemantics;
     sh:class rdfc:ChangeSemantics;

package/lib/LogUtil.d.ts

@@ -1,9 +0,0 @@
-import { Logger } from "winston";
-export declare function getLoggerFor(loggable: string | Instance): Logger;
-interface Constructor {
-    name: string;
-}
-interface Instance {
-    constructor: Constructor;
-}
-export {};

package/lib/LogUtil.js

@@ -1,54 +0,0 @@
-import winston, { format } from "winston";
-const PROCESSOR_NAME = "sparql-ingest";
-const consoleTransport = new winston.transports.Console({
-    stderrLevels: [
-        "error",
-        "warn",
-        "info",
-        "http",
-        "verbose",
-        "debug",
-        "silly"
-    ]
-});
-if (typeof process !== "undefined") {
-    consoleTransport.level =
-        process.env.LOG_LEVEL ||
-            (process.env.DEBUG?.includes(PROCESSOR_NAME) ||
-                process.env.DEBUG === "*"
-                ? "debug"
-                : "info");
-}
-const classLoggers = new WeakMap();
-const stringLoggers = new Map();
-export function getLoggerFor(loggable) {
-    let logger;
-    if (typeof loggable === "string") {
-        if (stringLoggers.has(loggable)) {
-            logger = stringLoggers.get(loggable);
-        }
-        else {
-            logger = createLogger(loggable);
-            stringLoggers.set(loggable, logger);
-        }
-    }
-    else {
-        const { constructor } = loggable;
-        if (classLoggers.has(constructor)) {
-            logger = classLoggers.get(constructor);
-        }
-        else {
-            logger = createLogger(constructor.name);
-            classLoggers.set(constructor, logger);
-        }
-    }
-    return logger;
-}
-function createLogger(label) {
-    return winston.createLogger({
-        format: format.combine(format.label({ label }), format.colorize(), format.timestamp(), format.metadata({
-            fillExcept: ["level", "timestamp", "label", "message"],
-        }), format.printf(({ level: levelInner, message, label: labelInner, timestamp, }) => `${timestamp} {${PROCESSOR_NAME}} [${labelInner}] ${levelInner}: ${message}`)),
-        transports: [consoleTransport],
-    });
-}