@rdfc/sparql-ingest-processor-ts 2.1.4 → 2.1.6

package/lib/SPARQLIngest.d.ts

@@ -1,4 +1,5 @@
 import { Processor } from "@rdfc/js-runner";
+import { Agent } from "undici";
 import { RdfStore } from "rdf-stores";
 import { Logger } from "winston";
 import type { Quad, Term } from "@rdfjs/types";
@@ -46,6 +47,7 @@ type SPARQLIngestArgs = {
     sparqlWriter?: Writer;
 };
 export declare class SPARQLIngest extends Processor<SPARQLIngestArgs> {
+    protected globalDispatcher: Agent;
     protected transactionMembers: TransactionMember[];
     protected memberBatch: Quad[];
     protected requestsPerformance: number[];

package/lib/SPARQLIngest.js

@@ -1,5 +1,6 @@
 import { Processor, extendLogger } from "@rdfc/js-runner";
 import { SDS } from "@treecg/types";
+import { Agent } from "undici";
 import { DataFactory } from "rdf-data-factory";
 import { RdfStore } from "rdf-stores";
 import { Parser } from "n3";
@@ -13,6 +14,7 @@ export var OperationMode;
     OperationMode["SYNC"] = "Sync";
 })(OperationMode || (OperationMode = {}));
 export class SPARQLIngest extends Processor {
+    globalDispatcher;
     transactionMembers = [];
     memberBatch = [];
     requestsPerformance = [];
@@ -22,6 +24,10 @@ export class SPARQLIngest extends Processor {
     async init() {
         this.createTransactionQueriesLogger = extendLogger(this.logger, "createTransactionQueries");
         this.doSPARQLRequestLogger = extendLogger(this.logger, "doSPARQLRequest");
+        this.globalDispatcher = new Agent({
+            headersTimeout: (this.config.measurePerformance?.queryTimeout || 600) * 1000,
+            bodyTimeout: (this.config.measurePerformance?.queryTimeout || 600) * 1000,
+        });
         if (!this.config.operationMode) {
             this.config.operationMode = OperationMode.SYNC;
         }
@@ -159,7 +165,7 @@ export class SPARQLIngest extends Processor {
                 if (this.config.graphStoreUrl) {
                     try {
                         const t0 = Date.now();
-                        await doSPARQLRequest(query, this.config, this.logger);
+                        await doSPARQLRequest(query, this.config, this.globalDispatcher, this.logger);
                         const reqTime = Date.now() - t0;
                         if (this.config.measurePerformance) {
                             this.requestsPerformance.push(reqTime);
@@ -193,7 +199,7 @@ export class SPARQLIngest extends Processor {
                 if (this.config.operationMode === OperationMode.REPLICATION) {
                     try {
                         const t0 = Date.now();
-                        await doSPARQLRequest(this.memberBatch, this.config, this.logger);
+                        await doSPARQLRequest(this.memberBatch, this.config, this.globalDispatcher, this.logger);
                         const reqTime = Date.now() - t0;
                         if (this.config.measurePerformance) {
                             this.requestsPerformance.push(reqTime);
@@ -222,7 +228,7 @@ export class SPARQLIngest extends Processor {
         if (this.config.operationMode === OperationMode.REPLICATION && this.memberBatch.length > 0) {
             try {
                 const t0 = Date.now();
-                await doSPARQLRequest(this.memberBatch, this.config, this.logger);
+                await doSPARQLRequest(this.memberBatch, this.config, this.globalDispatcher, this.logger);
                 const reqTime = Date.now() - t0;
                 if (this.config.measurePerformance) {
                     this.requestsPerformance.push(reqTime);
@@ -250,6 +256,7 @@ export class SPARQLIngest extends Processor {
         if (this.config.measurePerformance) {
             await writeFile(`${this.config.measurePerformance.outputPath}/${this.config.measurePerformance.name}.json`, JSON.stringify(this.requestsPerformance), "utf-8");
         }
+        await this.globalDispatcher.close();
     }
     async produce() {
     }

package/lib/SPARQLQueries.js

@@ -1,8 +1,8 @@
-import { RDF, SHACL } from "@treecg/types";
+import { SHACL } from "@treecg/types";
 import { Writer as N3Writer, Parser } from "n3";
 import { RdfStore } from "rdf-stores";
 import { DataFactory } from "rdf-data-factory";
-import { getObjects, getSubjects, splitStoreOnSize, splitStorePerNamedGraph } from "./Utils.js";
+import { getObjects, splitStoreOnSize, splitStorePerNamedGraph } from "./Utils.js";
 const df = new DataFactory();
 export const CREATE = (store, forVirtuoso) => {
     const queries = [];
@@ -123,36 +123,3 @@ function formatQuery(memberStore, memberIRI, memberShape, indexStart = 0) {
         return [queryBuilder.join("\n"), deleteQueryBuilder.join("\n")];
     }
 }
-function extractMainTargetClass(store) {
-    const nodeShapes = getSubjects(store, RDF.terms.type, SHACL.terms.NodeShape, null);
-    let mainNodeShape = null;
-    if (nodeShapes && nodeShapes.length > 0) {
-        for (const ns of nodeShapes) {
-            const isNotReferenced = getSubjects(store, null, ns, null).length === 0;
-            if (isNotReferenced) {
-                if (!mainNodeShape) {
-                    mainNodeShape = ns;
-                }
-                else {
-                    throw new Error("There are multiple main node shapes in a given shape."
-                        + " Unrelated shapes must be given as separate member shapes");
-                }
-            }
-        }
-        if (mainNodeShape) {
-            const tcq = getObjects(store, mainNodeShape, SHACL.terms.targetClass, null)[0];
-            if (tcq) {
-                return tcq;
-            }
-            else {
-                throw new Error("No target class found in main SHACL Node Shapes");
-            }
-        }
-        else {
-            throw new Error("No main SHACL Node Shapes found in given member shape");
-        }
-    }
-    else {
-        throw new Error("No SHACL Node Shapes found in given member shape");
-    }
-}

package/lib/Utils.d.ts

@@ -1,4 +1,5 @@
 import { RdfStore } from "rdf-stores";
+import { Agent } from "undici";
 import { Logger } from "winston";
 import type { Term, Quad_Subject, Quad_Object, Quad, Quad_Graph } from "@rdfjs/types";
 import type { IngestConfig } from "./SPARQLIngest.js";
@@ -10,4 +11,4 @@ export declare function splitStorePerNamedGraph(store: RdfStore): {
 }[];
 export declare function splitStoreOnSize(store: RdfStore, threshold: number): RdfStore[];
 export declare function sanitizeQuads(store: RdfStore): void;
-export declare function doSPARQLRequest(query: string[] | Quad[], config: IngestConfig, logger: Logger): Promise<void>;
+export declare function doSPARQLRequest(query: string[] | Quad[], config: IngestConfig, dispatcher: Agent, logger: Logger): Promise<void>;

package/lib/Utils.js

@@ -1,7 +1,6 @@
 import { XSD } from "@treecg/types";
 import { DataFactory } from "rdf-data-factory";
 import { RdfStore } from "rdf-stores";
-import { Agent } from "undici";
 import { Writer as N3Writer } from "n3";
 const df = new DataFactory();
 export function getSubjects(store, predicate, object, graph) {
@@ -76,7 +75,7 @@ export function sanitizeQuads(store) {
         }
     }
 }
-export async function doSPARQLRequest(query, config, logger) {
+export async function doSPARQLRequest(query, config, dispatcher, logger) {
     try {
         const timeout = config.measurePerformance?.queryTimeout || 1800;
         if (query.length > 0 && typeof query[0] !== 'string') {
@@ -99,10 +98,7 @@ export async function doSPARQLRequest(query, config, logger) {
                     'Content-Type': hasQuads ? 'application/n-quads' : 'application/n-triples',
                 },
                 body: serialized,
-                dispatcher: new Agent({
-                    headersTimeout: timeout * 1000,
-                    bodyTimeout: timeout * 1000,
-                }),
+                dispatcher,
             });
             if (!res.ok) {
                 throw new Error(`HTTP request failed with code ${res.status} and message: \n${await res.text()}`);
@@ -118,7 +114,6 @@ export async function doSPARQLRequest(query, config, logger) {
             queries.push(jointQuery);
         }
         for (const q of queries) {
-            logger.error(q);
             logger.debug(`[doSPARQLRequest] Executing SPARQL query: \n${q}`);
             const res = await fetch(config.graphStoreUrl, {
                 method: "POST",
@@ -126,10 +121,7 @@ export async function doSPARQLRequest(query, config, logger) {
                     'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                 },
                 body: `update=${fixedEncodeURIComponent(q)}${config.accessToken ? `&access-token=${config.accessToken}` : ''}`,
-                dispatcher: new Agent({
-                    headersTimeout: timeout * 1000,
-                    bodyTimeout: timeout * 1000,
-                }),
+                dispatcher,
             });
             if (!res.ok) {
                 throw new Error(`HTTP request failed with code ${res.status} and message: \n${await res.text()}`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@rdfc/sparql-ingest-processor-ts",
-    "version": "2.1.4",
+    "version": "2.1.6",
     "description": "SPARQL Update function to be within RDF-Connect pipelines",
     "author": "Julián Rojas",
     "contributors": [
@@ -31,7 +31,7 @@
         "n3": "^1.26.0",
         "rdf-data-factory": "^2.0.2",
         "rdf-stores": "^2.1.1",
-        "undici": "^7.21.0",
+        "undici": "^7.22.0",
         "winston": "^3.19.0"
     },
     "devDependencies": {
@@ -39,18 +39,18 @@
         "@rdfc/js-runner": "^3.0.2",
         "@rdfjs/types": "^2.0.1",
         "@types/n3": "^1.26.1",
-        "@types/node": "^25.2.2",
-        "@typescript-eslint/eslint-plugin": "^8.55.0",
-        "@typescript-eslint/parser": "^8.55.0",
+        "@types/node": "^25.3.0",
+        "@typescript-eslint/eslint-plugin": "^8.56.1",
+        "@typescript-eslint/parser": "^8.56.1",
         "@vitest/coverage-v8": "^4.0.18",
-        "eslint": "^9.39.2",
+        "eslint": "^9.39.3",
         "eslint-config-prettier": "^10.1.8",
         "fastify": "^5.7.4",
         "husky": "^9.1.7",
         "ts-patch": "^3.3.0",
         "tsc-alias": "^1.8.16",
         "typescript": "^5.9.3",
-        "vite-tsconfig-paths": "^6.1.0",
+        "vite-tsconfig-paths": "^6.1.1",
         "vitest": "^4.0.18"
     }
-}
+}