@rdfc/sparql-ingest-processor-ts 2.1.5 → 2.1.6

package/lib/SPARQLIngest.d.ts

@@ -1,4 +1,5 @@
 import { Processor } from "@rdfc/js-runner";
+import { Agent } from "undici";
 import { RdfStore } from "rdf-stores";
 import { Logger } from "winston";
 import type { Quad, Term } from "@rdfjs/types";
@@ -46,6 +47,7 @@ type SPARQLIngestArgs = {
     sparqlWriter?: Writer;
 };
 export declare class SPARQLIngest extends Processor<SPARQLIngestArgs> {
+    protected globalDispatcher: Agent;
     protected transactionMembers: TransactionMember[];
     protected memberBatch: Quad[];
     protected requestsPerformance: number[];

package/lib/SPARQLIngest.js

@@ -1,5 +1,6 @@
 import { Processor, extendLogger } from "@rdfc/js-runner";
 import { SDS } from "@treecg/types";
+import { Agent } from "undici";
 import { DataFactory } from "rdf-data-factory";
 import { RdfStore } from "rdf-stores";
 import { Parser } from "n3";
@@ -13,6 +14,7 @@ export var OperationMode;
     OperationMode["SYNC"] = "Sync";
 })(OperationMode || (OperationMode = {}));
 export class SPARQLIngest extends Processor {
+    globalDispatcher;
     transactionMembers = [];
     memberBatch = [];
     requestsPerformance = [];
@@ -22,6 +24,10 @@ export class SPARQLIngest extends Processor {
     async init() {
         this.createTransactionQueriesLogger = extendLogger(this.logger, "createTransactionQueries");
         this.doSPARQLRequestLogger = extendLogger(this.logger, "doSPARQLRequest");
+        this.globalDispatcher = new Agent({
+            headersTimeout: (this.config.measurePerformance?.queryTimeout || 600) * 1000,
+            bodyTimeout: (this.config.measurePerformance?.queryTimeout || 600) * 1000,
+        });
         if (!this.config.operationMode) {
             this.config.operationMode = OperationMode.SYNC;
         }
@@ -159,7 +165,7 @@ export class SPARQLIngest extends Processor {
                 if (this.config.graphStoreUrl) {
                     try {
                         const t0 = Date.now();
-                        await doSPARQLRequest(query, this.config, this.logger);
+                        await doSPARQLRequest(query, this.config, this.globalDispatcher, this.logger);
                         const reqTime = Date.now() - t0;
                         if (this.config.measurePerformance) {
                             this.requestsPerformance.push(reqTime);
@@ -193,7 +199,7 @@ export class SPARQLIngest extends Processor {
                 if (this.config.operationMode === OperationMode.REPLICATION) {
                     try {
                         const t0 = Date.now();
-                        await doSPARQLRequest(this.memberBatch, this.config, this.logger);
+                        await doSPARQLRequest(this.memberBatch, this.config, this.globalDispatcher, this.logger);
                         const reqTime = Date.now() - t0;
                         if (this.config.measurePerformance) {
                             this.requestsPerformance.push(reqTime);
@@ -222,7 +228,7 @@ export class SPARQLIngest extends Processor {
         if (this.config.operationMode === OperationMode.REPLICATION && this.memberBatch.length > 0) {
             try {
                 const t0 = Date.now();
-                await doSPARQLRequest(this.memberBatch, this.config, this.logger);
+                await doSPARQLRequest(this.memberBatch, this.config, this.globalDispatcher, this.logger);
                 const reqTime = Date.now() - t0;
                 if (this.config.measurePerformance) {
                     this.requestsPerformance.push(reqTime);
@@ -250,6 +256,7 @@ export class SPARQLIngest extends Processor {
         if (this.config.measurePerformance) {
             await writeFile(`${this.config.measurePerformance.outputPath}/${this.config.measurePerformance.name}.json`, JSON.stringify(this.requestsPerformance), "utf-8");
         }
+        await this.globalDispatcher.close();
     }
     async produce() {
     }

package/lib/Utils.d.ts

@@ -1,4 +1,5 @@
 import { RdfStore } from "rdf-stores";
+import { Agent } from "undici";
 import { Logger } from "winston";
 import type { Term, Quad_Subject, Quad_Object, Quad, Quad_Graph } from "@rdfjs/types";
 import type { IngestConfig } from "./SPARQLIngest.js";
@@ -10,4 +11,4 @@ export declare function splitStorePerNamedGraph(store: RdfStore): {
 }[];
 export declare function splitStoreOnSize(store: RdfStore, threshold: number): RdfStore[];
 export declare function sanitizeQuads(store: RdfStore): void;
-export declare function doSPARQLRequest(query: string[] | Quad[], config: IngestConfig, logger: Logger): Promise<void>;
+export declare function doSPARQLRequest(query: string[] | Quad[], config: IngestConfig, dispatcher: Agent, logger: Logger): Promise<void>;

package/lib/Utils.js

@@ -1,7 +1,6 @@
 import { XSD } from "@treecg/types";
 import { DataFactory } from "rdf-data-factory";
 import { RdfStore } from "rdf-stores";
-import { Agent } from "undici";
 import { Writer as N3Writer } from "n3";
 const df = new DataFactory();
 export function getSubjects(store, predicate, object, graph) {
@@ -76,7 +75,7 @@ export function sanitizeQuads(store) {
         }
     }
 }
-export async function doSPARQLRequest(query, config, logger) {
+export async function doSPARQLRequest(query, config, dispatcher, logger) {
     try {
         const timeout = config.measurePerformance?.queryTimeout || 1800;
         if (query.length > 0 && typeof query[0] !== 'string') {
@@ -99,10 +98,7 @@ export async function doSPARQLRequest(query, config, logger) {
                     'Content-Type': hasQuads ? 'application/n-quads' : 'application/n-triples',
                 },
                 body: serialized,
-                dispatcher: new Agent({
-                    headersTimeout: timeout * 1000,
-                    bodyTimeout: timeout * 1000,
-                }),
+                dispatcher,
             });
             if (!res.ok) {
                 throw new Error(`HTTP request failed with code ${res.status} and message: \n${await res.text()}`);
@@ -125,10 +121,7 @@ export async function doSPARQLRequest(query, config, logger) {
                     'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                 },
                 body: `update=${fixedEncodeURIComponent(q)}${config.accessToken ? `&access-token=${config.accessToken}` : ''}`,
-                dispatcher: new Agent({
-                    headersTimeout: timeout * 1000,
-                    bodyTimeout: timeout * 1000,
-                }),
+                dispatcher,
             });
             if (!res.ok) {
                 throw new Error(`HTTP request failed with code ${res.status} and message: \n${await res.text()}`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@rdfc/sparql-ingest-processor-ts",
-    "version": "2.1.5",
+    "version": "2.1.6",
     "description": "SPARQL Update function to be within RDF-Connect pipelines",
     "author": "Julián Rojas",
     "contributors": [
@@ -31,7 +31,7 @@
         "n3": "^1.26.0",
         "rdf-data-factory": "^2.0.2",
         "rdf-stores": "^2.1.1",
-        "undici": "^7.21.0",
+        "undici": "^7.22.0",
         "winston": "^3.19.0"
     },
     "devDependencies": {
@@ -39,18 +39,18 @@
         "@rdfc/js-runner": "^3.0.2",
         "@rdfjs/types": "^2.0.1",
         "@types/n3": "^1.26.1",
-        "@types/node": "^25.2.2",
-        "@typescript-eslint/eslint-plugin": "^8.55.0",
-        "@typescript-eslint/parser": "^8.55.0",
+        "@types/node": "^25.3.0",
+        "@typescript-eslint/eslint-plugin": "^8.56.1",
+        "@typescript-eslint/parser": "^8.56.1",
         "@vitest/coverage-v8": "^4.0.18",
-        "eslint": "^9.39.2",
+        "eslint": "^9.39.3",
         "eslint-config-prettier": "^10.1.8",
         "fastify": "^5.7.4",
         "husky": "^9.1.7",
         "ts-patch": "^3.3.0",
         "tsc-alias": "^1.8.16",
         "typescript": "^5.9.3",
-        "vite-tsconfig-paths": "^6.1.0",
+        "vite-tsconfig-paths": "^6.1.1",
         "vitest": "^4.0.18"
     }
-}
+}