@lde/pipeline 0.25.3 → 0.26.0

package/dist/distribution/probe.d.ts

@@ -16,7 +16,8 @@ declare abstract class ProbeResult {
     readonly statusText: string;
     readonly lastModified: Date | null;
     readonly contentType: string | null;
-    constructor(url: string, response: Response);
+    readonly failureReason: string | null;
+    constructor(url: string, response: Response, failureReason?: string | null);
     isSuccess(): boolean;
 }
 /**
@@ -31,9 +32,7 @@ export declare class SparqlProbeResult extends ProbeResult {
  */
 export declare class DataDumpProbeResult extends ProbeResult {
     readonly contentSize: number | null;
-    readonly failureReason: string | null;
     constructor(url: string, response: Response, failureReason?: string | null);
-    isSuccess(): boolean;
 }
 export type ProbeResultType = SparqlProbeResult | DataDumpProbeResult | NetworkError;
 /**

package/dist/distribution/probe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../../src/distribution/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;gBADf,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM;CAElC;AAED;;GAEG;AACH,uBAAe,WAAW;aAON,GAAG,EAAE,MAAM;IAN7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAGzB,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ;IAWb,SAAS,IAAI,OAAO;CAG5B;AAED;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,SAAgB,mBAAmB,qCAAqC;IAE/D,SAAS,IAAI,OAAO;CAM9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;IAClD,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;gBAG3C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,aAAa,GAAE,MAAM,GAAG,IAAW;IAU5B,SAAS,IAAI,OAAO;CAG9B;AAED,MAAM,MAAM,eAAe,GACvB,iBAAiB,GACjB,mBAAmB,GACnB,YAAY,CAAC;AAEjB;;;;;;;GAOG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,SAAO,GACb,OAAO,CAAC,eAAe,CAAC,CAY1B"}
+{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../../src/distribution/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;gBADf,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM;CAElC;AAED;;GAEG;AACH,uBAAe,WAAW;aAQN,GAAG,EAAE,MAAM;IAP7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;gBAG3B,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ,EAClB,aAAa,GAAE,MAAM,GAAG,IAAW;IAY9B,SAAS,IAAI,OAAO;CAO5B;AAID;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,SAAgB,mBAAmB,qCAAuB;IAEjD,SAAS,IAAI,OAAO;CAM9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAGhD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,aAAa,GAAE,MAAM,GAAG,IAAW;CAQtC;AAED,MAAM,MAAM,eAAe,GACvB,iBAAiB,GACjB,mBAAmB,GACnB,YAAY,CAAC;AAEjB;;;;;;;GAOG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,SAAO,GACb,OAAO,CAAC,eAAe,CAAC,CAY1B"}

package/dist/distribution/probe.js

@@ -19,25 +19,30 @@ class ProbeResult {
     statusText;
     lastModified = null;
     contentType;
-    constructor(url, response) {
+    failureReason;
+    constructor(url, response, failureReason = null) {
         this.url = url;
         this.statusCode = response.status;
         this.statusText = response.statusText;
         this.contentType = response.headers.get('Content-Type');
+        this.failureReason = failureReason;
         const lastModifiedHeader = response.headers.get('Last-Modified');
         if (lastModifiedHeader) {
             this.lastModified = new Date(lastModifiedHeader);
         }
     }
     isSuccess() {
-        return this.statusCode >= 200 && this.statusCode < 400;
+        return (this.statusCode >= 200 &&
+            this.statusCode < 400 &&
+            this.failureReason === null);
     }
 }
+const SPARQL_RESULTS_JSON = 'application/sparql-results+json';
 /**
  * Result of probing a SPARQL endpoint.
  */
 export class SparqlProbeResult extends ProbeResult {
-    acceptedContentType = 'application/sparql-results+json';
+    acceptedContentType = SPARQL_RESULTS_JSON;
     isSuccess() {
         return (super.isSuccess() &&
             (this.contentType?.startsWith(this.acceptedContentType) ?? false));
@@ -48,18 +53,13 @@ export class SparqlProbeResult extends ProbeResult {
  */
 export class DataDumpProbeResult extends ProbeResult {
     contentSize = null;
-    failureReason;
     constructor(url, response, failureReason = null) {
-        super(url, response);
-        this.failureReason = failureReason;
+        super(url, response, failureReason);
         const contentLengthHeader = response.headers.get('Content-Length');
         if (contentLengthHeader) {
             this.contentSize = parseInt(contentLengthHeader);
         }
     }
-    isSuccess() {
-        return super.isSuccess() && this.failureReason === null;
-    }
 }
 /**
  * Probe a distribution to check availability and gather metadata.
@@ -87,11 +87,38 @@ async function probeSparqlEndpoint(distribution, timeout) {
         method: 'POST',
         headers: {
             'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
-            Accept: 'application/sparql-results+json',
+            Accept: SPARQL_RESULTS_JSON,
         },
         body: `query=${encodeURIComponent('SELECT * { ?s ?p ?o } LIMIT 1')}`,
     });
-    return new SparqlProbeResult(url, response);
+    const isJsonResponse = response.headers
+        .get('Content-Type')
+        ?.startsWith(SPARQL_RESULTS_JSON);
+    let failureReason = null;
+    if (response.ok && isJsonResponse) {
+        failureReason = await validateSparqlResponse(response);
+    }
+    else {
+        // Drain unconsumed body to release the underlying connection.
+        await response.body?.cancel();
+    }
+    return new SparqlProbeResult(url, response, failureReason);
+}
+async function validateSparqlResponse(response) {
+    const body = await response.text();
+    if (body.length === 0) {
+        return 'SPARQL endpoint returned an empty response';
+    }
+    try {
+        const json = JSON.parse(body);
+        if (!json.results || typeof json.results !== 'object') {
+            return 'SPARQL endpoint did not return a valid results object';
+        }
+    }
+    catch {
+        return 'SPARQL endpoint returned invalid JSON';
+    }
+    return null;
 }
 async function probeDataDump(distribution, timeout) {
     const url = distribution.accessUrl.toString();

package/dist/distribution/report.js

@@ -1,5 +1,5 @@
 import { DataFactory } from 'n3';
-import { NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';
+import { NetworkError, SparqlProbeResult, } from './probe.js';
 const { quad, namedNode, blankNode, literal } = DataFactory;
 const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
 const SCHEMA = 'https://schema.org/';
@@ -30,7 +30,7 @@ export async function* probeResultsToQuads(probeResults, datasetIri, importResul
         else if (result.isSuccess()) {
             yield* successQuads(action, result, datasetIri);
         }
-        else if (result instanceof DataDumpProbeResult && result.failureReason) {
+        else if (result.failureReason) {
             yield quad(action, namedNode(`${SCHEMA}error`), literal(result.failureReason));
         }
         else {

package/dist/writer/sparqlUpdateWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sparqlUpdateWriter.d.ts","sourceRoot":"","sources":["../../src/writer/sparqlUpdateWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAiB,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;GAMG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;gBAEvC,OAAO,EAAE,mBAAmB;IAOlC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAc1D,UAAU;YAIV,WAAW;YAOX,aAAa;CAsB5B"}
+{"version":3,"file":"sparqlUpdateWriter.d.ts","sourceRoot":"","sources":["../../src/writer/sparqlUpdateWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAiB,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,QAAQ,EAAE,GAAG,CAAC;IACd;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;GAMG;AACH,qBAAa,kBAAmB,YAAW,MAAM;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAC/B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;gBAEvC,OAAO,EAAE,mBAAmB;IAOlC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAc1D,UAAU;YAIV,WAAW;YAOX,aAAa;CAuB5B"}

package/dist/writer/sparqlUpdateWriter.js

@@ -55,5 +55,6 @@ export class SparqlUpdateWriter {
             const body = await response.text();
             throw new Error(`SPARQL UPDATE failed with status ${response.status}: ${body}`);
         }
+        await response.body?.cancel();
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.25.3",
+  "version": "0.26.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"