@lde/pipeline 0.6.29 → 0.6.30

package/dist/distribution/probe.d.ts

@@ -31,7 +31,9 @@ export declare class SparqlProbeResult extends ProbeResult {
  */
 export declare class DataDumpProbeResult extends ProbeResult {
     readonly contentSize: number | null;
-    constructor(url: string, response: Response);
+    readonly failureReason: string | null;
+    constructor(url: string, response: Response, failureReason?: string | null);
+    isSuccess(): boolean;
 }
 export type ProbeResultType = SparqlProbeResult | DataDumpProbeResult | NetworkError;
 /**

package/dist/distribution/probe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../../src/distribution/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C;;GAEG;AACH,qBAAa,YAAY;aACK,GAAG,EAAE,MAAM;aAAkB,OAAO,EAAE,MAAM;gBAA5C,GAAG,EAAE,MAAM,EAAkB,OAAO,EAAE,MAAM;CACzE;AAED;;GAEG;AACH,uBAAe,WAAW;aAMI,GAAG,EAAE,MAAM;IALvC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEf,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ;IAUpD,SAAS,IAAI,OAAO;CAG5B;AAED;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,SAAgB,mBAAmB,qCAAqC;IAE/D,SAAS,IAAI,OAAO;CAM9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAEtC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ;CAO5C;AAED,MAAM,MAAM,eAAe,GACvB,iBAAiB,GACjB,mBAAmB,GACnB,YAAY,CAAC;AAEjB;;;;;;;GAOG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,SAAO,GACb,OAAO,CAAC,eAAe,CAAC,CAY1B"}
+{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../../src/distribution/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;gBADf,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM;CAElC;AAED;;GAEG;AACH,uBAAe,WAAW;aAON,GAAG,EAAE,MAAM;IAN7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAGzB,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ;IAWb,SAAS,IAAI,OAAO;CAG5B;AAED;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,SAAgB,mBAAmB,qCAAqC;IAE/D,SAAS,IAAI,OAAO;CAM9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;IAClD,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;gBAG3C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,aAAa,GAAE,MAAM,GAAG,IAAW;IAU5B,SAAS,IAAI,OAAO;CAG9B;AAED,MAAM,MAAM,eAAe,GACvB,iBAAiB,GACjB,mBAAmB,GACnB,YAAY,CAAC;AAEjB;;;;;;;GAOG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,SAAO,GACb,OAAO,CAAC,eAAe,CAAC,CAY1B"}

package/dist/distribution/probe.js

@@ -1,3 +1,4 @@
+import { Parser } from 'n3';
 /**
  * Result of a network error during probing.
  */
@@ -47,13 +48,18 @@ export class SparqlProbeResult extends ProbeResult {
  */
 export class DataDumpProbeResult extends ProbeResult {
     contentSize = null;
-    constructor(url, response) {
+    failureReason;
+    constructor(url, response, failureReason = null) {
         super(url, response);
+        this.failureReason = failureReason;
         const contentLengthHeader = response.headers.get('Content-Length');
         if (contentLengthHeader) {
             this.contentSize = parseInt(contentLengthHeader);
         }
     }
+    isSuccess() {
+        return super.isSuccess() && this.failureReason === null;
+    }
 }
 /**
  * Probe a distribution to check availability and gather metadata.
@@ -96,19 +102,48 @@ async function probeDataDump(distribution, timeout) {
             'Accept-Encoding': 'identity', // Return uncompressed responses.
         },
     };
-    let response = await fetch(url, {
+    const headResponse = await fetch(url, {
         method: 'HEAD',
         ...requestOptions,
     });
-    const contentLength = response.headers.get('Content-Length');
-    if (contentLength === null || contentLength === '0') {
-        // Retry as GET request for servers incorrectly returning HEAD request Content-Length,
-        // which *should* be the size of the response body when issuing a GET, not that of
-        // the response to a HEAD request, which is intentionally 0.
-        response = await fetch(url, {
+    const contentLength = headResponse.headers.get('Content-Length');
+    const contentLengthBytes = contentLength ? parseInt(contentLength) : 0;
+    // For small or unknown-size files, do a GET to validate body content.
+    // This also handles servers that incorrectly return 0 Content-Length for HEAD.
+    if (contentLengthBytes <= 10_240) {
+        const getResponse = await fetch(url, {
             method: 'GET',
             ...requestOptions,
         });
+        const body = await getResponse.text();
+        const isHttpSuccess = getResponse.status >= 200 && getResponse.status < 400;
+        const failureReason = isHttpSuccess
+            ? validateBody(body, getResponse.headers.get('Content-Type'))
+            : null;
+        return new DataDumpProbeResult(url, getResponse, failureReason);
+    }
+    return new DataDumpProbeResult(url, headResponse);
+}
+const rdfContentTypes = [
+    'text/turtle',
+    'application/n-triples',
+    'application/n-quads',
+];
+function validateBody(body, contentType) {
+    if (body.length === 0) {
+        return 'Distribution is empty';
+    }
+    if (contentType && rdfContentTypes.some((t) => contentType.startsWith(t))) {
+        try {
+            const parser = new Parser();
+            const quads = parser.parse(body);
+            if (quads.length === 0) {
+                return 'Distribution contains no RDF triples';
+            }
+        }
+        catch (e) {
+            return e instanceof Error ? e.message : String(e);
+        }
     }
-    return new DataDumpProbeResult(url, response);
+    return null;
 }

package/dist/distribution/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/distribution/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAe,KAAK,IAAI,EAAE,MAAM,IAAI,CAAC;AAC5C,OAAO,EAIL,KAAK,eAAe,EACrB,MAAM,YAAY,CAAC;AAUpB;;;;;;;;;GASG;AACH,wBAAuB,mBAAmB,CACxC,YAAY,EAAE,eAAe,EAAE,EAC/B,UAAU,EAAE,MAAM,EAClB,YAAY,CAAC,EAAE,YAAY,GAC1B,aAAa,CAAC,IAAI,CAAC,CAkCrB"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/distribution/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAe,KAAK,IAAI,EAAE,MAAM,IAAI,CAAC;AAC5C,OAAO,EAIL,KAAK,eAAe,EACrB,MAAM,YAAY,CAAC;AAUpB;;;;;;;;;GASG;AACH,wBAAuB,mBAAmB,CACxC,YAAY,EAAE,eAAe,EAAE,EAC/B,UAAU,EAAE,MAAM,EAClB,YAAY,CAAC,EAAE,YAAY,GAC1B,aAAa,CAAC,IAAI,CAAC,CAwCrB"}

package/dist/distribution/report.js

@@ -1,5 +1,5 @@
 import { DataFactory } from 'n3';
-import { NetworkError, SparqlProbeResult, } from './probe.js';
+import { NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';
 const { quad, namedNode, blankNode, literal } = DataFactory;
 const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
 const SCHEMA = 'https://schema.org/';
@@ -30,6 +30,9 @@ export async function* probeResultsToQuads(probeResults, datasetIri, importResul
         else if (result.isSuccess()) {
             yield* successQuads(action, result, datasetIri);
         }
+        else if (result instanceof DataDumpProbeResult && result.failureReason) {
+            yield quad(action, namedNode(`${SCHEMA}error`), literal(result.failureReason));
+        }
         else {
             // HTTP error
             const statusUri = `${HTTP_STATUS}${result.statusText.replace(/ /g, '')}`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.29",
+  "version": "0.6.30",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"