@lde/distribution-probe 0.1.0

package/README.md

@@ -0,0 +1,35 @@
+# Distribution Probe
+
+Probes a DCAT `Distribution` to check availability and gather metadata. Returns `SparqlProbeResult`, `DataDumpProbeResult`, or `NetworkError` – the probe never throws.
+
+```ts
+import { Distribution } from '@lde/dataset';
+import { probe } from '@lde/distribution-probe';
+
+const distribution = new Distribution(
+  new URL('https://example.org/data.ttl'),
+  'text/turtle',
+);
+const result = await probe(distribution);
+```
+
+## Behaviour
+
+### SPARQL endpoints
+
+Sends `POST` with `SELECT * { ?s ?p ?o } LIMIT 1` and `Accept: application/sparql-results+json`, then:
+
+- **Content-Type is enforced.** The response Content-Type must start with `application/sparql-results+json`; anything else fails the probe (`isSuccess() === false`). This rules out HTML error pages served with `200 OK`.
+- The JSON body must parse and contain a `results` object. Empty bodies, invalid JSON, and missing `results` all fail the probe with a `failureReason`.
+
+### Data dumps
+
+Sends `HEAD` with `Accept: <distribution.mimeType>` and `Accept-Encoding: identity`. If `Content-Length` is missing or ≤ 10 KB, retries with `GET` to validate the body – this also catches servers that return `0` from `HEAD`.
+
+- **Content-Type is checked as a soft warning, not a hard failure.** If the server’s Content-Type disagrees with the distribution’s declared `mimeType`, a message is appended to `result.warnings` but `isSuccess()` stays `true`. Compression wrappers (`application/gzip`, `application/x-gzip`, `application/octet-stream`) are skipped so a gzipped Turtle file doesn’t trigger a warning.
+- **Body is parse-validated only for Turtle, N-Triples, and N-Quads** (Content-Type starting with `text/turtle`, `application/n-triples`, or `application/n-quads`). Empty bodies and parse errors fail the probe. Other RDF serializations (RDF/XML, JSON-LD, TriG, …) are not parse-validated – only HTTP status and headers are checked.
+- Bodies larger than 10 KB are not fetched; only `HEAD` metadata is inspected.
+
+### Network errors
+
+Any thrown exception from `fetch` (DNS, connection refused, TLS, timeout after the configured `timeout` – default 5 000 ms) is caught and returned as a `NetworkError` with the original message.

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, type ProbeResultType, } from './probe.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1 @@
+export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';

package/dist/probe.d.ts

@@ -0,0 +1,76 @@
+import { Distribution } from '@lde/dataset';
+/**
+ * Options for {@link probe}.
+ */
+export interface ProbeOptions {
+    /** Request timeout in milliseconds. Defaults to 5 000. */
+    timeoutMs?: number;
+    /**
+     * Extra HTTP headers to send with the request. Merged with probe-generated
+     * headers; caller-supplied values take precedence on conflict.
+     */
+    headers?: Headers;
+    /**
+     * SPARQL query to use when probing a SPARQL endpoint. The query’s type
+     * (`ASK` / `SELECT` / `CONSTRUCT` / `DESCRIBE`) determines the `Accept`
+     * header and the response validation strategy. Ignored for data-dump
+     * distributions. Defaults to `SELECT * { ?s ?p ?o } LIMIT 1`.
+     */
+    sparqlQuery?: string;
+}
+/**
+ * Result of a network error during probing.
+ */
+export declare class NetworkError {
+    readonly url: string;
+    readonly message: string;
+    readonly responseTimeMs: number;
+    constructor(url: string, message: string, responseTimeMs: number);
+}
+/**
+ * Base class for successful probe results.
+ */
+declare abstract class ProbeResult {
+    readonly url: string;
+    readonly statusCode: number;
+    readonly statusText: string;
+    readonly lastModified: Date | null;
+    readonly contentType: string | null;
+    readonly failureReason: string | null;
+    readonly warnings: string[];
+    readonly responseTimeMs: number;
+    constructor(url: string, response: Response, responseTimeMs: number, failureReason?: string | null);
+    isSuccess(): boolean;
+}
+/**
+ * Result of probing a SPARQL endpoint.
+ */
+export declare class SparqlProbeResult extends ProbeResult {
+    readonly acceptedContentType: string;
+    constructor(url: string, response: Response, responseTimeMs: number, acceptedContentType: string, failureReason?: string | null);
+    isSuccess(): boolean;
+}
+/**
+ * Result of probing a data dump distribution.
+ */
+export declare class DataDumpProbeResult extends ProbeResult {
+    readonly contentSize: number | null;
+    constructor(url: string, response: Response, responseTimeMs: number, failureReason?: string | null);
+}
+export type ProbeResultType = SparqlProbeResult | DataDumpProbeResult | NetworkError;
+/**
+ * Probe a distribution to check availability and gather metadata.
+ *
+ * For SPARQL endpoints, issues the configured SPARQL query (default: a
+ * minimal `SELECT`). For data dumps, issues `HEAD` (with a `GET` fallback
+ * for small or unknown-size bodies).
+ *
+ * Returns a pure result object; never throws.
+ *
+ * @param distribution The distribution to probe.
+ * @param options Probe options. For back-compat, passing a `number` is
+ *   equivalent to `{ timeoutMs: number }` and is deprecated.
+ */
+export declare function probe(distribution: Distribution, options?: number | ProbeOptions): Promise<ProbeResultType>;
+export {};
+//# sourceMappingURL=probe.d.ts.map

package/dist/probe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../src/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAKD;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;aACf,cAAc,EAAE,MAAM;gBAFtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM;CAEzC;AAED;;GAEG;AACH,uBAAe,WAAW;aAUN,GAAG,EAAE,MAAM;IAT7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,SAAgB,QAAQ,EAAE,MAAM,EAAE,CAAM;IACxC,SAAgB,cAAc,EAAE,MAAM,CAAC;gBAGrB,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;IAa9B,SAAS,IAAI,OAAO;CAO5B;AAKD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,SAAgB,mBAAmB,EAAE,MAAM,CAAC;gBAG1C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,mBAAmB,EAAE,MAAM,EAC3B,aAAa,GAAE,MAAM,GAAG,IAAW;IAM5B,SAAS,IAAI,OAAO;CAM9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAGhD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;CAQtC;AAED,MAAM,MAAM,eAAe,GACvB,iBAAiB,GACjB,mBAAmB,GACnB,YAAY,CAAC;AAIjB;;;;;;;;;;;;GAYG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,MAAM,GAAG,YAAY,GAC9B,OAAO,CAAC,eAAe,CAAC,CAkC1B"}

package/dist/probe.js

@@ -0,0 +1,296 @@
+import { Parser } from 'n3';
+const DEFAULT_SPARQL_QUERY = 'SELECT * { ?s ?p ?o } LIMIT 1';
+const DEFAULT_TIMEOUT_MS = 5000;
+/**
+ * Result of a network error during probing.
+ */
+export class NetworkError {
+    url;
+    message;
+    responseTimeMs;
+    constructor(url, message, responseTimeMs) {
+        this.url = url;
+        this.message = message;
+        this.responseTimeMs = responseTimeMs;
+    }
+}
+/**
+ * Base class for successful probe results.
+ */
+class ProbeResult {
+    url;
+    statusCode;
+    statusText;
+    lastModified = null;
+    contentType;
+    failureReason;
+    warnings = [];
+    responseTimeMs;
+    constructor(url, response, responseTimeMs, failureReason = null) {
+        this.url = url;
+        this.statusCode = response.status;
+        this.statusText = response.statusText;
+        this.contentType = response.headers.get('Content-Type');
+        this.failureReason = failureReason;
+        this.responseTimeMs = responseTimeMs;
+        const lastModifiedHeader = response.headers.get('Last-Modified');
+        if (lastModifiedHeader) {
+            this.lastModified = new Date(lastModifiedHeader);
+        }
+    }
+    isSuccess() {
+        return (this.statusCode >= 200 &&
+            this.statusCode < 400 &&
+            this.failureReason === null);
+    }
+}
+const SPARQL_RESULTS_JSON = 'application/sparql-results+json';
+const SPARQL_RDF_RESULTS = 'application/n-triples';
+/**
+ * Result of probing a SPARQL endpoint.
+ */
+export class SparqlProbeResult extends ProbeResult {
+    acceptedContentType;
+    constructor(url, response, responseTimeMs, acceptedContentType, failureReason = null) {
+        super(url, response, responseTimeMs, failureReason);
+        this.acceptedContentType = acceptedContentType;
+    }
+    isSuccess() {
+        return (super.isSuccess() &&
+            (this.contentType?.startsWith(this.acceptedContentType) ?? false));
+    }
+}
+/**
+ * Result of probing a data dump distribution.
+ */
+export class DataDumpProbeResult extends ProbeResult {
+    contentSize = null;
+    constructor(url, response, responseTimeMs, failureReason = null) {
+        super(url, response, responseTimeMs, failureReason);
+        const contentLengthHeader = response.headers.get('Content-Length');
+        if (contentLengthHeader) {
+            this.contentSize = parseInt(contentLengthHeader);
+        }
+    }
+}
+/**
+ * Probe a distribution to check availability and gather metadata.
+ *
+ * For SPARQL endpoints, issues the configured SPARQL query (default: a
+ * minimal `SELECT`). For data dumps, issues `HEAD` (with a `GET` fallback
+ * for small or unknown-size bodies).
+ *
+ * Returns a pure result object; never throws.
+ *
+ * @param distribution The distribution to probe.
+ * @param options Probe options. For back-compat, passing a `number` is
+ *   equivalent to `{ timeoutMs: number }` and is deprecated.
+ */
+export async function probe(distribution, options) {
+    const resolved = resolveOptions(options);
+    const url = distribution.accessUrl?.toString() ?? 'unknown';
+    const [authUrl, authHeaders] = distribution.accessUrl !== undefined
+        ? extractUrlCredentials(distribution.accessUrl, resolved.headers)
+        : [new URL(url), new Headers(resolved.headers)];
+    const start = performance.now();
+    try {
+        if (distribution.isSparql()) {
+            return await probeSparqlEndpoint(authUrl.toString(), distribution, resolved, authHeaders, start);
+        }
+        return await probeDataDump(authUrl.toString(), distribution, resolved, authHeaders, start);
+    }
+    catch (e) {
+        const responseTimeMs = Math.round(performance.now() - start);
+        return new NetworkError(url, e instanceof Error ? e.message : String(e), responseTimeMs);
+    }
+}
+function resolveOptions(options) {
+    if (typeof options === 'number') {
+        return {
+            timeoutMs: options,
+            headers: new Headers(),
+            sparqlQuery: DEFAULT_SPARQL_QUERY,
+        };
+    }
+    return {
+        timeoutMs: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+        headers: options?.headers ?? new Headers(),
+        sparqlQuery: options?.sparqlQuery ?? DEFAULT_SPARQL_QUERY,
+    };
+}
+/**
+ * Strip `user:pass@` from a URL and turn it into an `Authorization: Basic`
+ * header. Returns the cleaned URL and a merged Headers object that preserves
+ * any caller-supplied headers.
+ */
+function extractUrlCredentials(url, baseHeaders) {
+    const headers = new Headers(baseHeaders);
+    if (url.username === '' && url.password === '') {
+        return [url, headers];
+    }
+    const credentials = `${decodeURIComponent(url.username)}:${decodeURIComponent(url.password)}`;
+    if (!headers.has('Authorization')) {
+        headers.set('Authorization', `Basic ${Buffer.from(credentials).toString('base64')}`);
+    }
+    const cleanUrl = new URL(url.toString());
+    cleanUrl.username = '';
+    cleanUrl.password = '';
+    return [cleanUrl, headers];
+}
+/**
+ * Classify a SPARQL query. Comments are stripped; the first keyword match
+ * wins. Falls back to `SELECT` when no keyword is found – robust enough for
+ * availability probing but not a full SPARQL parser.
+ */
+function detectSparqlQueryType(query) {
+    const withoutComments = query.replace(/#[^\n\r]*/g, ' ');
+    const match = /\b(ASK|SELECT|CONSTRUCT|DESCRIBE)\b/i.exec(withoutComments);
+    return (match?.[1].toUpperCase() ?? 'SELECT');
+}
+function acceptHeaderForQueryType(queryType) {
+    if (queryType === 'ASK' || queryType === 'SELECT') {
+        return SPARQL_RESULTS_JSON;
+    }
+    return SPARQL_RDF_RESULTS;
+}
+async function probeSparqlEndpoint(url, _distribution, options, authHeaders, start) {
+    const queryType = detectSparqlQueryType(options.sparqlQuery);
+    const accept = acceptHeaderForQueryType(queryType);
+    const headers = new Headers({
+        'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
+        Accept: accept,
+    });
+    for (const [key, value] of authHeaders) {
+        headers.set(key, value);
+    }
+    const response = await fetch(url, {
+        signal: AbortSignal.timeout(options.timeoutMs),
+        method: 'POST',
+        headers,
+        body: `query=${encodeURIComponent(options.sparqlQuery)}`,
+    });
+    const actualContentType = response.headers.get('Content-Type');
+    const contentTypeMatches = actualContentType?.startsWith(accept) ?? false;
+    let failureReason = null;
+    if (response.ok && contentTypeMatches) {
+        failureReason = await validateSparqlResponse(response, queryType);
+    }
+    else {
+        // Drain unconsumed body to release the underlying connection.
+        await response.body?.cancel();
+    }
+    const responseTimeMs = Math.round(performance.now() - start);
+    return new SparqlProbeResult(url, response, responseTimeMs, accept, failureReason);
+}
+async function validateSparqlResponse(response, queryType) {
+    const body = await response.text();
+    if (body.length === 0) {
+        return 'SPARQL endpoint returned an empty response';
+    }
+    if (queryType === 'CONSTRUCT' || queryType === 'DESCRIBE') {
+        // Body should be RDF; a non-empty response is sufficient to confirm the
+        // endpoint answered. Deep parse validation is the data-dump path’s job.
+        return null;
+    }
+    let json;
+    try {
+        json = JSON.parse(body);
+    }
+    catch {
+        return 'SPARQL endpoint returned invalid JSON';
+    }
+    if (queryType === 'ASK') {
+        if (typeof json.boolean !== 'boolean') {
+            return 'SPARQL endpoint did not return a valid ASK result';
+        }
+        return null;
+    }
+    // SELECT
+    if (!json.results || typeof json.results !== 'object') {
+        return 'SPARQL endpoint did not return a valid results object';
+    }
+    return null;
+}
+async function probeDataDump(url, distribution, options, authHeaders, start) {
+    const headers = new Headers({
+        Accept: distribution.mimeType ?? '*/*',
+        'Accept-Encoding': 'identity',
+    });
+    for (const [key, value] of authHeaders) {
+        headers.set(key, value);
+    }
+    const requestOptions = {
+        signal: AbortSignal.timeout(options.timeoutMs),
+        headers,
+    };
+    const headResponse = await fetch(url, {
+        method: 'HEAD',
+        ...requestOptions,
+    });
+    const contentLength = headResponse.headers.get('Content-Length');
+    const contentLengthBytes = contentLength ? parseInt(contentLength) : 0;
+    // For small or unknown-size files, do a GET to validate body content.
+    // This also handles servers that incorrectly return 0 Content-Length for HEAD.
+    if (contentLengthBytes <= 10_240) {
+        const getResponse = await fetch(url, {
+            method: 'GET',
+            ...requestOptions,
+        });
+        const body = await getResponse.text();
+        const isHttpSuccess = getResponse.status >= 200 && getResponse.status < 400;
+        const failureReason = isHttpSuccess
+            ? validateBody(body, getResponse.headers.get('Content-Type'))
+            : null;
+        const responseTimeMs = Math.round(performance.now() - start);
+        const result = new DataDumpProbeResult(url, getResponse, responseTimeMs, failureReason);
+        checkContentTypeMismatch(result, distribution.mimeType);
+        return result;
+    }
+    const responseTimeMs = Math.round(performance.now() - start);
+    const result = new DataDumpProbeResult(url, headResponse, responseTimeMs);
+    checkContentTypeMismatch(result, distribution.mimeType);
+    return result;
+}
+const rdfContentTypes = [
+    'text/turtle',
+    'application/n-triples',
+    'application/n-quads',
+];
+function validateBody(body, contentType) {
+    if (body.length === 0) {
+        return 'Distribution is empty';
+    }
+    if (contentType && rdfContentTypes.some((t) => contentType.startsWith(t))) {
+        try {
+            const parser = new Parser();
+            const quads = parser.parse(body);
+            if (quads.length === 0) {
+                return 'Distribution contains no RDF triples';
+            }
+        }
+        catch (e) {
+            return e instanceof Error ? e.message : String(e);
+        }
+    }
+    return null;
+}
+/** Content types that indicate compression, not the RDF serialization format. */
+const compressionTypes = new Set([
+    'application/gzip',
+    'application/x-gzip',
+    'application/octet-stream',
+]);
+/**
+ * Compare the declared MIME type from the dataset registry against the
+ * server's Content-Type header. Adds a warning when they disagree.
+ */
+function checkContentTypeMismatch(result, declaredMimeType) {
+    if (!result.isSuccess() || !declaredMimeType || !result.contentType)
+        return;
+    const actual = result.contentType.split(';')[0].trim();
+    if (compressionTypes.has(actual))
+        return;
+    if (actual !== declaredMimeType) {
+        result.warnings.push(`Server Content-Type ${actual} does not match declared media type ${declaredMimeType}`);
+    }
+}

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@lde/distribution-probe",
+  "version": "0.1.0",
+  "repository": {
+    "url": "git+https://github.com/ldelements/lde.git",
+    "directory": "packages/distribution-probe"
+  },
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "development": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo"
+  ],
+  "dependencies": {
+    "@lde/dataset": "0.7.2",
+    "n3": "^2.0.1",
+    "tslib": "^2.3.0"
+  }
+}