@lde/distribution-probe 0.4.0 → 0.5.0

package/dist/probe.d.ts

@@ -143,7 +143,9 @@ export type ProbeResultType = SparqlProbeResult | DataDumpProbeResult | NetworkE
  * For SPARQL endpoints, issues the configured SPARQL query (default: a
  * minimal `SELECT`). For data dumps, issues `HEAD` (with a `GET` fallback
  * for small or unknown-size bodies, reading only a bounded prefix so a large
- * streamed dump is never downloaded in full).
+ * streamed dump is never downloaded in full). A
+ * {@link Distribution.localFile | local file} is probed the same way, answered
+ * from the filesystem; a missing one is a {@link NetworkError}.
  *
  * Returns a pure result object; never throws.
  */

package/dist/probe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../src/probe.ts"],"names":[],"mappings":"AAKA,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AASnE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;CAC5D;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACzD;AAkCD;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;aACf,cAAc,EAAE,MAAM;gBAFtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM;CAEzC;AAED;;GAEG;AACH,uBAAe,WAAW;aAUN,GAAG,EAAE,MAAM;IAT7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,SAAgB,QAAQ,EAAE,MAAM,EAAE,CAAM;IACxC,SAAgB,cAAc,EAAE,MAAM,CAAC;gBAGrB,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;IAa9B,SAAS,IAAI,OAAO;CAO5B;AAqBD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD;;;;;OAKG;IACH,SAAgB,oBAAoB,EAAE,SAAS,MAAM,EAAE,CAAC;gBAGtD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,oBAAoB,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,EAChD,aAAa,GAAE,MAAM,GAAG,IAAW;IAS5B,SAAS,IAAI,OAAO;CAQ9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAGhD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;CAQtC;AAED,MAAM,MAAM,eAAe,GACzB,iBAAiB,GAAG,mBAAmB,GAAG,YAAY,CAAC;AAIzD;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,eAAe,CAAC,CAqD1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,SAAS,CAC7B,aAAa,EAAE,SAAS,YAAY,EAAE,EACtC,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,EAAE,CAAC,CAsC5B"}
+{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../src/probe.ts"],"names":[],"mappings":"AAKA,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AAWnE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;CAC5D;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACzD;AAkCD;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;aACf,cAAc,EAAE,MAAM;gBAFtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM;CAEzC;AAED;;GAEG;AACH,uBAAe,WAAW;aAUN,GAAG,EAAE,MAAM;IAT7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,SAAgB,QAAQ,EAAE,MAAM,EAAE,CAAM;IACxC,SAAgB,cAAc,EAAE,MAAM,CAAC;gBAGrB,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;IAa9B,SAAS,IAAI,OAAO;CAO5B;AAqBD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD;;;;;OAKG;IACH,SAAgB,oBAAoB,EAAE,SAAS,MAAM,EAAE,CAAC;gBAGtD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,oBAAoB,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,EAChD,aAAa,GAAE,MAAM,GAAG,IAAW;IAS5B,SAAS,IAAI,OAAO;CAQ9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAGhD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;CAQtC;AAED,MAAM,MAAM,eAAe,GACzB,iBAAiB,GAAG,mBAAmB,GAAG,YAAY,CAAC;AAIzD;;;;;;;;;;;GAWG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,eAAe,CAAC,CAqD1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,SAAS,CAC7B,aAAa,EAAE,SAAS,YAAY,EAAE,EACtC,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,EAAE,CAAC,CAsC5B"}

package/dist/probe.js

@@ -2,6 +2,8 @@ import { hostKeyOf, mapHostLimited, positiveIntOrDefault, } from '@lde/host-limi
 import { compressionMediaTypes } from '@lde/dataset';
 import { rdfParser } from 'rdf-parse';
 import { FetchDocumentLoader, } from 'jsonld-context-parser';
+import { constants, createReadStream } from 'node:fs';
+import { access, stat } from 'node:fs/promises';
 import { Readable } from 'node:stream';
 import { createGunzip } from 'node:zlib';
 const DEFAULT_SPARQL_QUERY = 'SELECT * { ?s ?p ?o } LIMIT 1';
@@ -133,7 +135,9 @@ export class DataDumpProbeResult extends ProbeResult {
  * For SPARQL endpoints, issues the configured SPARQL query (default: a
  * minimal `SELECT`). For data dumps, issues `HEAD` (with a `GET` fallback
  * for small or unknown-size bodies, reading only a bounded prefix so a large
- * streamed dump is never downloaded in full).
+ * streamed dump is never downloaded in full). A
+ * {@link Distribution.localFile | local file} is probed the same way, answered
+ * from the filesystem; a missing one is a {@link NetworkError}.
  *
  * Returns a pure result object; never throws.
  */
@@ -414,7 +418,8 @@ async function probeDataDump(url, distribution, options, authHeaders, start) {
         signal: AbortSignal.timeout(options.timeoutMs),
         headers,
     };
-    const headResponse = await fetch(url, {
+    const fetchDump = dumpFetcher(distribution);
+    const headResponse = await fetchDump(url, {
         method: 'HEAD',
         ...requestOptions,
     });
@@ -425,7 +430,7 @@ async function probeDataDump(url, distribution, options, authHeaders, start) {
     if (options.validateRdfContent &&
         isDeclaredRdf(distribution) &&
         isHttpSuccess(headResponse)) {
-        const { response, failureReason } = await validateDumpBody(url, headers, options, headResponse);
+        const { response, failureReason } = await validateDumpBody(url, headers, options, headResponse, fetchDump);
         return finalizeDataDump(url, distribution, response, start, failureReason);
     }
     // Reachability only. A successful HEAD is enough; otherwise confirm with a
@@ -433,10 +438,48 @@ async function probeDataDump(url, distribution, options, authHeaders, start) {
     if (isHttpSuccess(headResponse)) {
         return finalizeDataDump(url, distribution, headResponse, start, null);
     }
-    const getResponse = await fetch(url, { method: 'GET', ...requestOptions });
+    const getResponse = await fetchDump(url, {
+        method: 'GET',
+        ...requestOptions,
+    });
     await getResponse.body?.cancel();
     return finalizeDataDump(url, distribution, getResponse, start, null);
 }
+/**
+ * How to request a data dump: over HTTP, or for a local file from the filesystem
+ * in the shape of the HTTP response a static file server would give, so that
+ * reachability, content validation and the change fingerprint need no second
+ * path.
+ */
+function dumpFetcher(distribution) {
+    const { localPath } = distribution;
+    if (localPath === undefined) {
+        return fetch;
+    }
+    return async (_url, init) => {
+        const stats = await stat(localPath);
+        if (!stats.isFile()) {
+            throw new Error(`${localPath} is not a file`);
+        }
+        // An unreadable file would otherwise pass here and fail only at import,
+        // recorded against a fingerprint that a later chmod does not change.
+        await access(localPath, constants.R_OK);
+        return new Response(init.method === 'HEAD'
+            ? null
+            : Readable.toWeb(createReadStream(localPath)), {
+            headers: {
+                'Content-Length': stats.size.toString(),
+                'Last-Modified': stats.mtime.toUTCString(),
+                // The RDF type for gzip too: the body is inflated by its magic bytes
+                // before validation. A zip keeps its own type, as it holds entries.
+                // Distribution.localFile always derives a media type.
+                'Content-Type': (distribution.compressMimeType === 'application/zip'
+                    ? distribution.compressMimeType
+                    : distribution.mimeType),
+            },
+        });
+    };
+}
 /** Whether an HTTP response carries a success (2xx/3xx) status. */
 function isHttpSuccess(response) {
     return response.status >= 200 && response.status < 400;
@@ -461,14 +504,14 @@ function finalizeDataDump(url, distribution, response, start, failureReason) {
  * unvalidated), never a failure. Returns the response to draw metadata from
  * (the GET, or the HEAD when the GET could not start) alongside that reason.
  */
-async function validateDumpBody(url, headers, options, headResponse) {
+async function validateDumpBody(url, headers, options, headResponse, fetchDump) {
     const budgetMs = Math.min(options.rdfValidationBudgetMs, options.timeoutMs);
     // Aborting on budget expiry stops a slow endpoint from streaming on in the
     // background once we have given up waiting for a triple.
     const budgetController = new AbortController();
     let getResponse;
     try {
-        getResponse = await fetch(url, {
+        getResponse = await fetchDump(url, {
             method: 'GET',
             headers,
             signal: AbortSignal.any([

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/distribution-probe",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/distribution-probe"
@@ -24,8 +24,8 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "^0.9.0",
-    "@lde/host-limiter": "^0.2.0",
+    "@lde/dataset": "^0.10.0",
+    "@lde/host-limiter": "^0.2.1",
     "jsonld-context-parser": "^2.4.0",
     "rdf-parse": "^5.0.0",
     "tslib": "^2.3.0"