@lde/distribution-probe 0.2.0 → 0.2.2

package/README.md

@@ -48,3 +48,24 @@ When on, the probe `GET`s the dump — **regardless of size** — and reads only
 A thrown exception from `fetch` (DNS failure, connection refused, socket reset, TLS error, timeout after the configured `timeoutMs` – default 5 000 ms) is a connection-level failure. The probe retries these up to `retries` times (default 2) with a short backoff before giving up and returning a `NetworkError`. This turns a transient transport blip into a reliable single measurement without looking backward across checks. A genuine outage still resolves to a `NetworkError` on the current check – every attempt fails – but note each attempt gets its own `timeoutMs`, so an endpoint that fails only by timing out takes up to `(retries + 1) × timeoutMs` (plus backoff) to be reported down. HTTP error responses (4xx/5xx) and content-validation failures are real ‘down’ states and are **never** retried.
 
 `NetworkError.message` includes the underlying `error.cause` (e.g. `ECONNRESET`, `UND_ERR_SOCKET “other side closed”`) when Node wraps one, so observations record what actually failed rather than a bare ‘fetch failed’.
+
+## Probing many distributions
+
+`probeMany` probes an array of distributions concurrently and returns one result per input, in input order. Each distribution is probed once with `probe`, so every behaviour above applies per distribution; like `probe`, `probeMany` never throws – a probe that fails is reported as a `NetworkError` in its slot.
+
+```ts
+import { probeMany } from '@lde/distribution-probe';
+
+const results = await probeMany(distributions, {
+  concurrency: 20, // max probes in flight across all hosts (default 20)
+  perHostConcurrency: 4, // max probes in flight against one host (default 4)
+  validateRdfContent: true, // any ProbeOptions are forwarded to each probe
+});
+```
+
+Two caps bound the batch:
+
+- **`concurrency`** bounds the total fan-out, so a large catalogue does not exhaust sockets or buffer too many response bodies at once.
+- **`perHostConcurrency`** bounds the burst any one server sees, keeping the batch a polite client: a catalogue that declares many distributions on a single host (e.g. a download endpoint per named graph) will not trip that server’s rate limiter (HTTP 429). Distributions sharing a host (by `accessUrl`) contend for the same budget; a probe whose host is saturated waits while probes for other hosts proceed, so one busy host never idles the global pool.
+
+All other `ProbeOptions` (`timeoutMs`, `retries`, `validateRdfContent`, and the rest) are forwarded unchanged to every probe.

package/dist/index.d.ts

@@ -1,2 +1,2 @@
-export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, type ProbeOptions, type ProbeResultType, } from './probe.js';
+export { probe, probeMany, NetworkError, SparqlProbeResult, DataDumpProbeResult, type ProbeOptions, type ProbeManyOptions, type ProbeResultType, } from './probe.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,YAAY,EACjB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -1 +1 @@
-export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';
+export { probe, probeMany, NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';

package/dist/probe.d.ts

@@ -52,6 +52,35 @@ export interface ProbeOptions {
      */
     rdfValidationBudgetMs?: number;
 }
+/**
+ * Options for {@link probeMany}: the per-probe {@link ProbeOptions} plus the
+ * concurrency budgets that bound the batch.
+ */
+export interface ProbeManyOptions extends ProbeOptions {
+    /**
+     * Maximum number of probes to run at once across all hosts. Bounds the batch’s
+     * total fan-out so a large catalogue does not exhaust sockets or buffer too many
+     * response bodies at once. Default 20.
+     */
+    concurrency?: number;
+    /**
+     * Maximum number of probes to run at once against a single host. Bounds the
+     * burst any one server sees, so a catalogue that declares many distributions on
+     * one host (e.g. a download endpoint per named graph) does not trip its rate
+     * limiter (HTTP 429). A probe whose host is at this cap waits while probes for
+     * other hosts proceed, so this never idles the global pool. Default 4.
+     */
+    perHostConcurrency?: number;
+    /**
+     * Called once after each probe settles, with the number of probes completed so
+     * far and the total to run (`distributions.length`). Lets a caller drive a
+     * determinate progress indicator while a large batch runs. Fires `total` times,
+     * ending at `(total, total)`; the order reflects completion, not input order.
+     * Never called for an empty batch. A throwing callback rejects the batch, so
+     * keep it cheap and side-effect-only.
+     */
+    onProgress?: (completed: number, total: number) => void;
+}
 /**
  * Result of a network error during probing.
  */
@@ -109,5 +138,19 @@ export type ProbeResultType = SparqlProbeResult | DataDumpProbeResult | NetworkE
  * Returns a pure result object; never throws.
  */
 export declare function probe(distribution: Distribution, options?: ProbeOptions): Promise<ProbeResultType>;
+/**
+ * Probe many distributions concurrently, bounded by a global cap and a per-host
+ * cap, returning one result per input in input order. Like {@link probe}, this
+ * never throws: a probe that somehow fails is reported as a {@link NetworkError}
+ * in its slot.
+ *
+ * The per-host cap keeps the batch a polite client. Distributions sharing a host
+ * (by {@link Distribution.accessUrl}) contend for the same budget, so no single
+ * server is hit by the full global pool at once — the burst that trips a rate
+ * limiter (HTTP 429). When the next queued probe’s host is saturated it is
+ * skipped in favour of a later probe on a different host, so one busy host never
+ * idles the global pool (no head-of-line blocking).
+ */
+export declare function probeMany(distributions: readonly Distribution[], options?: ProbeManyOptions): Promise<ProbeResultType[]>;
 export {};
 //# sourceMappingURL=probe.d.ts.map

package/dist/probe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../src/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AAKnE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAgCD;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;aACf,cAAc,EAAE,MAAM;gBAFtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM;CAEzC;AAED;;GAEG;AACH,uBAAe,WAAW;aAUN,GAAG,EAAE,MAAM;IAT7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,SAAgB,QAAQ,EAAE,MAAM,EAAE,CAAM;IACxC,SAAgB,cAAc,EAAE,MAAM,CAAC;gBAGrB,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;IAa9B,SAAS,IAAI,OAAO;CAO5B;AAqBD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD;;;;;OAKG;IACH,SAAgB,oBAAoB,EAAE,SAAS,MAAM,EAAE,CAAC;gBAGtD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,oBAAoB,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,EAChD,aAAa,GAAE,MAAM,GAAG,IAAW;IAS5B,SAAS,IAAI,OAAO;CAQ9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAGhD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;CAQtC;AAED,MAAM,MAAM,eAAe,GACvB,iBAAiB,GACjB,mBAAmB,GACnB,YAAY,CAAC;AAIjB;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,eAAe,CAAC,CAqD1B"}
+{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../src/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AAKnE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACzD;AAkCD;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;aACf,cAAc,EAAE,MAAM;gBAFtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM;CAEzC;AAED;;GAEG;AACH,uBAAe,WAAW;aAUN,GAAG,EAAE,MAAM;IAT7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,SAAgB,QAAQ,EAAE,MAAM,EAAE,CAAM;IACxC,SAAgB,cAAc,EAAE,MAAM,CAAC;gBAGrB,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;IAa9B,SAAS,IAAI,OAAO;CAO5B;AAqBD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD;;;;;OAKG;IACH,SAAgB,oBAAoB,EAAE,SAAS,MAAM,EAAE,CAAC;gBAGtD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,oBAAoB,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,EAChD,aAAa,GAAE,MAAM,GAAG,IAAW;IAS5B,SAAS,IAAI,OAAO;CAQ9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAGhD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;CAQtC;AAED,MAAM,MAAM,eAAe,GACvB,iBAAiB,GACjB,mBAAmB,GACnB,YAAY,CAAC;AAIjB;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,eAAe,CAAC,CAqD1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,SAAS,CAC7B,aAAa,EAAE,SAAS,YAAY,EAAE,EACtC,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,EAAE,CAAC,CAuC5B"}

package/dist/probe.js

@@ -5,6 +5,8 @@ import { createGunzip } from 'node:zlib';
 const DEFAULT_SPARQL_QUERY = 'SELECT * { ?s ?p ?o } LIMIT 1';
 const DEFAULT_TIMEOUT_MS = 5000;
 const DEFAULT_RETRIES = 2;
+const DEFAULT_PROBE_CONCURRENCY = 20;
+const DEFAULT_PROBE_PER_HOST_CONCURRENCY = 4;
 /**
  * Default soft deadline for finding the first triple when content validation is
  * on (capped at `timeoutMs`). Two seconds comfortably covers a static file
@@ -169,6 +171,107 @@ export async function probe(distribution, options) {
     // real cost of a down endpoint.
     return new NetworkError(url, describeNetworkError(lastError), Math.round(performance.now() - overallStart));
 }
+/**
+ * Probe many distributions concurrently, bounded by a global cap and a per-host
+ * cap, returning one result per input in input order. Like {@link probe}, this
+ * never throws: a probe that somehow fails is reported as a {@link NetworkError}
+ * in its slot.
+ *
+ * The per-host cap keeps the batch a polite client. Distributions sharing a host
+ * (by {@link Distribution.accessUrl}) contend for the same budget, so no single
+ * server is hit by the full global pool at once — the burst that trips a rate
+ * limiter (HTTP 429). When the next queued probe’s host is saturated it is
+ * skipped in favour of a later probe on a different host, so one busy host never
+ * idles the global pool (no head-of-line blocking).
+ */
+export async function probeMany(distributions, options) {
+    // Clamp the budgets to a positive integer, mirroring how probe() treats an
+    // invalid retries value: a zero, negative, fractional, or NaN limit would
+    // otherwise stall the scheduler (no task ever starts, so the promise never
+    // resolves) or overrun the cap, so fall back to the default rather than trust
+    // the caller.
+    const globalLimit = positiveIntOrDefault(options?.concurrency, DEFAULT_PROBE_CONCURRENCY);
+    const perHostLimit = positiveIntOrDefault(options?.perHostConcurrency, DEFAULT_PROBE_PER_HOST_CONCURRENCY);
+    // Probes contend per host. An authority-less URL (e.g. urn:, file:) has an
+    // empty host, so it falls back to its full href and never shares a budget with
+    // an unrelated one.
+    const hostKeys = distributions.map((distribution) => distribution.accessUrl.host || distribution.accessUrl.href);
+    // Report progress as each probe settles. mapHostLimited resolves results in
+    // input order, but tasks complete out of order, so count completions here
+    // rather than rely on result position. The total is the batch size.
+    const onProgress = options?.onProgress;
+    const total = distributions.length;
+    let completed = 0;
+    return mapHostLimited(distributions, hostKeys, globalLimit, perHostLimit, async (distribution) => {
+        const result = await probe(distribution, options);
+        completed += 1;
+        onProgress?.(completed, total);
+        return result;
+    });
+}
+/**
+ * Coerce an optional concurrency budget to a usable value: a positive integer is
+ * taken as-is; undefined, zero, negative, fractional, or NaN falls back to the
+ * default. Matches probe()’s treatment of an invalid retries value.
+ */
+function positiveIntOrDefault(value, fallback) {
+    return value !== undefined && Number.isInteger(value) && value >= 1
+        ? value
+        : fallback;
+}
+/**
+ * Run `task` over `items` with two concurrency caps — a global cap and a per-host
+ * cap keyed by `hostKeys[index]` — resolving to results in input order. When the
+ * next queued item’s host is at the per-host cap it is skipped for a later item on
+ * a different host, so a saturated host never idles the global pool (no head-of-line
+ * blocking); the skipped host always has a task in flight, whose completion re-runs
+ * the scheduler, so the queue always drains. `task` must not reject — callers wrap
+ * failures into a result value — as a rejection would leave the promise pending.
+ */
+function mapHostLimited(items, hostKeys, globalLimit, perHostLimit, task) {
+    const results = new Array(items.length);
+    const perHostInFlight = new Map();
+    const pending = items.map((_unused, index) => index);
+    let globalInFlight = 0;
+    let settledCount = 0;
+    const adjustHost = (host, delta) => {
+        perHostInFlight.set(host, (perHostInFlight.get(host) ?? 0) + delta);
+    };
+    return new Promise((resolve) => {
+        const schedule = () => {
+            let cursor = 0;
+            while (cursor < pending.length && globalInFlight < globalLimit) {
+                const index = pending[cursor];
+                const host = hostKeys[index];
+                if ((perHostInFlight.get(host) ?? 0) >= perHostLimit) {
+                    cursor++; // Host saturated; leave it queued and try a later, different host.
+                    continue;
+                }
+                pending.splice(cursor, 1);
+                globalInFlight++;
+                adjustHost(host, 1);
+                void task(items[index]).then((result) => {
+                    results[index] = result;
+                    globalInFlight--;
+                    adjustHost(host, -1);
+                    settledCount++;
+                    if (settledCount === items.length) {
+                        resolve(results);
+                    }
+                    else {
+                        schedule();
+                    }
+                });
+                // pending[cursor] now holds the next queued item; do not advance cursor.
+            }
+        };
+        schedule();
+        // Resolve immediately when there is nothing to settle (empty input); a
+        // non-empty run resolves via the task completion above.
+        if (settledCount === items.length)
+            resolve(results);
+    });
+}
 function delay(milliseconds) {
     return new Promise((resolve) => setTimeout(resolve, milliseconds));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/distribution-probe",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/distribution-probe"