@lde/distribution-probe 0.2.8 → 0.2.10

package/dist/probe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../src/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AAKnE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACzD;AAkCD;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;aACf,cAAc,EAAE,MAAM;gBAFtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM;CAEzC;AAED;;GAEG;AACH,uBAAe,WAAW;aAUN,GAAG,EAAE,MAAM;IAT7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,SAAgB,QAAQ,EAAE,MAAM,EAAE,CAAM;IACxC,SAAgB,cAAc,EAAE,MAAM,CAAC;gBAGrB,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;IAa9B,SAAS,IAAI,OAAO;CAO5B;AAqBD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD;;;;;OAKG;IACH,SAAgB,oBAAoB,EAAE,SAAS,MAAM,EAAE,CAAC;gBAGtD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,oBAAoB,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,EAChD,aAAa,GAAE,MAAM,GAAG,IAAW;IAS5B,SAAS,IAAI,OAAO;CAQ9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAGhD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;CAQtC;AAED,MAAM,MAAM,eAAe,GACvB,iBAAiB,GACjB,mBAAmB,GACnB,YAAY,CAAC;AAIjB;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,eAAe,CAAC,CAqD1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,SAAS,CAC7B,aAAa,EAAE,SAAS,YAAY,EAAE,EACtC,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,EAAE,CAAC,CAuC5B"}
+{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../src/probe.ts"],"names":[],"mappings":"AAKA,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AAKnE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;OAUG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACzD;AAkCD;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;aACf,cAAc,EAAE,MAAM;gBAFtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM;CAEzC;AAED;;GAEG;AACH,uBAAe,WAAW;aAUN,GAAG,EAAE,MAAM;IAT7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,SAAgB,QAAQ,EAAE,MAAM,EAAE,CAAM;IACxC,SAAgB,cAAc,EAAE,MAAM,CAAC;gBAGrB,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;IAa9B,SAAS,IAAI,OAAO;CAO5B;AAqBD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD;;;;;OAKG;IACH,SAAgB,oBAAoB,EAAE,SAAS,MAAM,EAAE,CAAC;gBAGtD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,oBAAoB,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,EAChD,aAAa,GAAE,MAAM,GAAG,IAAW;IAS5B,SAAS,IAAI,OAAO;CAQ9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAGhD,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW;CAQtC;AAED,MAAM,MAAM,eAAe,GACzB,iBAAiB,GAAG,mBAAmB,GAAG,YAAY,CAAC;AAIzD;;;;;;;;;GASG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,eAAe,CAAC,CAqD1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,SAAS,CAC7B,aAAa,EAAE,SAAS,YAAY,EAAE,EACtC,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,EAAE,CAAC,CAsC5B"}

package/dist/probe.js

@@ -1,3 +1,4 @@
+import { hostKeyOf, mapHostLimited, positiveIntOrDefault, } from '@lde/host-limiter';
 import { compressionMediaTypes } from '@lde/dataset';
 import { rdfParser } from 'rdf-parse';
 import { Readable } from 'node:stream';
@@ -192,10 +193,10 @@ export async function probeMany(distributions, options) {
     // the caller.
     const globalLimit = positiveIntOrDefault(options?.concurrency, DEFAULT_PROBE_CONCURRENCY);
     const perHostLimit = positiveIntOrDefault(options?.perHostConcurrency, DEFAULT_PROBE_PER_HOST_CONCURRENCY);
-    // Probes contend per host. An authority-less URL (e.g. urn:, file:) has an
-    // empty host, so it falls back to its full href and never shares a budget with
-    // an unrelated one.
-    const hostKeys = distributions.map((distribution) => distribution.accessUrl.host || distribution.accessUrl.href);
+    // Probes contend per host, by the shared rule: an authority-less URL (e.g.
+    // urn:, file:) has an empty host, so it falls back to its full href and never
+    // shares a budget with an unrelated one.
+    const hostKeys = distributions.map((distribution) => hostKeyOf(distribution.accessUrl));
     // Report progress as each probe settles. mapHostLimited resolves results in
     // input order, but tasks complete out of order, so count completions here
     // rather than rely on result position. The total is the batch size.
@@ -209,69 +210,6 @@ export async function probeMany(distributions, options) {
         return result;
     });
 }
-/**
- * Coerce an optional concurrency budget to a usable value: a positive integer is
- * taken as-is; undefined, zero, negative, fractional, or NaN falls back to the
- * default. Matches probe()’s treatment of an invalid retries value.
- */
-function positiveIntOrDefault(value, fallback) {
-    return value !== undefined && Number.isInteger(value) && value >= 1
-        ? value
-        : fallback;
-}
-/**
- * Run `task` over `items` with two concurrency caps — a global cap and a per-host
- * cap keyed by `hostKeys[index]` — resolving to results in input order. When the
- * next queued item’s host is at the per-host cap it is skipped for a later item on
- * a different host, so a saturated host never idles the global pool (no head-of-line
- * blocking); the skipped host always has a task in flight, whose completion re-runs
- * the scheduler, so the queue always drains. `task` must not reject — callers wrap
- * failures into a result value — as a rejection would leave the promise pending.
- */
-function mapHostLimited(items, hostKeys, globalLimit, perHostLimit, task) {
-    const results = new Array(items.length);
-    const perHostInFlight = new Map();
-    const pending = items.map((_unused, index) => index);
-    let globalInFlight = 0;
-    let settledCount = 0;
-    const adjustHost = (host, delta) => {
-        perHostInFlight.set(host, (perHostInFlight.get(host) ?? 0) + delta);
-    };
-    return new Promise((resolve) => {
-        const schedule = () => {
-            let cursor = 0;
-            while (cursor < pending.length && globalInFlight < globalLimit) {
-                const index = pending[cursor];
-                const host = hostKeys[index];
-                if ((perHostInFlight.get(host) ?? 0) >= perHostLimit) {
-                    cursor++; // Host saturated; leave it queued and try a later, different host.
-                    continue;
-                }
-                pending.splice(cursor, 1);
-                globalInFlight++;
-                adjustHost(host, 1);
-                void task(items[index]).then((result) => {
-                    results[index] = result;
-                    globalInFlight--;
-                    adjustHost(host, -1);
-                    settledCount++;
-                    if (settledCount === items.length) {
-                        resolve(results);
-                    }
-                    else {
-                        schedule();
-                    }
-                });
-                // pending[cursor] now holds the next queued item; do not advance cursor.
-            }
-        };
-        schedule();
-        // Resolve immediately when there is nothing to settle (empty input); a
-        // non-empty run resolves via the task completion above.
-        if (settledCount === items.length)
-            resolve(results);
-    });
-}
 function delay(milliseconds) {
     return new Promise((resolve) => setTimeout(resolve, milliseconds));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/distribution-probe",
-  "version": "0.2.8",
+  "version": "0.2.10",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/distribution-probe"
@@ -24,7 +24,8 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "^0.7.9",
+    "@lde/dataset": "^0.7.10",
+    "@lde/host-limiter": "^0.0.2",
     "rdf-parse": "^5.0.0",
     "tslib": "^2.3.0"
   }