@thotischner/observability-mcp 3.8.2 → 3.8.3

package/dist/enrich/rdap.d.ts

@@ -6,7 +6,27 @@ export type FetchLike = (url: string, init?: {
     ok: boolean;
     status: number;
     json: () => Promise<unknown>;
+    /** Optional — used to honor `Retry-After` on a throttle response. */
+    headers?: {
+        get(name: string): string | null;
+    };
 }>;
+/** Why a lookup failed in a way that is NOT a stable property of the address —
+ *  retrying later (or in a smaller batch) may succeed. Issue #523. */
+export type RdapTransientReason = "rate_limited" | "timeout" | "upstream_error" | "network_error";
+/** Outcome of a single RDAP lookup. `not_found` is a genuine negative (the
+ *  address is not in any registry / carries no enrichment) and is safe to
+ *  cache; `transient` is an upstream failure (throttle, timeout, 5xx) that must
+ *  NOT be conflated with a negative and is never cached. */
+export type RdapOutcome = {
+    status: "ok";
+    value: IpEnrichment;
+} | {
+    status: "not_found";
+} | {
+    status: "transient";
+    reason: RdapTransientReason;
+};
 export interface RdapResolverOptions {
     /** Bootstrap base; rdap.org redirects to the authoritative RIR. */
     baseUrl?: string;
@@ -18,6 +38,14 @@ export interface RdapResolverOptions {
     fetch?: FetchLike;
     /** Max cache entries (LRU-ish trim). */
     maxCache?: number;
+    /** Retries on a TRANSIENT failure (throttle/timeout/5xx). Default 2. A true
+     *  negative (404 / no-data) is never retried. */
+    maxRetries?: number;
+    /** Base backoff in ms; attempt N waits min(base * 2^N, 5000), unless the
+     *  throttle response carries a usable `Retry-After`. Default 250. */
+    backoffMs?: number;
+    /** Injected sleep (tests pass a no-op to stay fast). Defaults to setTimeout. */
+    sleep?: (ms: number) => Promise<void>;
 }
 /** Parse an RDAP IP-network response into our enrichment shape. country +
  *  org/name only; RDAP carries no city or hosting flag. */
@@ -29,12 +57,23 @@ export declare class RdapResolver {
     private readonly timeoutMs;
     private readonly fetch;
     private readonly maxCache;
+    private readonly maxRetries;
+    private readonly backoffMs;
+    private readonly sleep;
     private cache;
     /** Monotonic clock injected for tests; defaults to Date.now via a getter. */
     now: () => number;
     constructor(opts?: RdapResolverOptions);
-    /** Look up one IP via RDAP. Returns null on miss/error (never throws —
-     *  a flaky RIR must not fail the batch). Cached by IP with a TTL. */
+    /** Look up one IP via RDAP. Returns the enrichment on a hit, or null on a
+     *  miss OR a transient failure (never throws). Back-compat shim — callers
+     *  that need to tell a true negative from a throttle should use {@link resolve}. */
     lookup(ip: string): Promise<IpEnrichment | null>;
+    /** Look up one IP via RDAP, distinguishing a genuine negative (`not_found`,
+     *  cached) from a transient upstream failure (`transient`, never cached so a
+     *  later retry can succeed). Bounded retry with backoff on transient. Never
+     *  throws — a flaky RIR must not fail the batch (issue #523). */
+    resolve(ip: string): Promise<RdapOutcome>;
+    /** One RDAP HTTP attempt mapped to an outcome (+ a Retry-After hint). */
+    private attempt;
     private put;
 }

package/dist/enrich/rdap.js

@@ -71,6 +71,9 @@ export class RdapResolver {
     timeoutMs;
     fetch;
     maxCache;
+    maxRetries;
+    backoffMs;
+    sleep;
     cache = new Map();
     /** Monotonic clock injected for tests; defaults to Date.now via a getter. */
     now;
@@ -81,34 +84,74 @@ export class RdapResolver {
         this.timeoutMs = opts.timeoutMs ?? 4000;
         this.fetch = opts.fetch ?? globalThis.fetch;
         this.maxCache = opts.maxCache ?? 10_000;
+        this.maxRetries = opts.maxRetries ?? 2;
+        this.backoffMs = opts.backoffMs ?? 250;
+        this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
         this.now = () => Date.now();
     }
-    /** Look up one IP via RDAP. Returns null on miss/error (never throws —
-     *  a flaky RIR must not fail the batch). Cached by IP with a TTL. */
+    /** Look up one IP via RDAP. Returns the enrichment on a hit, or null on a
+     *  miss OR a transient failure (never throws). Back-compat shim — callers
+     *  that need to tell a true negative from a throttle should use {@link resolve}. */
     async lookup(ip) {
+        const o = await this.resolve(ip);
+        return o.status === "ok" ? o.value : null;
+    }
+    /** Look up one IP via RDAP, distinguishing a genuine negative (`not_found`,
+     *  cached) from a transient upstream failure (`transient`, never cached so a
+     *  later retry can succeed). Bounded retry with backoff on transient. Never
+     *  throws — a flaky RIR must not fail the batch (issue #523). */
+    async resolve(ip) {
         if (ipv4ToInt(ip) === null && ipv6ToBigInt(ip) === null)
-            return null;
+            return { status: "not_found" };
         const cached = this.cache.get(ip);
-        if (cached && cached.expiresAt > this.now())
-            return cached.value;
-        let value = null;
+        if (cached && cached.expiresAt > this.now()) {
+            return cached.value ? { status: "ok", value: cached.value } : { status: "not_found" };
+        }
+        for (let attempt = 0;; attempt++) {
+            const { outcome, retryAfterMs } = await this.attempt(ip);
+            if (outcome.status === "ok") {
+                this.put(ip, { value: outcome.value, expiresAt: this.now() + this.ttlMs });
+                return outcome;
+            }
+            if (outcome.status === "not_found") {
+                this.put(ip, { value: null, expiresAt: this.now() + this.negTtlMs });
+                return outcome;
+            }
+            // transient — retry with backoff, but never cache it as a negative.
+            if (attempt >= this.maxRetries)
+                return outcome;
+            const backoff = retryAfterMs ?? Math.min(this.backoffMs * 2 ** attempt, 5000);
+            await this.sleep(backoff);
+        }
+    }
+    /** One RDAP HTTP attempt mapped to an outcome (+ a Retry-After hint). */
+    async attempt(ip) {
+        const ac = new AbortController();
+        const timer = setTimeout(() => ac.abort(), this.timeoutMs);
         try {
-            const ac = new AbortController();
-            const timer = setTimeout(() => ac.abort(), this.timeoutMs);
-            try {
-                const res = await this.fetch(`${this.baseUrl}/ip/${encodeURIComponent(ip)}`, { signal: ac.signal });
-                if (res.ok)
-                    value = parseRdapResponse(await res.json());
+            const res = await this.fetch(`${this.baseUrl}/ip/${encodeURIComponent(ip)}`, { signal: ac.signal });
+            if (res.ok) {
+                const parsed = parseRdapResponse(await res.json());
+                // A 2xx with no country/org is a genuine "no enrichment for this IP".
+                return { outcome: parsed ? { status: "ok", value: parsed } : { status: "not_found" } };
             }
-            finally {
-                clearTimeout(timer);
+            // 429/403 are the throttle responses RIRs use; 5xx is upstream trouble —
+            // both are transient. 404 (and other malformed-query 4xx) is a genuine
+            // negative for this address.
+            if (res.status === 429 || res.status === 403) {
+                return { outcome: { status: "transient", reason: "rate_limited" }, retryAfterMs: retryAfter(res) };
             }
+            if (res.status >= 500)
+                return { outcome: { status: "transient", reason: "upstream_error" } };
+            return { outcome: { status: "not_found" } };
         }
         catch {
-            value = null; // network/timeout/parse — treat as a miss
+            // AbortController fired → our own timeout; otherwise a network error.
+            return { outcome: { status: "transient", reason: ac.signal.aborted ? "timeout" : "network_error" } };
+        }
+        finally {
+            clearTimeout(timer);
         }
-        this.put(ip, { value, expiresAt: this.now() + (value ? this.ttlMs : this.negTtlMs) });
-        return value;
     }
     put(ip, entry) {
         if (this.cache.size >= this.maxCache) {
@@ -120,3 +163,14 @@ export class RdapResolver {
         this.cache.set(ip, entry);
     }
 }
+/** Parse a `Retry-After` header (delta-seconds form) into ms, capped at 5s so a
+ *  hostile/huge value can't stall a batch. Ignores the HTTP-date form. */
+function retryAfter(res) {
+    const raw = res.headers?.get?.("retry-after");
+    if (!raw)
+        return undefined;
+    const secs = Number(raw.trim());
+    if (!Number.isFinite(secs) || secs < 0)
+        return undefined;
+    return Math.min(secs * 1000, 5000);
+}

package/dist/enrich/rdap.test.js

@@ -76,3 +76,85 @@ describe("RdapResolver", () => {
         assert.equal(await r.lookup("8.8.8.8"), null);
     });
 });
+// Issue #523: a rate-limit / upstream failure must be distinguishable from a
+// true negative, and must NOT poison the cache as one.
+describe("RdapResolver.resolve — transient vs true negative (#523)", () => {
+    const noSleep = async () => { };
+    it("maps a 200 hit to ok", async () => {
+        const { fetch } = stubFetch(() => ({ ok: true, status: 200, body: RDAP_GOOGLE }));
+        const r = new RdapResolver({ fetch, sleep: noSleep });
+        assert.deepEqual(await r.resolve("8.8.8.8"), { status: "ok", value: { country: "US", org: "Google LLC" } });
+    });
+    it("maps a 404 to a true negative (not_found)", async () => {
+        const { fetch } = stubFetch(() => ({ ok: false, status: 404, body: {} }));
+        const r = new RdapResolver({ fetch, sleep: noSleep });
+        assert.deepEqual(await r.resolve("203.0.113.7"), { status: "not_found" });
+    });
+    it("maps a 200 with no country/org to not_found, not a bogus hit", async () => {
+        const { fetch } = stubFetch(() => ({ ok: true, status: 200, body: { handle: "x" } }));
+        const r = new RdapResolver({ fetch, sleep: noSleep });
+        assert.deepEqual(await r.resolve("203.0.113.8"), { status: "not_found" });
+    });
+    it("maps a 429 to transient:rate_limited after exhausting retries", async () => {
+        const { fetch, calls } = stubFetch(() => ({ ok: false, status: 429, body: {} }));
+        const r = new RdapResolver({ fetch, sleep: noSleep, maxRetries: 2 });
+        assert.deepEqual(await r.resolve("203.0.113.10"), { status: "transient", reason: "rate_limited" });
+        assert.equal(calls.length, 3, "1 initial + 2 retries");
+    });
+    it("maps a 5xx to transient:upstream_error", async () => {
+        const { fetch } = stubFetch(() => ({ ok: false, status: 503, body: {} }));
+        const r = new RdapResolver({ fetch, sleep: noSleep, maxRetries: 0 });
+        assert.deepEqual(await r.resolve("203.0.113.11"), { status: "transient", reason: "upstream_error" });
+    });
+    it("maps a thrown network error to transient:network_error", async () => {
+        const r = new RdapResolver({ fetch: (async () => { throw new Error("ECONNRESET"); }), sleep: noSleep, maxRetries: 0 });
+        assert.deepEqual(await r.resolve("8.8.8.8"), { status: "transient", reason: "network_error" });
+    });
+    it("does NOT cache a transient failure — a later success resolves", async () => {
+        let mode = "throttle";
+        const calls = [];
+        const fetch = async (url) => {
+            calls.push(url);
+            return mode === "throttle"
+                ? { ok: false, status: 429, json: async () => ({}) }
+                : { ok: true, status: 200, json: async () => RDAP_GOOGLE };
+        };
+        const r = new RdapResolver({ fetch, sleep: noSleep, maxRetries: 0 });
+        assert.deepEqual(await r.resolve("8.8.8.8"), { status: "transient", reason: "rate_limited" });
+        mode = "ok";
+        assert.deepEqual(await r.resolve("8.8.8.8"), { status: "ok", value: { country: "US", org: "Google LLC" } });
+        assert.equal(calls.length, 2, "transient was not cached, so the retry re-fetched");
+    });
+    it("retries a transient then succeeds, returning ok", async () => {
+        let n = 0;
+        const fetch = async () => {
+            n++;
+            return n === 1
+                ? { ok: false, status: 429, json: async () => ({}) }
+                : { ok: true, status: 200, json: async () => RDAP_GOOGLE };
+        };
+        const r = new RdapResolver({ fetch, sleep: noSleep, maxRetries: 2 });
+        assert.deepEqual(await r.resolve("8.8.8.8"), { status: "ok", value: { country: "US", org: "Google LLC" } });
+        assert.equal(n, 2, "succeeded on the first retry");
+    });
+    it("honors a numeric Retry-After for backoff (capped)", async () => {
+        const slept = [];
+        let n = 0;
+        const fetch = async () => {
+            n++;
+            if (n === 1)
+                return { ok: false, status: 429, json: async () => ({}), headers: { get: (h) => (h.toLowerCase() === "retry-after" ? "2" : null) } };
+            return { ok: true, status: 200, json: async () => RDAP_GOOGLE };
+        };
+        const r = new RdapResolver({ fetch, sleep: async (ms) => { slept.push(ms); }, maxRetries: 1 });
+        await r.resolve("8.8.8.8");
+        assert.deepEqual(slept, [2000], "waited the Retry-After delta (2s) before retrying");
+    });
+    it("caches a true negative (no re-fetch within negTtl)", async () => {
+        const { fetch, calls } = stubFetch(() => ({ ok: false, status: 404, body: {} }));
+        const r = new RdapResolver({ fetch, sleep: noSleep });
+        await r.resolve("203.0.113.7");
+        await r.resolve("203.0.113.7");
+        assert.equal(calls.length, 1, "negative cached");
+    });
+});

package/dist/index.js

@@ -794,6 +794,7 @@ async function main() {
             "Resolve a batch of IPv4 or IPv6 addresses to geo (country/city), ASN/org, and a hosting/proxy flag.",
             "When to use: answering 'where are these visitors from?' or 'which of these IPs are bots / datacenter / VPN exit nodes?' over access logs, without an out-of-band geo-API call per IP. Both IPv4 and IPv6 clients are resolved — don't pre-filter v6 out.",
             "Behavior: read-only. By default looks each IP up in a LOCAL offline dataset the operator configured (OMCP_IP_ENRICH_FILE) with NO external network call — safe in air-gapped deployments. Optionally, if the operator enabled OMCP_IP_ENRICH_RDAP, IPs the dataset doesn't cover fall back to an online RDAP query (country/org only) and the result carries via:'rdap'; the offline dataset is always preferred. Returns one row per input IP with found=true/false plus any known fields. If neither is configured it returns a clear notice explaining how to enable them.",
+            "RDAP rate-limits: a row with found=false AND transient:true (error names the cause, e.g. 'rate_limited') is NOT a confirmed negative — the registry throttled or failed the lookup, so the IP may resolve on a later retry or in a smaller batch. Such rows are counted in summary.transient (separate from summary.unmatched) and a top-level `note` is added. Don't treat transient rows as 'unknown/suspicious'; retry them (results are cached, so repeats are cheap).",
             "Related: pull the IPs from `query_logs` (use `labels`/`aggregate` to find the IPs of interest first).",
         ].join(" "), {
             ips: z

package/dist/tools/enrich-ips.d.ts

@@ -19,6 +19,12 @@ export interface IpEnrichmentResult {
     /** Which backend produced the hit — "dataset" (offline CSV) or "rdap"
      *  (online fallback). Absent when not found. */
     via?: "dataset" | "rdap";
+    /** True when `found:false` is NOT a confirmed negative but an RDAP upstream
+     *  failure (throttle/timeout/5xx) — the address may resolve on a later retry.
+     *  Issue #523: never conflate a rate-limit with "not in any registry". */
+    transient?: boolean;
+    /** Machine-readable reason when `transient` — e.g. "rate_limited". */
+    error?: string;
 }
 export declare function enrichIpsHandler(dataset: IpEnrichmentDataset | null, args: EnrichIpsArgs, _ctx?: RequestContext, rdap?: RdapResolver | null): Promise<{
     content: {

package/dist/tools/enrich-ips.js

@@ -38,6 +38,7 @@ rdap) {
     let invalid = 0;
     let matched = 0;
     let viaRdap = 0;
+    let transient = 0;
     for (const ip of ips) {
         if (typeof ip !== "string" || !isValidIp(ip)) {
             invalid++;
@@ -53,16 +54,26 @@ rdap) {
             continue;
         }
         if (rdap) {
-            const r = await rdap.lookup(ip);
-            if (r) {
+            const r = await rdap.resolve(ip);
+            if (r.status === "ok") {
                 matched++;
                 viaRdap++;
-                results.push({ ip, found: true, via: "rdap", ...r });
+                results.push({ ip, found: true, via: "rdap", ...r.value });
+                continue;
+            }
+            if (r.status === "transient") {
+                // NOT a confirmed negative — an RDAP throttle/timeout/5xx. Mark it so an
+                // agent doesn't treat the IP as "unknown" and can retry later (#523).
+                transient++;
+                results.push({ ip, found: false, transient: true, error: r.reason });
                 continue;
             }
         }
         results.push({ ip, found: false });
     }
+    // unmatched = confirmed negatives only; transient failures are reported
+    // separately so the all-clear can't silently absorb a wall of rate-limits.
+    const unmatched = ips.length - matched - invalid - transient;
     return {
         content: [
             {
@@ -72,12 +83,19 @@ rdap) {
                     summary: {
                         total: ips.length,
                         matched,
-                        unmatched: ips.length - matched - invalid,
+                        unmatched,
                         invalid,
-                        ...(rdap ? { viaRdap } : {}),
+                        ...(rdap ? { viaRdap, transient } : {}),
                     },
                     datasetSize: dataset?.size ?? 0,
                     ...(rdap ? { rdapEnabled: true } : {}),
+                    ...(transient > 0
+                        ? {
+                            note: `${transient} RDAP lookup(s) failed transiently (e.g. rate-limited by the ` +
+                                `registry) and are marked transient:true — these are NOT confirmed negatives. ` +
+                                `Retry them later or in a smaller batch; results are cached so repeats are cheap.`,
+                        }
+                        : {}),
                 }, null, 2),
             },
         ],

package/dist/tools/enrich-ips.test.js

@@ -48,13 +48,21 @@ describe("enrichIpsHandler (R6, issue #415 Gap B)", () => {
     });
 });
 describe("enrichIpsHandler — optional RDAP fallback (issue #477)", () => {
-    // Minimal RdapResolver stub: returns a fixed hit for one IP, null otherwise,
-    // and records which IPs it was asked about (to prove CSV-first).
-    function rdapStub(hit) {
+    // Minimal RdapResolver stub: returns a fixed hit for one IP, a true negative
+    // otherwise, and records which IPs it was asked about (to prove CSV-first).
+    // `transient` IPs resolve to a transient outcome (e.g. rate-limited) — #523.
+    function rdapStub(hit, transient = {}) {
         const asked = [];
         return {
             asked,
-            resolver: { lookup: async (ip) => { asked.push(ip); return hit[ip] ?? null; } },
+            resolver: {
+                resolve: async (ip) => {
+                    asked.push(ip);
+                    if (transient[ip])
+                        return { status: "transient", reason: transient[ip] };
+                    return hit[ip] ? { status: "ok", value: hit[ip] } : { status: "not_found" };
+                },
+            },
         };
     }
     it("with no dataset but RDAP enabled → not 'not configured'; resolves via RDAP", async () => {
@@ -90,4 +98,29 @@ describe("enrichIpsHandler — optional RDAP fallback (issue #477)", () => {
         assert.match(out.error, /not configured/i);
         assert.match(out.error, /OMCP_IP_ENRICH_RDAP/);
     });
+    // Issue #523: a rate-limited lookup must NOT masquerade as a confirmed negative.
+    it("marks a rate-limited lookup transient (not a confirmed negative)", async () => {
+        const { resolver } = rdapStub({ "8.8.8.8": { country: "US", org: "Google LLC" } }, { "203.0.113.10": "rate_limited" });
+        const out = parse(await enrichIpsHandler(null, { ips: ["8.8.8.8", "203.0.113.10"] }, undefined, resolver));
+        const hit = out.results.find((r) => r.ip === "8.8.8.8");
+        assert.equal(hit.found, true);
+        const throttled = out.results.find((r) => r.ip === "203.0.113.10");
+        assert.equal(throttled.found, false);
+        assert.equal(throttled.transient, true);
+        assert.equal(throttled.error, "rate_limited");
+        // The throttled IP is counted as transient, NOT folded into `unmatched`.
+        assert.equal(out.summary.matched, 1);
+        assert.equal(out.summary.transient, 1);
+        assert.equal(out.summary.unmatched, 0);
+        assert.match(out.note, /NOT confirmed negatives/i);
+    });
+    it("a genuine miss stays a clean negative — no transient marker, no note", async () => {
+        const { resolver } = rdapStub({ "8.8.8.8": { country: "US", org: "Google LLC" } });
+        const out = parse(await enrichIpsHandler(null, { ips: ["8.8.8.8", "203.0.113.9"] }, undefined, resolver));
+        const miss = out.results.find((r) => r.ip === "203.0.113.9");
+        assert.equal(miss.found, false);
+        assert.equal(miss.transient, undefined);
+        assert.equal(out.summary.transient, 0);
+        assert.equal(out.note, undefined);
+    });
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thotischner/observability-mcp",
-  "version": "3.8.2",
+  "version": "3.8.3",
   "description": "Unified observability gateway for AI agents — one MCP server for Prometheus, Loki, and any backend",
   "type": "module",
   "license": "Apache-2.0",