@boy-offi9-inc/reqkit 0.7.1 → 0.8.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@boy-offi9-inc/reqkit",
-  "version": "0.7.1",
-  "description": "Small, composable HTTP helpers — retry with backoff and Retry-After, timeouts, normalized errors, rate limiter, circuit breaker, response cache, resumable downloads, SSE/NDJSON streams, mock fetch, and an opt-in createFetch. Not a client; works alongside fetch/axios/anything.",
+  "version": "0.8.0",
+  "description": "Small, composable HTTP helpers \u2014 retry with backoff and Retry-After, timeouts, normalized errors, rate limiter, circuit breaker, response cache, resumable downloads, SSE/NDJSON streams, mock fetch, and an opt-in createFetch. Not a client; works alongside fetch/axios/anything.",
   "main": "./src/index.js",
   "types": "./types/index.d.ts",
   "exports": {
@@ -27,7 +27,8 @@
     "test": "node --test",
     "typecheck": "tsc -p tsconfig.json",
     "smoke": "node smoke/run.mjs",
-    "prepublishOnly": "npm run build && npm test && npm run typecheck"
+    "check": "npm run build && npm test && npm run typecheck && npm run smoke",
+    "prepublishOnly": "npm run check"
   },
   "keywords": [
     "http",
@@ -61,7 +62,10 @@
     "pagination",
     "auth",
     "token-refresh",
-    "link-header"
+    "link-header",
+    "stale-if-error",
+    "observability",
+    "logging"
   ],
   "author": "Boy Offi9",
   "license": "MIT",
@@ -83,8 +87,8 @@
     "src",
     "types",
     "dist",
+    "docs/recipes.md",
     "README.md",
-    "RECIPES.md",
     "LICENSE"
   ],
   "devDependencies": {

package/src/{createFetch.js → http/createFetch.js}

@@ -1,15 +1,15 @@
-const { withRetry, RetryAbortedError } = require("./withRetry");
-const { withTimeout } = require("./withTimeout");
-const { withDedupe } = require("./withDedupe");
+const { withRetry, RetryAbortedError } = require("../resilience/retry");
+const { withTimeout } = require("../resilience/timeout");
+const { withDedupe } = require("../resilience/dedupe");
 const { buildQuery } = require("./buildQuery");
 const { parseJsonSafe } = require("./parseJsonSafe");
-const { normalizeError } = require("./normalizeError");
-const { HTTPError } = require("./HTTPError");
-const { createRateLimiter } = require("./createRateLimiter");
-const { createCircuitBreaker } = require("./createCircuitBreaker");
-const { createMemoryCache } = require("./createMemoryCache");
-const { cacheKey, snapshotResponse, entryToResponse } = require("./_cache");
-const { discardBody, getOrigin } = require("./_http");
+const { normalizeError } = require("../errors/normalizeError");
+const { HTTPError } = require("../errors/HTTPError");
+const { createRateLimiter } = require("../resilience/rateLimiter");
+const { createCircuitBreaker } = require("../resilience/circuitBreaker");
+const { createMemoryCache } = require("./cache/memoryCache");
+const { cacheKey, snapshotResponse, entryToResponse } = require("./cache/entry");
+const { discardBody, getOrigin } = require("../internal/http");
 
 const DEFAULT_RETRY_STATUS_CODES = [408, 425, 429, 500, 502, 503, 504];
 // Safe to repeat without side effects. POST/PATCH are deliberately absent.
@@ -94,8 +94,11 @@ function mergeDefaults(base, extra) {
  *  - `rateLimit`: stay under a budget and obey X-RateLimit-* / Retry-After
  *    *before* the server has to say 429
  *  - `circuitBreaker`: fail fast against a service that keeps failing
- *  - `cache`: in-memory GET cache with TTL, stale-while-revalidate and
- *    ETag / Last-Modified revalidation (304s)
+ *  - `cache`: in-memory GET cache with TTL, stale-while-revalidate,
+ *    stale-if-error and ETag / Last-Modified revalidation (304s)
+ *  - `onEvent`: one callback for everything that happens (request, attempt,
+ *    retry, response, error, cache, rate-limit waits, circuit changes, token
+ *    refreshes) — plug it into your logger or metrics
  *  - concurrent GET de-duplication (withDedupe)
  *  - `auth`: attach a token, and on a 401 refresh it ONCE (shared by all
  *    concurrent requests) and replay the request
@@ -117,8 +120,9 @@ function mergeDefaults(base, extra) {
  *   createRateLimiter() instance to share one budget
  * @param {boolean | object | Function} [defaults.circuitBreaker] - options for
  *   a per-origin breaker, a createCircuitBreaker() instance, or `(url) => breaker`
- * @param {boolean | object} [defaults.cache] - `{ ttl, staleWhileRevalidate,
- *   maxEntries, maxBodyBytes, store, onEvent }`; `true` = 60s TTL
+ * @param {boolean | object} [defaults.cache] - `{ ttl, staleWhileRevalidate, staleIfError,
+ *   maxEntries, maxBodyBytes, store, onEvent }`; `true` = 60s TTL. `staleIfError` (ms) lets a
+ *   cached copy be served for that long past its TTL when the network fails or the server answers 5xx.
  * @param {object} [defaults.auth] - `{ getToken, refresh, header?, scheme?, shouldRefresh? }`.
  *   `getToken()` supplies the current token (sync or async); on a 401 (or
  *   whatever `shouldRefresh(response)` says) `refresh({ token, response })` is
@@ -133,6 +137,9 @@ function mergeDefaults(base, extra) {
  * @param {(ctx: {url: string, init: object}) => void | Promise<void>} [defaults.onRequest] -
  *   runs once per call; may mutate ctx.url / ctx.init (e.g. add a header)
  * @param {(ctx: {url: string, init: object, response: Response}) => void | Promise<void>} [defaults.onResponse]
+ * @param {(event: object) => void | Promise<void>} [defaults.onEvent] - receives `{ type, ... }` events:
+ *   "request", "attempt", "retry", "response", "error", "cache", "ratelimit", "circuit", "auth".
+ *   Errors thrown by the handler are swallowed — logging must never break a request.
  */
 function createFetch(defaults = {}) {
   const {
@@ -150,10 +157,22 @@ function createFetch(defaults = {}) {
     fetch: customFetch,
     onRequest,
     onResponse,
+    onEvent,
   } = defaults;
 
   const doFetch = (...args) => (customFetch || globalThis.fetch)(...args);
 
+  // A throwing (or rejecting) handler must never break a request.
+  const emitEvent = (event) => {
+    if (!onEvent) return;
+    try {
+      const result = onEvent(event);
+      if (result && typeof result.catch === "function") result.catch(() => {});
+    } catch {
+      // ignore
+    }
+  };
+
   // Origins allowed to receive default credentials. Empty = no baseUrl to compare against = unrestricted.
   const trusted = new Set();
   for (const o of [baseUrl, ...(trustedOrigins || [])]) {
@@ -183,7 +202,13 @@ function createFetch(defaults = {}) {
         : (url) => {
             const origin = originOf(url);
             if (!breakerRegistry.has(origin)) {
-              breakerRegistry.set(origin, createCircuitBreaker(circuitBreaker === true ? {} : circuitBreaker));
+              const options = circuitBreaker === true ? {} : { ...circuitBreaker };
+              const userOnStateChange = options.onStateChange;
+              options.onStateChange = (state, previous) => {
+                if (userOnStateChange) userOnStateChange(state, previous);
+                emitEvent({ type: "circuit", origin, state, previous });
+              };
+              breakerRegistry.set(origin, createCircuitBreaker(options));
             }
             return breakerRegistry.get(origin);
           };
@@ -193,6 +218,7 @@ function createFetch(defaults = {}) {
     : {
         ttl: 60000,
         staleWhileRevalidate: 0,
+        staleIfError: 0,
         maxEntries: 100,
         maxBodyBytes: undefined,
         ...(cache === true ? {} : cache),
@@ -206,7 +232,10 @@ function createFetch(defaults = {}) {
   }
   const pendingStores = cacheCfg ? cacheCfg.pending : null;
   const revalidating = cacheCfg ? cacheCfg.revalidating : null;
-  const emit = (type, url) => cacheCfg && cacheCfg.onEvent && cacheCfg.onEvent({ type, url });
+  const emit = (type, url) => {
+    if (cacheCfg && cacheCfg.onEvent) cacheCfg.onEvent({ type, url });
+    emitEvent({ type: "cache", url, method: "GET", event: type });
+  };
 
   // Shared in-flight GETs. Every caller gets its own clone of the response so
   // body reads don't fight each other.
@@ -230,9 +259,12 @@ function createFetch(defaults = {}) {
   // Returns a token newer than `usedToken`, refreshing at most once no matter
   // how many requests were rejected at the same time. If someone else already
   // refreshed (the current token isn't the one we used), just use that.
-  async function obtainFreshToken(authCfg, tokenUsed, response) {
+  async function obtainFreshToken(authCfg, tokenUsed, response, info) {
     const current = await authCfg.getToken();
-    if (current && current !== tokenUsed) return current;
+    if (current && current !== tokenUsed) {
+      emitEvent({ type: "auth", ...info, outcome: "reused" });
+      return current;
+    }
 
     const state = refreshStateFor(authCfg);
     if (!state.inflight) {
@@ -245,10 +277,21 @@ function createFetch(defaults = {}) {
         }
       })();
     }
-    return state.inflight;
+    try {
+      const token = await state.inflight;
+      emitEvent({ type: "auth", ...info, outcome: "refreshed" });
+      return token;
+    } catch (error) {
+      emitEvent({ type: "auth", ...info, outcome: "failed", error });
+      throw error;
+    }
   }
 
-  async function request(input, init = {}) {
+  // May a cached copy be served because the origin is failing?
+  const withinStaleIfError = (entry) =>
+    cacheCfg && cacheCfg.staleIfError > 0 && Date.now() - entry.storedAt < cacheCfg.ttl + cacheCfg.staleIfError;
+
+  async function execute(input, init, state) {
     const {
       query,
       json,
@@ -263,6 +306,8 @@ function createFetch(defaults = {}) {
 
     const method = String(fetchInit.method || "GET").toUpperCase();
     const url = joinUrl(baseUrl, input, query);
+    state.url = url;
+    state.method = method;
     const credentialsAllowed = isTrustedTarget(url);
 
     const defaultHeaderSet = await resolveHeaders(defaultHeaders);
@@ -285,6 +330,9 @@ function createFetch(defaults = {}) {
     if (onRequest) await onRequest(ctx);
     const finalHeaders = ctx.init.headers instanceof Headers ? ctx.init.headers : new Headers(ctx.init.headers);
     const finalMethod = String(ctx.init.method || method).toUpperCase();
+    state.url = ctx.url;
+    state.method = finalMethod;
+    emitEvent({ type: "request", url: ctx.url, method: finalMethod });
 
     // --- auth: attach the current token (unless the caller set the header) ----
     const authHeader = auth ? String(auth.header || "authorization").toLowerCase() : null;
@@ -310,8 +358,13 @@ function createFetch(defaults = {}) {
     // `netSignal` are parameters so background revalidation can reuse this.
     function network(netHeaders, netSignal) {
       const attemptOnce = () => {
+        state.attempts++;
+        emitEvent({ type: "attempt", url: ctx.url, method: finalMethod, attempt: state.attempts });
         const attempt = async () => {
-          if (limiter) await limiter.acquire({ signal: netSignal });
+          if (limiter) {
+            const waitedMs = await limiter.acquire({ signal: netSignal });
+            if (waitedMs > 0) emitEvent({ type: "ratelimit", url: ctx.url, method: finalMethod, waitedMs });
+          }
           const res = await withTimeout(
             (attemptSignal) => {
               const attemptInit = { ...ctx.init, method: finalMethod, headers: netHeaders, signal: attemptSignal };
@@ -343,6 +396,19 @@ function createFetch(defaults = {}) {
             ...retryRest,
             retries,
             retryStatusCodes: statusCodes || DEFAULT_RETRY_STATUS_CODES,
+            onRetry: (err, attempt, delayMs, rctx) => {
+              if (retryRest.onRetry) retryRest.onRetry(err, attempt, delayMs, rctx);
+              emitEvent({
+                type: "retry",
+                url: ctx.url,
+                method: finalMethod,
+                attempt: attempt + 1,
+                delayMs,
+                reason: rctx.reason === "result" ? "status" : "error",
+                status: rctx.status,
+                error: err,
+              });
+            },
             shouldRetry:
               customShouldRetry ||
               ((err) => {
@@ -374,6 +440,17 @@ function createFetch(defaults = {}) {
     let entry = null;
 
     const finish = async (response) => {
+      state.responded = true;
+      emitEvent({
+        type: "response",
+        url: ctx.url,
+        method: finalMethod,
+        status: response.status,
+        ok: response.ok,
+        durationMs: Date.now() - state.startedAt,
+        attempts: state.attempts,
+        cache: state.cache,
+      });
       if (onResponse) await onResponse({ url: ctx.url, init: ctx.init, response });
       if (throwHttpErrors && !response.ok) {
         throw new HTTPError(response, { method: finalMethod, url: ctx.url });
@@ -388,10 +465,12 @@ function createFetch(defaults = {}) {
         const age = Date.now() - entry.storedAt;
         if (age < cacheCfg.ttl) {
           emit("hit", ctx.url);
+          state.cache = "hit";
           return finish(entryToResponse(entry));
         }
         if (age < cacheCfg.ttl + cacheCfg.staleWhileRevalidate) {
           emit("stale", ctx.url);
+          state.cache = "stale";
           if (!revalidating.has(key)) {
             revalidating.add(key);
             const staleEntry = entry;
@@ -428,13 +507,19 @@ function createFetch(defaults = {}) {
     } catch (err) {
       // Present a user abort as the plain AbortError they caused, not as retry internals.
       if (err instanceof RetryAbortedError && signal && signal.aborted) throw signal.reason ?? err;
+      // The network failed but we still hold a usable copy: serve it (never for a deliberate abort).
+      if (entry && withinStaleIfError(entry) && !(signal && signal.aborted) && !(err && err.name === "AbortError")) {
+        emit("stale-if-error", ctx.url);
+        state.cache = "stale-if-error";
+        return finish(entryToResponse(entry));
+      }
       throw err;
     }
 
     // --- auth: a rejected token is refreshed once, then the request replays ----
     const needsRefresh = auth && (auth.shouldRefresh ? auth.shouldRefresh(response) : response.status === 401);
     if (usedToken && needsRefresh && !isStreamBody(ctx.init.body)) {
-      const fresh = await obtainFreshToken(auth, usedToken, response);
+      const fresh = await obtainFreshToken(auth, usedToken, response, { url: ctx.url, method: finalMethod });
       if (fresh) {
         await discardBody(response);
         finalHeaders.set(authHeader, formatToken(fresh));
@@ -449,15 +534,25 @@ function createFetch(defaults = {}) {
       }
     }
 
+    // The server answered, but with an error, and we still hold a usable copy.
+    if (useCache && entry && response.status >= 500 && withinStaleIfError(entry)) {
+      await discardBody(response);
+      emit("stale-if-error", ctx.url);
+      state.cache = "stale-if-error";
+      return finish(entryToResponse(entry));
+    }
+
     if (useCache) {
       if (response.status === 304 && entry) {
         entry.storedAt = Date.now();
         await cacheCfg.store.set(key, entry);
         emit("revalidated", ctx.url);
+        state.cache = "revalidated";
         await discardBody(response);
         response = entryToResponse(entry);
       } else {
         emit("miss", ctx.url);
+        state.cache = "miss";
         if (response.status === 200) scheduleStore(key, response, ctx.url);
       }
     }
@@ -465,6 +560,19 @@ function createFetch(defaults = {}) {
     return finish(response);
   }
 
+  async function request(input, init = {}) {
+    const state = { startedAt: Date.now(), attempts: 0, responded: false, cache: undefined, url: undefined, method: undefined };
+    try {
+      return await execute(input, init, state);
+    } catch (error) {
+      // A non-2xx HTTPError was already reported as a "response" event.
+      if (!state.responded) {
+        emitEvent({ type: "error", url: state.url, method: state.method, error, durationMs: Date.now() - state.startedAt, attempts: state.attempts });
+      }
+      throw error;
+    }
+  }
+
   /** The `baseUrl` this instance was created with (paginateLinks uses it to stay on-origin). */
   request.baseUrl = baseUrl;
 

package/src/index.js

@@ -1,21 +1,21 @@
-const { withRetry, RetryAbortedError, StopRetry } = require("./withRetry");
-const { withTimeout, TimeoutError } = require("./withTimeout");
-const { normalizeError } = require("./normalizeError");
-const { parseJsonSafe } = require("./parseJsonSafe");
-const { buildQuery } = require("./buildQuery");
-const { retryAfterAware, parseRetryAfter, getWaitMsFromHeaders } = require("./retryAfterAware");
-const { withProgress, BodyTooLargeError } = require("./withProgress");
-const { withDedupe } = require("./withDedupe");
-const { createLimiter } = require("./createLimiter");
-const { createFetch, HTTPError } = require("./createFetch");
-const { createRateLimiter } = require("./createRateLimiter");
-const { createCircuitBreaker, CircuitOpenError } = require("./createCircuitBreaker");
-const { createMemoryCache } = require("./createMemoryCache");
-const { createMockFetch } = require("./createMockFetch");
-const { downloadResumable } = require("./downloadResumable");
-const { readLines, readNDJSON, readSSE } = require("./streams");
-const { streamSSE } = require("./streamSSE");
-const { paginate, paginatePages, paginateLinks, parseLinkHeader } = require("./paginate");
+const { withRetry, RetryAbortedError, StopRetry } = require("./resilience/retry");
+const { withTimeout, TimeoutError } = require("./resilience/timeout");
+const { normalizeError } = require("./errors/normalizeError");
+const { parseJsonSafe } = require("./http/parseJsonSafe");
+const { buildQuery } = require("./http/buildQuery");
+const { retryAfterAware, parseRetryAfter, getWaitMsFromHeaders } = require("./resilience/retryAfterAware");
+const { withProgress, BodyTooLargeError } = require("./streams/progress");
+const { withDedupe } = require("./resilience/dedupe");
+const { createLimiter } = require("./resilience/limiter");
+const { createFetch, HTTPError } = require("./http/createFetch");
+const { createRateLimiter } = require("./resilience/rateLimiter");
+const { createCircuitBreaker, CircuitOpenError } = require("./resilience/circuitBreaker");
+const { createMemoryCache } = require("./http/cache/memoryCache");
+const { createMockFetch } = require("./testing/createMockFetch");
+const { downloadResumable } = require("./streams/download");
+const { readLines, readNDJSON, readSSE } = require("./streams/readers");
+const { streamSSE } = require("./streams/streamSSE");
+const { paginate, paginatePages, paginateLinks, parseLinkHeader } = require("./pagination/paginate");
 
 module.exports = {
   withRetry,

package/src/{paginate.js → pagination/paginate.js}

@@ -1,5 +1,5 @@
-const { HTTPError } = require("./HTTPError");
-const { getOrigin } = require("./_http");
+const { HTTPError } = require("../errors/HTTPError");
+const { getOrigin } = require("../internal/http");
 
 const MAX_LINK_HEADER_LENGTH = 64 * 1024;
 

package/src/{createRateLimiter.js → resilience/rateLimiter.js}

@@ -1,5 +1,5 @@
-const { abortableSleep } = require("./_sleep");
-const { getRateLimitInfo } = require("./_http");
+const { abortableSleep } = require("../internal/sleep");
+const { getRateLimitInfo } = require("../internal/http");
 
 function abortable(promise, signal) {
   if (!signal) return promise;
@@ -50,6 +50,7 @@ function createRateLimiter(opts = {}) {
   let tail = Promise.resolve();
 
   async function take(signal) {
+    let waitedMs = 0;
     for (;;) {
       signal?.throwIfAborted?.();
       const now = Date.now();
@@ -61,13 +62,15 @@ function createRateLimiter(opts = {}) {
 
       if (waitMs <= 0) {
         stamps.push(now);
-        return;
+        return waitedMs;
       }
+      const waitStartedAt = Date.now();
       await abortableSleep(waitMs, signal);
+      waitedMs += Date.now() - waitStartedAt;
     }
   }
 
-  /** Resolves when it's your turn. Rejects with the signal's reason if aborted while waiting. */
+  /** Resolves with the time spent waiting. Rejects if aborted while waiting. */
   function acquire({ signal } = {}) {
     const turn = tail.then(() => take(signal));
     tail = turn.catch(() => {});

package/src/{withRetry.js → resilience/retry.js}

@@ -1,5 +1,5 @@
-const { RetryAbortedError, sleep } = require("./_sleep");
-const { getWaitMsFromHeaders, discardBody } = require("./_http");
+const { RetryAbortedError, sleep } = require("../internal/sleep");
+const { getWaitMsFromHeaders, discardBody } = require("../internal/http");
 
 function defaultShouldRetry(err) {
   // Retry thrown errors except deliberate aborts. Narrow it with `shouldRetry`

package/src/{retryAfterAware.js → resilience/retryAfterAware.js}

@@ -1,6 +1,6 @@
-const { withRetry } = require("./withRetry");
-const { normalizeError } = require("./normalizeError");
-const { parseRetryAfter, getWaitMsFromHeaders } = require("./_http");
+const { withRetry } = require("./retry");
+const { normalizeError } = require("../errors/normalizeError");
+const { parseRetryAfter, getWaitMsFromHeaders } = require("../internal/http");
 
 function defaultShouldRetryError(err) {
   const normalized = normalizeError(err);

package/src/{downloadResumable.js → streams/download.js}

@@ -1,8 +1,8 @@
-const { HTTPError } = require("./HTTPError");
-const { BodyTooLargeError } = require("./withProgress");
-const { normalizeError } = require("./normalizeError");
-const { getWaitMsFromHeaders } = require("./_http");
-const { abortableSleep } = require("./_sleep");
+const { HTTPError } = require("../errors/HTTPError");
+const { BodyTooLargeError } = require("./progress");
+const { normalizeError } = require("../errors/normalizeError");
+const { getWaitMsFromHeaders } = require("../internal/http");
+const { abortableSleep } = require("../internal/sleep");
 
 const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
 

package/src/{streamSSE.js → streams/streamSSE.js}

@@ -1,8 +1,8 @@
-const { readSSE } = require("./streams");
-const { HTTPError } = require("./HTTPError");
-const { normalizeError } = require("./normalizeError");
-const { getWaitMsFromHeaders, discardBody } = require("./_http");
-const { abortableSleep } = require("./_sleep");
+const { readSSE } = require("./readers");
+const { HTTPError } = require("../errors/HTTPError");
+const { normalizeError } = require("../errors/normalizeError");
+const { getWaitMsFromHeaders, discardBody } = require("../internal/http");
+const { abortableSleep } = require("../internal/sleep");
 
 function isRetryableStatus(status) {
   return status === 408 || status === 425 || status === 429 || status >= 500;

package/src/{createMockFetch.js → testing/createMockFetch.js}

@@ -1,4 +1,4 @@
-const { abortableSleep } = require("./_sleep");
+const { abortableSleep } = require("../internal/sleep");
 
 const BODYLESS = new Set([101, 204, 205, 304]);
 

package/types/errors.d.ts

@@ -0,0 +1,24 @@
+// Error types and classification — src/errors/
+
+export interface NormalizedError {
+  message: string;
+  status: number | null;
+  code: string | null;
+  isNetworkError: boolean;
+  isTimeout: boolean;
+  /** True when the request was cancelled on purpose (AbortController, axios cancel). */
+  isAborted: boolean;
+  cause: unknown;
+}
+
+export function normalizeError(err: unknown): NormalizedError;
+
+export class HTTPError extends Error {
+  name: "HTTPError";
+  status: number;
+  statusText: string;
+  response: Response;
+  url: string;
+  method: string;
+  constructor(response: Response, request: { method: string; url: string });
+}