@boy-offi9-inc/reqkit 0.5.0 → 0.7.0

package/dist/reqkit.mjs

@@ -1,4 +1,4 @@
-/*! @boy-offi9-inc/reqkit v0.5.0 — ESM bundle (generated by scripts/build-esm.js, do not edit) */
+/*! @boy-offi9-inc/reqkit v0.7.0 — ESM bundle (generated by scripts/build-esm.js, do not edit) */
 const __defs = {
 "./HTTPError": function (module, exports, require) {
 class HTTPError extends Error {
@@ -454,6 +454,18 @@ function originOf(url) {
   }
 }
 
+// One in-flight token refresh per `auth` config object, so concurrent 401s —
+// even across extend() children — trigger exactly one refresh.
+const refreshStates = new WeakMap();
+function refreshStateFor(auth) {
+  let state = refreshStates.get(auth);
+  if (!state) {
+    state = { inflight: null };
+    refreshStates.set(auth, state);
+  }
+  return state;
+}
+
 function mergeDefaults(base, extra) {
   const merged = { ...base, ...extra };
 
@@ -487,6 +499,8 @@ function mergeDefaults(base, extra) {
  *  - `cache`: in-memory GET cache with TTL, stale-while-revalidate and
  *    ETag / Last-Modified revalidation (304s)
  *  - concurrent GET de-duplication (withDedupe)
+ *  - `auth`: attach a token, and on a 401 refresh it ONCE (shared by all
+ *    concurrent requests) and replay the request
  *  - `HTTPError` on non-2xx (turn off with `throwHttpErrors: false`)
  *  - `onRequest` / `onResponse` hooks
  *
@@ -504,6 +518,11 @@ function mergeDefaults(base, extra) {
  *   a per-origin breaker, a createCircuitBreaker() instance, or `(url) => breaker`
  * @param {boolean | object} [defaults.cache] - `{ ttl, staleWhileRevalidate,
  *   maxEntries, maxBodyBytes, store, onEvent }`; `true` = 60s TTL
+ * @param {object} [defaults.auth] - `{ getToken, refresh, header?, scheme?, shouldRefresh? }`.
+ *   `getToken()` supplies the current token (sync or async); on a 401 (or
+ *   whatever `shouldRefresh(response)` says) `refresh({ token, response })` is
+ *   called once for all concurrent callers and the request is replayed with the
+ *   new token. `header` defaults to "authorization", `scheme` to "Bearer" ("" for none).
  * @param {boolean} [defaults.dedupe=false] - merge identical concurrent GET/HEAD calls
  * @param {boolean} [defaults.throwHttpErrors=true]
  * @param {typeof fetch} [defaults.fetch=globalThis.fetch]
@@ -520,6 +539,7 @@ function createFetch(defaults = {}) {
     rateLimit,
     circuitBreaker,
     cache,
+    auth,
     dedupe = false,
     throwHttpErrors: defaultThrow = true,
     fetch: customFetch,
@@ -590,6 +610,27 @@ function createFetch(defaults = {}) {
     pendingStores.set(key, p);
   }
 
+  // Returns a token newer than `usedToken`, refreshing at most once no matter
+  // how many requests were rejected at the same time. If someone else already
+  // refreshed (the current token isn't the one we used), just use that.
+  async function obtainFreshToken(authCfg, tokenUsed, response) {
+    const current = await authCfg.getToken();
+    if (current && current !== tokenUsed) return current;
+
+    const state = refreshStateFor(authCfg);
+    if (!state.inflight) {
+      state.inflight = (async () => {
+        try {
+          const out = await authCfg.refresh({ token: tokenUsed, response });
+          return typeof out === "string" && out ? out : await authCfg.getToken();
+        } finally {
+          state.inflight = null;
+        }
+      })();
+    }
+    return state.inflight;
+  }
+
   async function request(input, init = {}) {
     const {
       query,
@@ -617,6 +658,15 @@ function createFetch(defaults = {}) {
     const finalHeaders = ctx.init.headers instanceof Headers ? ctx.init.headers : new Headers(ctx.init.headers);
     const finalMethod = String(ctx.init.method || method).toUpperCase();
 
+    // --- auth: attach the current token (unless the caller set the header) ----
+    const authHeader = auth ? String(auth.header || "authorization").toLowerCase() : null;
+    const formatToken = (token) => (auth.scheme === "" ? String(token) : `${auth.scheme === undefined ? "Bearer" : auth.scheme} ${token}`);
+    let usedToken = null;
+    if (auth && !finalHeaders.has(authHeader)) {
+      usedToken = (await auth.getToken()) || null;
+      if (usedToken) finalHeaders.set(authHeader, formatToken(usedToken));
+    }
+
     // --- retry policy ------------------------------------------------------
     const retryCfg = retryOpt === false || defaultRetry === false ? false : { ...defaultRetry, ...retryOpt };
     const methods = (retryCfg && retryCfg.methods) || DEFAULT_RETRY_METHODS;
@@ -692,7 +742,7 @@ function createFetch(defaults = {}) {
       !ctx.init.body &&
       cacheMode !== false &&
       cacheMode !== "no-store";
-    const key = useCache ? cacheKey(finalMethod, ctx.url, finalHeaders) : null;
+    let key = useCache ? cacheKey(finalMethod, ctx.url, finalHeaders) : null;
     let entry = null;
 
     const finish = async (response) => {
@@ -753,6 +803,24 @@ function createFetch(defaults = {}) {
       throw err;
     }
 
+    // --- auth: a rejected token is refreshed once, then the request replays ----
+    const needsRefresh = auth && (auth.shouldRefresh ? auth.shouldRefresh(response) : response.status === 401);
+    if (usedToken && needsRefresh && !isStreamBody(ctx.init.body)) {
+      const fresh = await obtainFreshToken(auth, usedToken, response);
+      if (fresh) {
+        await discardBody(response);
+        finalHeaders.set(authHeader, formatToken(fresh));
+        entry = null; // belongs to the old token's cache key
+        if (useCache) key = cacheKey(finalMethod, ctx.url, finalHeaders);
+        try {
+          response = await network(finalHeaders, signal);
+        } catch (err) {
+          if (err instanceof RetryAbortedError && signal && signal.aborted) throw signal.reason ?? err;
+          throw err;
+        }
+      }
+    }
+
     if (useCache) {
       if (response.status === 304 && entry) {
         entry.storedAt = Date.now();
@@ -1413,6 +1481,8 @@ const { createMemoryCache } = require("./createMemoryCache");
 const { createMockFetch } = require("./createMockFetch");
 const { downloadResumable } = require("./downloadResumable");
 const { readLines, readNDJSON, readSSE } = require("./streams");
+const { streamSSE } = require("./streamSSE");
+const { paginate, paginatePages, paginateLinks, parseLinkHeader } = require("./paginate");
 
 module.exports = {
   withRetry,
@@ -1441,6 +1511,11 @@ module.exports = {
   readLines,
   readNDJSON,
   readSSE,
+  streamSSE,
+  paginate,
+  paginatePages,
+  paginateLinks,
+  parseLinkHeader,
 };
 
 },
@@ -1501,7 +1576,7 @@ function isNetworkCode(code) {
 
 // Browsers and Node throw a bare TypeError for network-level fetch failures.
 const FETCH_NETWORK_MESSAGE =
-  /fetch failed|failed to fetch|network ?request failed|load failed|networkerror when attempting to fetch/i;
+  /fetch failed|failed to fetch|network ?request failed|load failed|networkerror when attempting to fetch|^terminated$|^network error$/i;
 
 function isFetchNetworkTypeError(err) {
   return err.name === "TypeError" && FETCH_NETWORK_MESSAGE.test(err.message || "");
@@ -1570,6 +1645,175 @@ function normalizeError(err) {
 
 module.exports = { normalizeError };
 
+},
+"./paginate": function (module, exports, require) {
+const { HTTPError } = require("./HTTPError");
+
+/**
+ * Parses an RFC 8288 `Link` header (GitHub, GitLab, WordPress... use it for
+ * pagination) into `{ rel: url }`. Pass `base` to resolve relative URLs.
+ *
+ * ```js
+ * parseLinkHeader('<https://api.x/items?page=2>; rel="next", <https://api.x/items?page=9>; rel="last"');
+ * // { next: "https://api.x/items?page=2", last: "https://api.x/items?page=9" }
+ * ```
+ *
+ * A link with several rels (`rel="next last"`) is listed under each; the first
+ * link wins when a rel appears twice. Commas inside `<...>` are handled.
+ *
+ * @param {string | null | undefined} header
+ * @param {string} [base]
+ * @returns {Record<string, string>}
+ */
+function parseLinkHeader(header, base) {
+  const links = {};
+  if (!header) return links;
+
+  // <url> followed by any number of `; name` / `; name=token` / `; name="quoted, value"` params
+  const linkRe = /<([^>]*)>((?:\s*;\s*[^=;,<\s]+(?:\s*=\s*(?:"[^"]*"|[^;,<\s]*))?)*)/g;
+  let match;
+  while ((match = linkRe.exec(header)) !== null) {
+    const rel = /;\s*rel\s*=\s*(?:"([^"]*)"|([^\s;,]+))/i.exec(match[2]);
+    if (!rel) continue;
+
+    let url = match[1].trim();
+    if (base) {
+      try {
+        url = new URL(url, base).href;
+      } catch {
+        // leave it as written
+      }
+    }
+    for (const name of (rel[1] ?? rel[2]).toLowerCase().split(/\s+/).filter(Boolean)) {
+      if (!(name in links)) links[name] = url;
+    }
+  }
+  return links;
+}
+
+function stuckError(message) {
+  return Object.assign(new Error(message), { code: "PAGINATE_STUCK" });
+}
+
+/**
+ * Walks a paginated API page by page. You write `fetchPage(cursor, ctx)` —
+ * it returns `{ items, next }`, where `next` is whatever identifies the next
+ * page (a cursor token, an offset, a URL...) or `undefined`/`null` on the
+ * last page — and you get an async iterator of pages.
+ *
+ * ```js
+ * for await (const { items, index } of paginatePages(async (cursor) => {
+ *   const r = await api.json("/users", { query: { cursor, limit: 100 } });
+ *   return { items: r.data, next: r.next_cursor };
+ * })) { ... }
+ * ```
+ *
+ * A `next` that equals the cursor just used throws `code: "PAGINATE_STUCK"`
+ * instead of looping forever.
+ *
+ * @template T
+ * @param {(cursor: any, ctx: { index: number, signal?: AbortSignal }) => Promise<{ items: T[], next?: any }>} fetchPage
+ * @param {object} [opts]
+ * @param {any} [opts.start] - the cursor for the first page (default undefined)
+ * @param {number} [opts.maxPages=Infinity]
+ * @param {AbortSignal} [opts.signal]
+ * @returns {AsyncGenerator<{ items: T[], index: number, cursor: any }>}
+ */
+async function* paginatePages(fetchPage, opts = {}) {
+  const { start, maxPages = Infinity, signal } = opts;
+  let cursor = start;
+  let index = 0;
+
+  for (;;) {
+    if (index >= maxPages) return;
+    signal?.throwIfAborted?.();
+
+    const result = await fetchPage(cursor, { index, signal });
+    if (!result || !Array.isArray(result.items)) {
+      throw new TypeError("paginate: fetchPage must return { items: array, next?: cursor }");
+    }
+    yield { items: result.items, index, cursor };
+
+    const next = result.next;
+    if (next === undefined || next === null) return;
+    if (next === cursor) {
+      throw stuckError("paginate: `next` equals the cursor just used, so the page didn't advance — stopping to avoid an infinite loop");
+    }
+    cursor = next;
+    index++;
+  }
+}
+
+/**
+ * Like `paginatePages`, but yields the items one by one. `maxItems` stops
+ * early (mid-page) — and since nothing is fetched until you ask for it,
+ * `break`-ing out of the loop never fetches a page you didn't need.
+ *
+ * ```js
+ * for await (const user of paginate(fetchUsersPage, { maxItems: 250 })) { ... }
+ * ```
+ *
+ * @template T
+ * @param {(cursor: any, ctx: { index: number, signal?: AbortSignal }) => Promise<{ items: T[], next?: any }>} fetchPage
+ * @param {object} [opts] - same as paginatePages, plus `maxItems`
+ * @param {number} [opts.maxItems=Infinity]
+ * @returns {AsyncGenerator<T>}
+ */
+async function* paginate(fetchPage, opts = {}) {
+  const { maxItems = Infinity } = opts;
+  if (maxItems <= 0) return;
+
+  let count = 0;
+  for await (const page of paginatePages(fetchPage, opts)) {
+    for (const item of page.items) {
+      yield item;
+      if (++count >= maxItems) return;
+    }
+  }
+}
+
+function defaultGetItems(data) {
+  if (Array.isArray(data)) return data;
+  for (const key of ["items", "data", "results", "records"]) {
+    if (data && Array.isArray(data[key])) return data[key];
+  }
+  throw new TypeError("paginateLinks: couldn't find the items array in the response — pass `getItems(data)`");
+}
+
+/**
+ * Follows `Link: <...>; rel="next"` headers (GitHub-style) and yields the
+ * items of every page. `request` is anything fetch-like — a `createFetch`
+ * instance or plain `fetch` (a non-2xx response throws `HTTPError`).
+ *
+ * ```js
+ * for await (const repo of paginateLinks(api, "/orgs/acme/repos?per_page=100")) { ... }
+ * ```
+ *
+ * @param {(url: string, init?: object) => Promise<Response>} request
+ * @param {string} url - the first page
+ * @param {object} [opts]
+ * @param {(data: any, response: Response) => any[]} [opts.getItems] - default: the body if it's an array, else its `items` / `data` / `results` / `records`
+ * @param {number} [opts.maxPages=Infinity]
+ * @param {number} [opts.maxItems=Infinity]
+ * @param {AbortSignal} [opts.signal]
+ * @param {object} [opts.init] - extra options passed to every `request`
+ */
+async function* paginateLinks(request, url, opts = {}) {
+  const { getItems = defaultGetItems, init = {}, signal } = opts;
+
+  async function fetchPage(pageUrl) {
+    const res = await request(pageUrl, { ...init, signal });
+    if (!res.ok) throw new HTTPError(res, { method: init.method || "GET", url: pageUrl });
+    const items = getItems(await res.json(), res);
+    const next = parseLinkHeader(res.headers.get("link"), pageUrl).next;
+    return { items, next };
+  }
+
+  yield* paginate(fetchPage, { ...opts, start: url });
+}
+
+module.exports = { paginate, paginatePages, paginateLinks, parseLinkHeader };
+
 },
 "./parseJsonSafe": function (module, exports, require) {
 /**
@@ -1677,6 +1921,206 @@ function retryAfterAware(fn, opts = {}) {
 
 module.exports = { retryAfterAware, parseRetryAfter, getWaitMsFromHeaders };
 
+},
+"./streamSSE": function (module, exports, require) {
+const { readSSE } = require("./streams");
+const { HTTPError } = require("./HTTPError");
+const { normalizeError } = require("./normalizeError");
+const { getWaitMsFromHeaders, discardBody } = require("./_http");
+const { abortableSleep } = require("./_sleep");
+
+function isRetryableStatus(status) {
+  return status === 408 || status === 425 || status === 429 || status >= 500;
+}
+
+function isRetryableError(err) {
+  const n = normalizeError(err);
+  return !n.isAborted && (n.isNetworkError || n.isTimeout);
+}
+
+async function resolveHeaders(source) {
+  const value = typeof source === "function" ? await source() : source;
+  return new Headers(value || undefined);
+}
+
+/**
+ * Connects to a Server-Sent Events endpoint and yields its events — and when
+ * the connection drops (or the server closes it), reconnects by itself,
+ * sending `Last-Event-ID` so the server can resume where you left off. This is
+ * what the browser's `EventSource` does, but with `fetch`: custom headers,
+ * POST bodies, AbortSignal, your own retry/backoff rules.
+ *
+ * ```js
+ * for await (const { data } of streamSSE("https://api.example.com/events", { signal })) {
+ *   console.log(data);
+ * }
+ * ```
+ *
+ * Reconnect behavior:
+ *  - `Last-Event-ID` is sent on every reconnect (the last `id:` seen).
+ *  - A server `retry: <ms>` field sets the base reconnect delay, like EventSource.
+ *  - Waits back off exponentially (capped at `maxDelayMs`) while connecting keeps
+ *    failing; 429/503 `Retry-After` is honored. A connection that delivered at
+ *    least one event resets the failure count (only no-progress attempts count
+ *    toward `retries`).
+ *  - HTTP 204 means "stop reconnecting" (iteration ends). A non-retryable
+ *    status (e.g. 401/404) or a wrong content-type throws immediately.
+ *  - Only GET reconnects by default. For other methods (e.g. a POST that starts
+ *    an LLM stream) re-sending the request could repeat side effects, so
+ *    nothing is retried unless you pass `reconnect: true`.
+ *  - Breaking out of the loop, or aborting `signal`, closes the connection.
+ *
+ * @param {string} url
+ * @param {object} [opts]
+ * @param {typeof fetch} [opts.fetch=globalThis.fetch] - e.g. a createFetch() instance
+ * @param {string} [opts.method="GET"]
+ * @param {HeadersInit | (() => HeadersInit | Promise<HeadersInit>)} [opts.headers] -
+ *   a function is called before every (re)connect — handy for refreshing a token
+ * @param {BodyInit} [opts.body]
+ * @param {AbortSignal} [opts.signal]
+ * @param {string} [opts.lastEventId] - resume from here on the very first connect
+ * @param {boolean} [opts.reconnect] - default: true for GET/HEAD, false otherwise
+ * @param {number} [opts.retries=10] - consecutive failed connections before giving up (Infinity = never)
+ * @param {number} [opts.reconnectDelayMs=1000] - base delay (the server's `retry:` overrides it)
+ * @param {number} [opts.maxDelayMs=30000]
+ * @param {(event: object) => boolean} [opts.stopWhen] - end the stream (no reconnect) after this event, e.g. `e => e.data === "[DONE]"`
+ * @param {(info: {connections: number, lastEventId: string}) => void} [opts.onOpen]
+ * @param {(info: {reason: "error"|"end"|"status", error?: any, status?: number, attempt: number, delayMs: number, lastEventId: string}) => void} [opts.onReconnect]
+ * @returns {AsyncGenerator<{event: string, data: string, id: string, retry: number|undefined}>}
+ */
+async function* streamSSE(url, opts = {}) {
+  const {
+    fetch: fetchFn = globalThis.fetch,
+    method = "GET",
+    headers,
+    body,
+    signal,
+    reconnect = method === "GET" || method === "HEAD",
+    retries = 10,
+    reconnectDelayMs = 1000,
+    maxDelayMs = 30000,
+    stopWhen,
+    onOpen,
+    onReconnect,
+  } = opts;
+
+  let lastId = opts.lastEventId || "";
+  let baseDelay = reconnectDelayMs;
+  let failures = 0; // consecutive connections that delivered nothing
+  let connections = 0;
+
+  for (;;) {
+    signal?.throwIfAborted?.();
+    let failure = null; // { reason, error?, status?, response? }
+    let res = null;
+
+    // ---- connect -----------------------------------------------------------
+    try {
+      const h = await resolveHeaders(headers);
+      if (!h.has("accept")) h.set("accept", "text/event-stream");
+      if (!h.has("cache-control")) h.set("cache-control", "no-cache");
+      if (lastId) h.set("last-event-id", lastId);
+      res = await fetchFn(url, { method, headers: h, body, signal });
+    } catch (err) {
+      if (signal?.aborted) throw signal.reason ?? err;
+      if (err instanceof HTTPError) {
+        if (!isRetryableStatus(err.status)) throw err;
+        failure = { reason: "status", status: err.status, response: err.response, error: err };
+      } else if (isRetryableError(err)) {
+        failure = { reason: "error", error: err };
+      } else {
+        throw err;
+      }
+    }
+
+    // ---- inspect the response ---------------------------------------------
+    if (!failure) {
+      if (res.status === 204) {
+        await discardBody(res);
+        return; // the server's way of saying "don't reconnect"
+      }
+      if (!res.ok) {
+        if (!isRetryableStatus(res.status)) {
+          await discardBody(res);
+          throw new HTTPError(res, { method, url });
+        }
+        failure = { reason: "status", status: res.status, response: res };
+      } else {
+        const type = res.headers.get("content-type") || "";
+        if (!/text\/event-stream/i.test(type)) {
+          await discardBody(res);
+          throw Object.assign(new Error(`streamSSE: expected "text/event-stream" but the server sent "${type || "no content-type"}"`), {
+            code: "SSE_BAD_CONTENT_TYPE",
+          });
+        }
+      }
+    }
+
+    // ---- stream ------------------------------------------------------------
+    let gotEvent = false;
+    if (!failure) {
+      connections++;
+      if (onOpen) onOpen({ connections, lastEventId: lastId });
+      try {
+        for await (const event of readSSE(res, { signal })) {
+          gotEvent = true;
+          lastId = event.id;
+          if (event.retry !== undefined) baseDelay = event.retry;
+          yield event;
+          if (stopWhen && stopWhen(event)) return;
+        }
+        failure = { reason: "end" };
+      } catch (err) {
+        if (signal?.aborted) throw signal.reason ?? err;
+        if (!isRetryableError(err)) throw err;
+        failure = { reason: "error", error: err };
+      }
+    }
+
+    // ---- decide whether and when to reconnect ------------------------------
+    if (!reconnect) {
+      if (failure.reason === "end") return;
+      if (failure.reason === "status") {
+        throw failure.error || new HTTPError(failure.response, { method, url });
+      }
+      throw failure.error;
+    }
+
+    // A connection that delivered events made progress: reconnect promptly and
+    // start the failure count over. Only no-progress attempts count toward `retries`.
+    failures = gotEvent ? 0 : failures + 1;
+    if (failures > retries) {
+      if (failure.reason === "status") throw failure.error || new HTTPError(failure.response, { method, url });
+      if (failure.reason === "error") throw failure.error;
+      throw Object.assign(new Error(`streamSSE: the server keeps closing the stream without sending events — gave up after ${retries} retries`), {
+        code: "SSE_GAVE_UP",
+      });
+    }
+
+    let delayMs = Math.min(baseDelay * 2 ** Math.max(failures - 1, 0), maxDelayMs);
+    if (failure.reason === "status" && failure.response) {
+      const headerMs = getWaitMsFromHeaders(failure.response.headers);
+      if (headerMs !== null) delayMs = Math.min(headerMs, 60000);
+      await discardBody(failure.response);
+    }
+    delayMs += Math.random() * delayMs * 0.1; // a little jitter so clients don't reconnect in lockstep
+
+    if (onReconnect) {
+      onReconnect({
+        reason: failure.reason,
+        error: failure.error,
+        status: failure.status,
+        attempt: Math.max(failures, 1),
+        delayMs,
+        lastEventId: lastId,
+      });
+    }
+    await abortableSleep(delayMs, signal);
+  }
+}
+
+module.exports = { streamSSE };
+
 },
 "./streams": function (module, exports, require) {
 const encoder = new TextEncoder();
@@ -2290,12 +2734,17 @@ export const {
   downloadResumable,
   getWaitMsFromHeaders,
   normalizeError,
+  paginate,
+  paginateLinks,
+  paginatePages,
   parseJsonSafe,
+  parseLinkHeader,
   parseRetryAfter,
   readLines,
   readNDJSON,
   readSSE,
   retryAfterAware,
+  streamSSE,
   withDedupe,
   withProgress,
   withRetry,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boy-offi9-inc/reqkit",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "Small, composable HTTP helpers — retry with backoff and Retry-After, timeouts, normalized errors, rate limiter, circuit breaker, response cache, resumable downloads, SSE/NDJSON streams, mock fetch, and an opt-in createFetch. Not a client; works alongside fetch/axios/anything.",
   "main": "./src/index.js",
   "types": "./types/index.d.ts",
@@ -57,7 +57,11 @@
     "deno",
     "bun",
     "cloudflare-workers",
-    "esm"
+    "esm",
+    "pagination",
+    "auth",
+    "token-refresh",
+    "link-header"
   ],
   "author": "Boy Offi9",
   "license": "MIT",