@ayepi/rate 0.3.0 → 0.4.0

package/README.md

@@ -97,6 +97,16 @@ const doer = rateLimitedDoer({ limit: 100, window: 60_000, algorithm: 'token-buc
 const w = createWork({ work: [sendEmail] as const, doer })   // ≤ 100 sends/min
 ```
 
+**Two-tier limiting.** Add a `groupLimit` (a default, or per task) and a task must clear both
+the global limit **and** its `group`'s limit — e.g. a global API cap plus a per-user/per-model
+cap. A hot group is skipped (not blocking others), groups needn't be known up front (buckets
+auto-evict when idle), and a shared `store`/`groupStore` (Redis) enforces both tiers across a fleet:
+
+```ts
+rateLimitedDoer({ limit: 1000, window: 60_000, groupLimit: 50 })
+  .do(() => callModel(req), { group: `${userId}:${modelId}`, groupLimit: 200 }) // per-request override
+```
+
 ## Algorithms
 
 - `fixed-window` (default) — simple counter per window.

package/ayepi-rate-stores-doer.md

@@ -49,15 +49,26 @@ Memcached, …); only `consume` is required.
 ### `memoryStore` (default, bundled)
 
 ```ts
-function memoryStore(): RateLimitStore
+function memoryStore(opts?: MemoryStoreOptions): RateLimitStore
+
+interface MemoryStoreOptions {
+  /** Sweep expired/idle entries once every this many `consume` calls (default 1000). */
+  readonly sweepEvery?: number;
+  /** Drop an idle token-bucket after this long with no activity (ms, default 10 min). */
+  readonly idleMs?: number;
+}
 ```
 
 An in-process store implementing all three algorithms, zero dependencies. It is the
-default when no `store` is passed. Expired counters and idle token buckets are swept
-lazily (amortized: a sweep runs once per ~1000 `consume` calls; idle buckets are dropped
-after ~10 minutes of inactivity since they refill to full anyway). **Single process only**
-— two server instances each get their own independent budget. Use the Redis store to share
-a limit across pods.
+default when no `store` is passed. **Single process only** — two server instances each get
+their own independent budget; use the Redis store to share a limit across pods.
+
+**Bounded memory for dynamic keys.** Entries are swept lazily, so an unbounded, not-known-in-advance
+key space — e.g. per-user/per-model rate-limit **groups** (see `rateLimitedDoer` below) — doesn't grow
+without bound: a windowed counter is dropped once its window has elapsed, and an idle `token-bucket`
+after `idleMs` (default 10 min; they refill to full anyway). Active keys live long enough to enforce
+the limit; idle ones go away. For high churn, lower `sweepEvery` (tighter memory, a little more scan
+cost) and/or `idleMs`: `memoryStore({ sweepEvery: 200, idleMs: 30_000 })`.
 
 ### `redisStore` — `@ayepi/rate/redis`
 
@@ -111,11 +122,28 @@ distributed store (verified by the integration test). `reset(key)` issues a `DEL
 ## `rateLimitedDoer` — gating task start rate
 
 ```ts
-function rateLimitedDoer(opts: RateLimitedDoerOptions): Doer
+function rateLimitedDoer(opts: RateLimitedDoerOptions): RateLimitedDoer
 
 interface RateLimitedDoerOptions extends LimiterOptions {
-  /** Limit key — a single shared bucket by default (`'doer'`), or derived per task. */
+  /** Limit key for the GLOBAL bucket — a single shared bucket by default (`'doer'`), or derived per task. */
   readonly key?: string | ((opts: DoerTaskOptions) => string);
+
+  // ── two-tier: an optional per-`group` limit on top of the global one ──
+  /** Default group limit (a second gate bucketed by a task's `group`). Per-task `groupLimit` overrides it; omit for global-only. */
+  readonly groupLimit?: number;
+  /** Derive a task's group bucket key (default `opts.group`). Return `undefined` to skip the group gate for a task. */
+  readonly groupKey?: (opts: DoerTaskOptions) => string | undefined;
+  /** Store for the group buckets (default: the global `store`). A distributed store limits groups across a fleet. */
+  readonly groupStore?: RateLimitStore;
+
+  // ── sustained-backlog watch (observational) ──
+  /** Fire while the pending queue stays continuously non-empty past `backlogAfterMs`. Must not throw. */
+  readonly onBacklog?: (info: RateLimitedBacklogInfo) => void;
+  /** How long the queue must stay non-empty before `onBacklog` first fires (ms). */
+  readonly backlogAfterMs?: number;
+  /** Re-fire `onBacklog` every this many ms while still backed up (omit to fire once). */
+  readonly backlogEveryMs?: number;
+
   /** Floor on the re-check delay for deferred tasks (ms, default 50). */
   readonly retryFloor?: number;
   /** Clock injection (default `Date.now`). */
@@ -123,6 +151,23 @@ interface RateLimitedDoerOptions extends LimiterOptions {
   /** The doer that actually runs admitted tasks (default `unlimitedDoer`). Compose policies. */
   readonly doer?: Doer;
 }
+
+/** Per-task options — DoerTaskOptions (`group`/`priority`/`createdAt`) plus a per-request group-limit override. */
+interface RateTaskOptions extends DoerTaskOptions {
+  readonly groupLimit?: number;      // overrides the configured default; enables the group gate for this task
+  readonly groupWindow?: number;     // default: the global `window`
+  readonly groupAlgorithm?: Algorithm; // default: the global `algorithm`
+}
+
+/** A Doer whose `do` accepts RateTaskOptions (assignable to Doer). */
+interface RateLimitedDoer extends Doer {
+  do(task: () => Promise<void>, opts?: RateTaskOptions): void;
+}
+
+interface RateLimitedBacklogInfo {
+  readonly pending: number;         // tasks waiting on the rate limit
+  readonly nonEmptyForMs: number;   // how long the queue has been continuously non-empty
+}
 ```
 
 (`RateLimitedDoerOptions` extends `LimiterOptions` — see `ayepi-rate.md` for
@@ -135,8 +180,50 @@ A `Doer` (from `@ayepi/core/doer`) that caps the **start rate** of tasks using t
 policy. Excess tasks wait, **oldest-first**; with a distributed store this rate-limits
 **across a fleet**.
 
-A `Doer` exposes `available()`, `do(task, opts?)`, and `done()` — see the doer section of
-the core docs. `rateLimitedDoer`'s `available()` is `min(limit − pending, inner.available())`.
+A `Doer` exposes `available()`, `do(task, opts?)`, and `done()` — see the doer section
+below. `rateLimitedDoer`'s `available()` is `min(limit − pending, inner.available())`.
+
+### Two-tier: a global limit + a per-group limit
+
+Set a `groupLimit` (a configured default, or per task via `RateTaskOptions`) and a task must
+clear **both** the global limit and its `group`'s limit to be admitted — e.g. a global API cap
+plus a per-user / per-model / per-model-family cap. `group` and `groupLimit` are read at
+**admission time**, so groups need not be known up front.
+
+```ts
+import { rateLimitedDoer } from '@ayepi/rate'
+
+const doer = rateLimitedDoer({
+  limit: 1000, window: 60_000, algorithm: 'token-bucket', // global cap
+  groupLimit: 50,                                          // default per-group cap
+})
+
+// per-request override — the limit known at request time (e.g. a plan tier):
+doer.do(() => callModel(req), { group: `${userId}:${modelId}`, groupLimit: tier === 'pro' ? 200 : 20 })
+```
+
+Semantics:
+
+- **Both gates, group first.** The group gate is checked before the global. A denied check
+  consumes nothing (the group rule forces `countRejected: false`), so a task whose group is at
+  its limit is **skipped**, not admitted — and no global token is spent on it.
+- **Head-of-line avoidance.** A skipped (group-capped) task is left pending while **other
+  groups' tasks proceed**, so a hot user/model can't block everyone. Within a group, oldest-first.
+- **Global exhaustion stops the round.** When the shared global bucket is out, nothing more is
+  admitted this pass. Edge case: a task that cleared its group but then hits an exhausted global
+  had its group token spent (bounded, self-corrects on retry) — only when global is the bottleneck.
+- **Global-only when no group.** A task with no `group`, or no group limit in scope, passes straight to the global gate.
+- **Fleet-shared, bounded memory.** Point `groupStore` (and `store`) at `@ayepi/rate/redis` to
+  share both tiers across pods. The dynamic group key space is auto-evicted (memory sweep / Redis
+  TTL), so it doesn't grow without bound — see `memoryStore` above.
+
+### Sustained-backlog watch
+
+`onBacklog` fires with `{ pending, nonEmptyForMs }` once the pending queue has stayed
+continuously non-empty past `backlogAfterMs` (repeating every `backlogEveryMs`, or once if
+omitted) — a signal that the rate limit can't admit fast enough. Purely observational
+(alerting/autoscaling), a throw is ignored, and the timer is `unref`'d and cleared when the
+queue drains.
 
 ### Example — cap an `@ayepi/work` engine
 
@@ -177,6 +264,33 @@ With a static `key` string (or the default `'doer'`) all tasks share one bucket.
 
 ---
 
+## The core `Doer` (`@ayepi/core/doer`)
+
+`rateLimitedDoer` builds on the runtime-agnostic `Doer` primitive:
+
+```ts
+interface Doer {
+  available(): number;                                   // how many tasks it will accept right now
+  do(task: () => Promise<void>, opts?: DoerTaskOptions): void; // fire-and-forget (swallows errors)
+  done(): Promise<void>;                                 // resolves when all accepted tasks settle
+}
+interface DoerTaskOptions { group?: string; priority?: number; createdAt?: number }
+```
+
+Bundled policies: **`unlimitedDoer`** (no cap), **`balancedDoer`** (cap N; fair-share across
+`group`s, then priority, then age), **`priorityDoer`** (cap N; highest priority first), **`ageDoer`**
+(cap N; oldest first). The three bounded doers share `BoundedDoerOptions` (`max`, `buffer?`, `now?`).
+
+- **`doWith(doer, fn, opts?)`** — submit a task and get its **result** back (`Promise<T>`), the
+  result-returning counterpart to the fire-and-forget `do` (which returns void and swallows the
+  error). Use it when a doer governs request-scoped work: `const user = await doWith(apiDoer, () => fetchUser(id), { group: tenantId })`.
+- **Sustained-backlog watch** — the bounded doers accept `onBacklog: (info) => void` with
+  `backlogAfterMs` / `backlogEveryMs`, firing `{ pending, running, nonEmptyForMs }` once the pending
+  queue stays continuously non-empty past the threshold (the doer analogue of `rateLimitedDoer`'s
+  `onBacklog`). Purely observational; one `unref`'d timer, only while backed up.
+
+---
+
 ## How it works under the hood
 
 ### The three algorithms
@@ -240,18 +354,21 @@ limiter's dependency middleware run first and their context is available — see
 
 ### The doer abstraction
 
-`rateLimitedDoer` keeps a `pending` queue. On each `drain`:
-
-- if the inner doer has no capacity (`inner.available() <= 0`), it arms a short re-check
-  timer and stops;
-- otherwise it picks the **oldest** pending task (by `createdAt`, then submission `seq`),
-  calls `limiter.check(keyOf(task), now())`;
-- if denied, it arms a timer for `max(retryFloor, retryAfter)` and stops (drains again when
-  the limiter would allow);
-- if admitted, it removes the task and calls `inner.do(task.run, task.opts)`.
-
-`done()` resolves once `pending` is empty **and** the inner doer's `done()` resolves. The
-re-check timer is `unref`'d, so it won't keep a process alive on its own.
+`rateLimitedDoer` keeps a `pending` queue. Each `drain` scans it **oldest-first** and admits
+every task that clears its gates, skipping (not blocking on) group-capped ones:
+
+- if the inner doer has no capacity (`inner.available() <= 0`), it records a short re-check and stops;
+- for the oldest not-yet-skipped task, it checks the **group gate first** (when a `group` + group
+  limit apply): a denied group check consumes nothing, so the task is **skipped** (left pending) and
+  the scan continues to other groups — recording that group's `retryAfter`;
+- then the **global gate** (`limiter.check(key, now)`): if denied, the shared bucket is out, so no
+  further task can be admitted this round and the scan stops (recording the global `retryAfter`);
+- if both allow, it removes the task and calls `inner.do(task.run, task.opts)`.
+
+After the scan, if tasks remain it arms **one** timer for the **soonest** recorded re-check
+(`max(retryFloor, soonest)`), so it drains again exactly when the earliest gate would allow. A
+store error on either gate is reported to `onError` and backs the round off by `retryFloor`. `done()`
+resolves once `pending` is empty **and** the inner doer's `done()` resolves. All timers are `unref`'d.
 
 ---
 

package/ayepi-rate.md

@@ -96,8 +96,8 @@ Everything below is exported. `@internal` symbols are intentionally omitted.
 | `limiter` | function | Standalone limiter primitive (`check`/`reset`/`rule`). |
 | `rateLimitResponse` | function | Build a 429 `Response` from limiter info. |
 | `rateLimitHeaders` | function | Compute the `RateLimit-*` header map from limiter info. |
-| `memoryStore` | function | The bundled in-process store (all three algorithms). |
-| `rateLimitedDoer` | function | A `Doer` that caps task **start rate**. |
+| `memoryStore` | function | The bundled in-process store (all three algorithms); `memoryStore(opts?)` — tunable eviction (`sweepEvery`/`idleMs`). |
+| `rateLimitedDoer` | function | A `Doer` that caps task **start rate** — global + optional **per-group** limit, plus a backlog watch. |
 | `Algorithm` | type | `'fixed-window' \| 'sliding-window' \| 'token-bucket'`. |
 | `RateLimitInfo` | interface | Limiter state exposed to handlers + headers. |
 | `RateLimitResult` | interface | `RateLimitInfo` + `allowed`. |
@@ -107,7 +107,11 @@ Everything below is exported. `@internal` symbols are intentionally omitted.
 | `LimiterOptions` | interface | Base config (`limit`/`window`/`algorithm`/`store`/`prefix`). |
 | `RateLimitResponseOptions` | interface | `status`/`message`/`headers` for `rateLimitResponse`. |
 | `RateLimitDefOptions` | interface | Options for the `rateLimit` def (`name`/`requires`). |
-| `RateLimitedDoerOptions` | interface | Options for `rateLimitedDoer`. |
+| `RateLimitedDoerOptions` | interface | Options for `rateLimitedDoer` (global + group limits, backlog watch). |
+| `RateLimitedDoer` | interface | The doer `rateLimitedDoer` returns (a `Doer` whose `do` takes `RateTaskOptions`). |
+| `RateTaskOptions` | interface | Per-task options — `DoerTaskOptions` + `groupLimit`/`groupWindow`/`groupAlgorithm`. |
+| `RateLimitedBacklogInfo` | interface | `{ pending, nonEmptyForMs }` passed to the doer's `onBacklog`. |
+| `MemoryStoreOptions` | interface | `{ sweepEvery?, idleMs? }` — tune `memoryStore` eviction. |
 | `Limiter` | interface | The object `limiter()` returns. |
 
 ### Server subpath `@ayepi/rate/server`
@@ -222,7 +226,7 @@ interface RateLimitServerOptions<M extends AnyMiddleware> {
 > `onError` to observe the failure (it fires either way). `rateLimitedDoer` takes an `onError`
 > too: a store error there is reported and admission retried — it never strands pending tasks.
 
-> The standalone `limiter()` primitive (and `rateLimitedDoer`) still takes a **static**
+> The standalone `limiter()` primitive (and `rateLimitedDoer`) takes a **static**
 > `LimiterOptions` (`limit`/`window`/`algorithm`/`store`/`prefix`/`countRejected`) — dynamic
 > resolution is a middleware feature, since only the request path carries an `io`.
 

package/dist/index.cjs

@@ -151,18 +151,78 @@ const DOER_RETRY_FLOOR = 50;
 * const doer = rateLimitedDoer({ limit: 100, window: 60_000, algorithm: 'token-bucket' })
 * const w = createWork({ work: [sendEmail] as const, doer })
 * ```
+*
+* **Two-tier limiting.** Set a `groupLimit` (default, or per task via {@link RateTaskOptions}) and a
+* task must clear both the global limit **and** its `group`'s limit — e.g. a global API cap plus a
+* per-user / per-model cap. The group gate is checked first (a denied group check consumes nothing),
+* so a task whose group is at its limit is **skipped** rather than blocking other groups, and the
+* shared global bucket isn't spent on it. `group`/`groupLimit` are read at admission time.
 */
 function rateLimitedDoer(opts) {
-	const lim = limiter(opts);
+	const store = opts.store ?? memoryStore();
+	const lim = limiter({
+		...opts,
+		store
+	});
 	const inner = opts.doer ?? (0, _ayepi_core_doer.unlimitedDoer)();
 	const now = opts.now ?? Date.now;
 	const floor = opts.retryFloor ?? DOER_RETRY_FLOOR;
 	const keyOf = (o) => typeof opts.key === "function" ? opts.key(o ?? {}) : opts.key ?? "doer";
+	const groupStore = opts.groupStore ?? store;
+	const groupPrefix = (opts.prefix ?? "rl:") + "grp:";
+	const groupKeyOf = (o) => opts.groupKey ? opts.groupKey(o ?? {}) : o?.group;
+	const groupRuleFor = (o) => {
+		const limit = o?.groupLimit ?? opts.groupLimit;
+		if (limit === void 0) return;
+		return {
+			limit,
+			window: o?.groupWindow ?? opts.window,
+			algorithm: o?.groupAlgorithm ?? opts.algorithm ?? "fixed-window",
+			countRejected: false
+		};
+	};
+	const report = (err) => {
+		try {
+			opts.onError?.(err);
+		} catch {}
+	};
 	const pending = [];
 	const idle = [];
 	let seq = 0;
 	let draining = false;
 	let timer = null;
+	const backlogWatch = opts.onBacklog !== void 0 && opts.backlogAfterMs !== void 0;
+	let nonEmptySince = null;
+	let backlogTimer = null;
+	const armBacklog = (ms) => {
+		backlogTimer = setTimeout(fireBacklog, ms);
+		backlogTimer.unref?.();
+	};
+	function fireBacklog() {
+		try {
+			opts.onBacklog({
+				pending: pending.length,
+				nonEmptyForMs: now() - nonEmptySince
+			});
+		} catch {}
+		if (opts.backlogEveryMs !== void 0) armBacklog(opts.backlogEveryMs);
+		else backlogTimer = null;
+	}
+	const syncBacklog = () => {
+		if (!backlogWatch) return;
+		if (pending.length > 0) {
+			if (nonEmptySince === null) {
+				nonEmptySince = now();
+				armBacklog(opts.backlogAfterMs);
+			}
+		} else {
+			nonEmptySince = null;
+			if (backlogTimer) {
+				clearTimeout(backlogTimer);
+				backlogTimer = null;
+			}
+		}
+	};
 	const arm = (ms) => {
 		if (timer) return;
 		timer = setTimeout(() => {
@@ -171,40 +231,67 @@ function rateLimitedDoer(opts) {
 		}, Math.max(floor, ms));
 		timer.unref?.();
 	};
+	/**
+	* The oldest pending task not skipped this round (a fresh scan, so tasks queued mid-drain are
+	* seen). `pending` is in insertion (seq) order, so equal `createdAt` ties keep the earliest.
+	*/
+	const nextPending = (skipped) => {
+		let best;
+		for (const t of pending) {
+			if (skipped.has(t)) continue;
+			if (!best || t.createdAt < best.createdAt) best = t;
+		}
+		return best;
+	};
 	const drain = async () => {
 		if (draining) return;
 		draining = true;
 		try {
-			while (pending.length > 0) {
+			const skipped = /* @__PURE__ */ new Set();
+			let soonest = Infinity;
+			for (;;) {
 				if (inner.available() <= 0) {
-					arm(floor);
+					soonest = Math.min(soonest, floor);
 					break;
 				}
-				let best = 0;
-				for (let i = 1; i < pending.length; i++) {
-					const a = pending[i];
-					const b = pending[best];
-					if (a.createdAt < b.createdAt || a.createdAt === b.createdAt && a.seq < b.seq) best = i;
+				const task = nextPending(skipped);
+				if (!task) break;
+				const o = task.opts;
+				const groupRule = groupRuleFor(o);
+				const gk = groupKeyOf(o);
+				if (groupRule && gk !== void 0) {
+					let gr;
+					try {
+						gr = await groupStore.consume(groupPrefix + gk, groupRule, now());
+					} catch (err) {
+						report(err);
+						soonest = Math.min(soonest, floor);
+						break;
+					}
+					if (!gr.allowed) {
+						soonest = Math.min(soonest, gr.retryAfter);
+						skipped.add(task);
+						continue;
+					}
 				}
-				const task = pending[best];
 				let res;
 				try {
-					res = await lim.check(keyOf(task.opts), now());
+					res = await lim.check(keyOf(o), now());
 				} catch (err) {
-					try {
-						opts.onError?.(err);
-					} catch {}
-					arm(floor);
+					report(err);
+					soonest = Math.min(soonest, floor);
 					break;
 				}
 				if (!res.allowed) {
-					arm(res.retryAfter);
+					soonest = Math.min(soonest, res.retryAfter);
 					break;
 				}
-				pending.splice(best, 1);
+				pending.splice(pending.indexOf(task), 1);
 				inner.do(task.run, task.opts);
 			}
 			if (pending.length === 0) for (const r of idle.splice(0)) r();
+			else arm(soonest);
+			syncBacklog();
 		} finally {
 			draining = false;
 		}
@@ -218,6 +305,7 @@ function rateLimitedDoer(opts) {
 				createdAt: o?.createdAt ?? now(),
 				seq: seq++
 			});
+			syncBacklog();
 			drain();
 		},
 		done: () => pending.length === 0 ? inner.done() : new Promise((r) => idle.push(() => void inner.done().then(r)))
@@ -298,19 +386,26 @@ function tokenBucket(buckets, key, rule, now) {
 	};
 }
 /**
-* An in-process {@link RateLimitStore} implementing all three algorithms. The
-* default store — fine for a single instance; use a distributed store (e.g.
-* `@ayepi/rate/redis`) to share limits across pods. Expired entries are swept
-* lazily.
+* An in-process {@link RateLimitStore} implementing all three algorithms. The default store — fine
+* for a single instance; use a distributed store (e.g. `@ayepi/rate/redis`, which sets key TTLs) to
+* share limits across pods.
+*
+* **Bounded memory for dynamic keys.** Entries are swept lazily so an unbounded, not-known-in-advance
+* key space (per-user/per-model rate-limit **groups**) doesn't grow without bound: a windowed counter
+* is dropped once its window elapses, and an idle token-bucket after {@link MemoryStoreOptions.idleMs}.
+* Active keys live long enough to enforce the limit; idle ones go away. Tune {@link MemoryStoreOptions}
+* for high churn.
 */
-function memoryStore() {
+function memoryStore(opts = {}) {
+	const sweepEvery = opts.sweepEvery ?? SWEEP_EVERY;
+	const idleMs = opts.idleMs ?? BUCKET_IDLE_MS;
 	const counters = /* @__PURE__ */ new Map();
 	const buckets = /* @__PURE__ */ new Map();
 	let ops = 0;
 	const sweep = (now) => {
-		if (++ops % SWEEP_EVERY !== 0) return;
+		if (++ops % sweepEvery !== 0) return;
 		for (const [k, e] of counters) if (e.reset <= now) counters.delete(k);
-		for (const [k, b] of buckets) if (now - b.ts > BUCKET_IDLE_MS) buckets.delete(k);
+		for (const [k, b] of buckets) if (now - b.ts > idleMs) buckets.delete(k);
 	};
 	return {
 		consume(key, rule, now) {

package/dist/index.d.cts

@@ -148,16 +148,68 @@ declare function rateLimit<const R extends readonly AnyMiddleware[] = readonly [
 type RateLimitDef<R extends readonly AnyMiddleware[] = readonly []> = MiddlewareDef<{
   ratelimit: RateLimitInfo;
 }, R>;
+/** Details of a sustained backlog, passed to {@link RateLimitedDoerOptions.onBacklog}. */
+interface RateLimitedBacklogInfo {
+  /** Tasks waiting on the rate limit to admit them (the queue depth). */
+  readonly pending: number;
+  /** How long the queue has been *continuously* non-empty (ms). */
+  readonly nonEmptyForMs: number;
+}
+/**
+ * Per-task options for a {@link rateLimitedDoer} — {@link DoerTaskOptions} (`group`/`priority`/
+ * `createdAt`) plus a per-request **group limit** override. When a `group` and a group limit are
+ * present, the task must clear both the global limit **and** its group's limit to be admitted.
+ */
+interface RateTaskOptions extends DoerTaskOptions {
+  /** Per-task group rate limit (a second gate, bucketed by `group`). Overrides {@link RateLimitedDoerOptions.groupLimit}. */
+  readonly groupLimit?: number;
+  /** Window for the group limit (ms; default: the global `window`). */
+  readonly groupWindow?: number;
+  /** Algorithm for the group limit (default: the global `algorithm`). */
+  readonly groupAlgorithm?: Algorithm;
+}
+/** A {@link Doer} whose `do` accepts {@link RateTaskOptions} — i.e. a per-task group limit. */
+interface RateLimitedDoer extends Doer {
+  /** Accept a task; `opts` may carry a per-task `group` + `groupLimit` (a second, per-group gate). */
+  do(task: () => Promise<void>, opts?: RateTaskOptions): void;
+}
 /** Options for {@link rateLimitedDoer} — a {@link LimiterOptions} plus doer-specific knobs. */
 interface RateLimitedDoerOptions extends LimiterOptions {
-  /** Limit key — a single shared bucket by default (`'doer'`), or derived per task. */
+  /** Limit key for the **global** bucket — a single shared bucket by default (`'doer'`), or derived per task. */
   readonly key?: string | ((opts: DoerTaskOptions) => string);
+  /**
+   * Default **group** limit — a second gate bucketed by a task's `group`, on top of the global limit.
+   * A per-task {@link RateTaskOptions.groupLimit} overrides it. Omit for no group gate (global only).
+   * The group window/algorithm default to the global ones unless overridden per task.
+   */
+  readonly groupLimit?: number;
+  /** Derive a task's group bucket key (default: `opts.group`). Return `undefined` to skip the group gate for a task. */
+  readonly groupKey?: (opts: DoerTaskOptions) => string | undefined;
+  /**
+   * Store for the group buckets (default: the global `store`). A distributed store limits groups
+   * across a fleet. Groups need not be known up front — bucket state is per-group and **auto-evicted
+   * when idle** (the {@link memoryStore} sweeps expired/idle keys; `@ayepi/rate/redis` sets TTLs), so
+   * an unbounded, dynamic group space (per-user/per-model) won't grow without bound. For high churn,
+   * pass a tuned store, e.g. `groupStore: memoryStore({ sweepEvery: 200, idleMs: 30_000 })`.
+   */
+  readonly groupStore?: RateLimitStore;
   /** Floor on the re-check delay for deferred tasks (ms, default 50). */
   readonly retryFloor?: number;
   /** Clock injection (default `Date.now`). */
   readonly now?: () => number;
   /** The doer that actually runs admitted tasks (default {@link unlimitedDoer}). Compose policies. */
   readonly doer?: Doer;
+  /**
+   * Notified when the pending queue stays **continuously non-empty** past
+   * {@link RateLimitedDoerOptions.backlogAfterMs} — a sustained backlog of tasks the rate limit can't
+   * admit fast enough. Purely observational (for alerting/autoscaling); it must not throw — if it
+   * does, the throw is ignored. Requires `backlogAfterMs`.
+   */
+  readonly onBacklog?: (info: RateLimitedBacklogInfo) => void;
+  /** How long the queue must stay non-empty before {@link RateLimitedDoerOptions.onBacklog} first fires (ms). */
+  readonly backlogAfterMs?: number;
+  /** Re-fire `onBacklog` every this many ms while still backed up. Omit to fire once per backlog episode. */
+  readonly backlogEveryMs?: number;
   /**
    * Observe a store error during admission (e.g. a distributed store hiccup). The drain loop
    * never crashes on it — the task stays pending and admission is retried shortly. Off by
@@ -180,14 +232,40 @@ interface RateLimitedDoerOptions extends LimiterOptions {
  * const doer = rateLimitedDoer({ limit: 100, window: 60_000, algorithm: 'token-bucket' })
  * const w = createWork({ work: [sendEmail] as const, doer })
  * ```
+ *
+ * **Two-tier limiting.** Set a `groupLimit` (default, or per task via {@link RateTaskOptions}) and a
+ * task must clear both the global limit **and** its `group`'s limit — e.g. a global API cap plus a
+ * per-user / per-model cap. The group gate is checked first (a denied group check consumes nothing),
+ * so a task whose group is at its limit is **skipped** rather than blocking other groups, and the
+ * shared global bucket isn't spent on it. `group`/`groupLimit` are read at admission time.
  */
-declare function rateLimitedDoer(opts: RateLimitedDoerOptions): Doer;
+declare function rateLimitedDoer(opts: RateLimitedDoerOptions): RateLimitedDoer;
+/** Options for {@link memoryStore}. */
+interface MemoryStoreOptions {
+  /**
+   * Sweep expired/idle entries once every this many `consume` calls (default 1000). Lower bounds
+   * memory tighter when keys churn — e.g. many short-lived rate-limit **groups** (per-user/per-model)
+   * that aren't known up front — at a little more periodic scan cost.
+   */
+  readonly sweepEvery?: number;
+  /**
+   * Drop an idle **token-bucket** key after this long with no activity (ms, default 10 min).
+   * Windowed (`fixed-window`/`sliding-window`) counters are always dropped once their window has
+   * elapsed, regardless of this.
+   */
+  readonly idleMs?: number;
+}
 /**
- * An in-process {@link RateLimitStore} implementing all three algorithms. The
- * default store — fine for a single instance; use a distributed store (e.g.
- * `@ayepi/rate/redis`) to share limits across pods. Expired entries are swept
- * lazily.
+ * An in-process {@link RateLimitStore} implementing all three algorithms. The default store — fine
+ * for a single instance; use a distributed store (e.g. `@ayepi/rate/redis`, which sets key TTLs) to
+ * share limits across pods.
+ *
+ * **Bounded memory for dynamic keys.** Entries are swept lazily so an unbounded, not-known-in-advance
+ * key space (per-user/per-model rate-limit **groups**) doesn't grow without bound: a windowed counter
+ * is dropped once its window elapses, and an idle token-bucket after {@link MemoryStoreOptions.idleMs}.
+ * Active keys live long enough to enforce the limit; idle ones go away. Tune {@link MemoryStoreOptions}
+ * for high churn.
  */
-declare function memoryStore(): RateLimitStore;
+declare function memoryStore(opts?: MemoryStoreOptions): RateLimitStore;
 //#endregion
-export { Algorithm, DEFAULT_ALGORITHM, DEFAULT_PREFIX, Limiter, LimiterOptions, RateKeyIO, RateLimitDef, RateLimitDefOptions, RateLimitInfo, RateLimitResponseOptions, RateLimitResult, RateLimitRule, RateLimitStore, RateLimitedDoerOptions, limiter, memoryStore, rateLimit, rateLimitHeaders, rateLimitResponse, rateLimitedDoer };
+export { Algorithm, DEFAULT_ALGORITHM, DEFAULT_PREFIX, Limiter, LimiterOptions, MemoryStoreOptions, RateKeyIO, RateLimitDef, RateLimitDefOptions, RateLimitInfo, RateLimitResponseOptions, RateLimitResult, RateLimitRule, RateLimitStore, RateLimitedBacklogInfo, RateLimitedDoer, RateLimitedDoerOptions, RateTaskOptions, limiter, memoryStore, rateLimit, rateLimitHeaders, rateLimitResponse, rateLimitedDoer };

package/dist/index.d.ts

@@ -148,16 +148,68 @@ declare function rateLimit<const R extends readonly AnyMiddleware[] = readonly [
 type RateLimitDef<R extends readonly AnyMiddleware[] = readonly []> = MiddlewareDef<{
   ratelimit: RateLimitInfo;
 }, R>;
+/** Details of a sustained backlog, passed to {@link RateLimitedDoerOptions.onBacklog}. */
+interface RateLimitedBacklogInfo {
+  /** Tasks waiting on the rate limit to admit them (the queue depth). */
+  readonly pending: number;
+  /** How long the queue has been *continuously* non-empty (ms). */
+  readonly nonEmptyForMs: number;
+}
+/**
+ * Per-task options for a {@link rateLimitedDoer} — {@link DoerTaskOptions} (`group`/`priority`/
+ * `createdAt`) plus a per-request **group limit** override. When a `group` and a group limit are
+ * present, the task must clear both the global limit **and** its group's limit to be admitted.
+ */
+interface RateTaskOptions extends DoerTaskOptions {
+  /** Per-task group rate limit (a second gate, bucketed by `group`). Overrides {@link RateLimitedDoerOptions.groupLimit}. */
+  readonly groupLimit?: number;
+  /** Window for the group limit (ms; default: the global `window`). */
+  readonly groupWindow?: number;
+  /** Algorithm for the group limit (default: the global `algorithm`). */
+  readonly groupAlgorithm?: Algorithm;
+}
+/** A {@link Doer} whose `do` accepts {@link RateTaskOptions} — i.e. a per-task group limit. */
+interface RateLimitedDoer extends Doer {
+  /** Accept a task; `opts` may carry a per-task `group` + `groupLimit` (a second, per-group gate). */
+  do(task: () => Promise<void>, opts?: RateTaskOptions): void;
+}
 /** Options for {@link rateLimitedDoer} — a {@link LimiterOptions} plus doer-specific knobs. */
 interface RateLimitedDoerOptions extends LimiterOptions {
-  /** Limit key — a single shared bucket by default (`'doer'`), or derived per task. */
+  /** Limit key for the **global** bucket — a single shared bucket by default (`'doer'`), or derived per task. */
   readonly key?: string | ((opts: DoerTaskOptions) => string);
+  /**
+   * Default **group** limit — a second gate bucketed by a task's `group`, on top of the global limit.
+   * A per-task {@link RateTaskOptions.groupLimit} overrides it. Omit for no group gate (global only).
+   * The group window/algorithm default to the global ones unless overridden per task.
+   */
+  readonly groupLimit?: number;
+  /** Derive a task's group bucket key (default: `opts.group`). Return `undefined` to skip the group gate for a task. */
+  readonly groupKey?: (opts: DoerTaskOptions) => string | undefined;
+  /**
+   * Store for the group buckets (default: the global `store`). A distributed store limits groups
+   * across a fleet. Groups need not be known up front — bucket state is per-group and **auto-evicted
+   * when idle** (the {@link memoryStore} sweeps expired/idle keys; `@ayepi/rate/redis` sets TTLs), so
+   * an unbounded, dynamic group space (per-user/per-model) won't grow without bound. For high churn,
+   * pass a tuned store, e.g. `groupStore: memoryStore({ sweepEvery: 200, idleMs: 30_000 })`.
+   */
+  readonly groupStore?: RateLimitStore;
   /** Floor on the re-check delay for deferred tasks (ms, default 50). */
   readonly retryFloor?: number;
   /** Clock injection (default `Date.now`). */
   readonly now?: () => number;
   /** The doer that actually runs admitted tasks (default {@link unlimitedDoer}). Compose policies. */
   readonly doer?: Doer;
+  /**
+   * Notified when the pending queue stays **continuously non-empty** past
+   * {@link RateLimitedDoerOptions.backlogAfterMs} — a sustained backlog of tasks the rate limit can't
+   * admit fast enough. Purely observational (for alerting/autoscaling); it must not throw — if it
+   * does, the throw is ignored. Requires `backlogAfterMs`.
+   */
+  readonly onBacklog?: (info: RateLimitedBacklogInfo) => void;
+  /** How long the queue must stay non-empty before {@link RateLimitedDoerOptions.onBacklog} first fires (ms). */
+  readonly backlogAfterMs?: number;
+  /** Re-fire `onBacklog` every this many ms while still backed up. Omit to fire once per backlog episode. */
+  readonly backlogEveryMs?: number;
   /**
    * Observe a store error during admission (e.g. a distributed store hiccup). The drain loop
    * never crashes on it — the task stays pending and admission is retried shortly. Off by
@@ -180,14 +232,40 @@ interface RateLimitedDoerOptions extends LimiterOptions {
  * const doer = rateLimitedDoer({ limit: 100, window: 60_000, algorithm: 'token-bucket' })
  * const w = createWork({ work: [sendEmail] as const, doer })
  * ```
+ *
+ * **Two-tier limiting.** Set a `groupLimit` (default, or per task via {@link RateTaskOptions}) and a
+ * task must clear both the global limit **and** its `group`'s limit — e.g. a global API cap plus a
+ * per-user / per-model cap. The group gate is checked first (a denied group check consumes nothing),
+ * so a task whose group is at its limit is **skipped** rather than blocking other groups, and the
+ * shared global bucket isn't spent on it. `group`/`groupLimit` are read at admission time.
  */
-declare function rateLimitedDoer(opts: RateLimitedDoerOptions): Doer;
+declare function rateLimitedDoer(opts: RateLimitedDoerOptions): RateLimitedDoer;
+/** Options for {@link memoryStore}. */
+interface MemoryStoreOptions {
+  /**
+   * Sweep expired/idle entries once every this many `consume` calls (default 1000). Lower bounds
+   * memory tighter when keys churn — e.g. many short-lived rate-limit **groups** (per-user/per-model)
+   * that aren't known up front — at a little more periodic scan cost.
+   */
+  readonly sweepEvery?: number;
+  /**
+   * Drop an idle **token-bucket** key after this long with no activity (ms, default 10 min).
+   * Windowed (`fixed-window`/`sliding-window`) counters are always dropped once their window has
+   * elapsed, regardless of this.
+   */
+  readonly idleMs?: number;
+}
 /**
- * An in-process {@link RateLimitStore} implementing all three algorithms. The
- * default store — fine for a single instance; use a distributed store (e.g.
- * `@ayepi/rate/redis`) to share limits across pods. Expired entries are swept
- * lazily.
+ * An in-process {@link RateLimitStore} implementing all three algorithms. The default store — fine
+ * for a single instance; use a distributed store (e.g. `@ayepi/rate/redis`, which sets key TTLs) to
+ * share limits across pods.
+ *
+ * **Bounded memory for dynamic keys.** Entries are swept lazily so an unbounded, not-known-in-advance
+ * key space (per-user/per-model rate-limit **groups**) doesn't grow without bound: a windowed counter
+ * is dropped once its window elapses, and an idle token-bucket after {@link MemoryStoreOptions.idleMs}.
+ * Active keys live long enough to enforce the limit; idle ones go away. Tune {@link MemoryStoreOptions}
+ * for high churn.
  */
-declare function memoryStore(): RateLimitStore;
+declare function memoryStore(opts?: MemoryStoreOptions): RateLimitStore;
 //#endregion
-export { Algorithm, DEFAULT_ALGORITHM, DEFAULT_PREFIX, Limiter, LimiterOptions, RateKeyIO, RateLimitDef, RateLimitDefOptions, RateLimitInfo, RateLimitResponseOptions, RateLimitResult, RateLimitRule, RateLimitStore, RateLimitedDoerOptions, limiter, memoryStore, rateLimit, rateLimitHeaders, rateLimitResponse, rateLimitedDoer };
+export { Algorithm, DEFAULT_ALGORITHM, DEFAULT_PREFIX, Limiter, LimiterOptions, MemoryStoreOptions, RateKeyIO, RateLimitDef, RateLimitDefOptions, RateLimitInfo, RateLimitResponseOptions, RateLimitResult, RateLimitRule, RateLimitStore, RateLimitedBacklogInfo, RateLimitedDoer, RateLimitedDoerOptions, RateTaskOptions, limiter, memoryStore, rateLimit, rateLimitHeaders, rateLimitResponse, rateLimitedDoer };

package/dist/index.js

@@ -150,18 +150,78 @@ const DOER_RETRY_FLOOR = 50;
 * const doer = rateLimitedDoer({ limit: 100, window: 60_000, algorithm: 'token-bucket' })
 * const w = createWork({ work: [sendEmail] as const, doer })
 * ```
+*
+* **Two-tier limiting.** Set a `groupLimit` (default, or per task via {@link RateTaskOptions}) and a
+* task must clear both the global limit **and** its `group`'s limit — e.g. a global API cap plus a
+* per-user / per-model cap. The group gate is checked first (a denied group check consumes nothing),
+* so a task whose group is at its limit is **skipped** rather than blocking other groups, and the
+* shared global bucket isn't spent on it. `group`/`groupLimit` are read at admission time.
 */
 function rateLimitedDoer(opts) {
-	const lim = limiter(opts);
+	const store = opts.store ?? memoryStore();
+	const lim = limiter({
+		...opts,
+		store
+	});
 	const inner = opts.doer ?? unlimitedDoer();
 	const now = opts.now ?? Date.now;
 	const floor = opts.retryFloor ?? DOER_RETRY_FLOOR;
 	const keyOf = (o) => typeof opts.key === "function" ? opts.key(o ?? {}) : opts.key ?? "doer";
+	const groupStore = opts.groupStore ?? store;
+	const groupPrefix = (opts.prefix ?? "rl:") + "grp:";
+	const groupKeyOf = (o) => opts.groupKey ? opts.groupKey(o ?? {}) : o?.group;
+	const groupRuleFor = (o) => {
+		const limit = o?.groupLimit ?? opts.groupLimit;
+		if (limit === void 0) return;
+		return {
+			limit,
+			window: o?.groupWindow ?? opts.window,
+			algorithm: o?.groupAlgorithm ?? opts.algorithm ?? "fixed-window",
+			countRejected: false
+		};
+	};
+	const report = (err) => {
+		try {
+			opts.onError?.(err);
+		} catch {}
+	};
 	const pending = [];
 	const idle = [];
 	let seq = 0;
 	let draining = false;
 	let timer = null;
+	const backlogWatch = opts.onBacklog !== void 0 && opts.backlogAfterMs !== void 0;
+	let nonEmptySince = null;
+	let backlogTimer = null;
+	const armBacklog = (ms) => {
+		backlogTimer = setTimeout(fireBacklog, ms);
+		backlogTimer.unref?.();
+	};
+	function fireBacklog() {
+		try {
+			opts.onBacklog({
+				pending: pending.length,
+				nonEmptyForMs: now() - nonEmptySince
+			});
+		} catch {}
+		if (opts.backlogEveryMs !== void 0) armBacklog(opts.backlogEveryMs);
+		else backlogTimer = null;
+	}
+	const syncBacklog = () => {
+		if (!backlogWatch) return;
+		if (pending.length > 0) {
+			if (nonEmptySince === null) {
+				nonEmptySince = now();
+				armBacklog(opts.backlogAfterMs);
+			}
+		} else {
+			nonEmptySince = null;
+			if (backlogTimer) {
+				clearTimeout(backlogTimer);
+				backlogTimer = null;
+			}
+		}
+	};
 	const arm = (ms) => {
 		if (timer) return;
 		timer = setTimeout(() => {
@@ -170,40 +230,67 @@ function rateLimitedDoer(opts) {
 		}, Math.max(floor, ms));
 		timer.unref?.();
 	};
+	/**
+	* The oldest pending task not skipped this round (a fresh scan, so tasks queued mid-drain are
+	* seen). `pending` is in insertion (seq) order, so equal `createdAt` ties keep the earliest.
+	*/
+	const nextPending = (skipped) => {
+		let best;
+		for (const t of pending) {
+			if (skipped.has(t)) continue;
+			if (!best || t.createdAt < best.createdAt) best = t;
+		}
+		return best;
+	};
 	const drain = async () => {
 		if (draining) return;
 		draining = true;
 		try {
-			while (pending.length > 0) {
+			const skipped = /* @__PURE__ */ new Set();
+			let soonest = Infinity;
+			for (;;) {
 				if (inner.available() <= 0) {
-					arm(floor);
+					soonest = Math.min(soonest, floor);
 					break;
 				}
-				let best = 0;
-				for (let i = 1; i < pending.length; i++) {
-					const a = pending[i];
-					const b = pending[best];
-					if (a.createdAt < b.createdAt || a.createdAt === b.createdAt && a.seq < b.seq) best = i;
+				const task = nextPending(skipped);
+				if (!task) break;
+				const o = task.opts;
+				const groupRule = groupRuleFor(o);
+				const gk = groupKeyOf(o);
+				if (groupRule && gk !== void 0) {
+					let gr;
+					try {
+						gr = await groupStore.consume(groupPrefix + gk, groupRule, now());
+					} catch (err) {
+						report(err);
+						soonest = Math.min(soonest, floor);
+						break;
+					}
+					if (!gr.allowed) {
+						soonest = Math.min(soonest, gr.retryAfter);
+						skipped.add(task);
+						continue;
+					}
 				}
-				const task = pending[best];
 				let res;
 				try {
-					res = await lim.check(keyOf(task.opts), now());
+					res = await lim.check(keyOf(o), now());
 				} catch (err) {
-					try {
-						opts.onError?.(err);
-					} catch {}
-					arm(floor);
+					report(err);
+					soonest = Math.min(soonest, floor);
 					break;
 				}
 				if (!res.allowed) {
-					arm(res.retryAfter);
+					soonest = Math.min(soonest, res.retryAfter);
 					break;
 				}
-				pending.splice(best, 1);
+				pending.splice(pending.indexOf(task), 1);
 				inner.do(task.run, task.opts);
 			}
 			if (pending.length === 0) for (const r of idle.splice(0)) r();
+			else arm(soonest);
+			syncBacklog();
 		} finally {
 			draining = false;
 		}
@@ -217,6 +304,7 @@ function rateLimitedDoer(opts) {
 				createdAt: o?.createdAt ?? now(),
 				seq: seq++
 			});
+			syncBacklog();
 			drain();
 		},
 		done: () => pending.length === 0 ? inner.done() : new Promise((r) => idle.push(() => void inner.done().then(r)))
@@ -297,19 +385,26 @@ function tokenBucket(buckets, key, rule, now) {
 	};
 }
 /**
-* An in-process {@link RateLimitStore} implementing all three algorithms. The
-* default store — fine for a single instance; use a distributed store (e.g.
-* `@ayepi/rate/redis`) to share limits across pods. Expired entries are swept
-* lazily.
+* An in-process {@link RateLimitStore} implementing all three algorithms. The default store — fine
+* for a single instance; use a distributed store (e.g. `@ayepi/rate/redis`, which sets key TTLs) to
+* share limits across pods.
+*
+* **Bounded memory for dynamic keys.** Entries are swept lazily so an unbounded, not-known-in-advance
+* key space (per-user/per-model rate-limit **groups**) doesn't grow without bound: a windowed counter
+* is dropped once its window elapses, and an idle token-bucket after {@link MemoryStoreOptions.idleMs}.
+* Active keys live long enough to enforce the limit; idle ones go away. Tune {@link MemoryStoreOptions}
+* for high churn.
 */
-function memoryStore() {
+function memoryStore(opts = {}) {
+	const sweepEvery = opts.sweepEvery ?? SWEEP_EVERY;
+	const idleMs = opts.idleMs ?? BUCKET_IDLE_MS;
 	const counters = /* @__PURE__ */ new Map();
 	const buckets = /* @__PURE__ */ new Map();
 	let ops = 0;
 	const sweep = (now) => {
-		if (++ops % SWEEP_EVERY !== 0) return;
+		if (++ops % sweepEvery !== 0) return;
 		for (const [k, e] of counters) if (e.reset <= now) counters.delete(k);
-		for (const [k, b] of buckets) if (now - b.ts > BUCKET_IDLE_MS) buckets.delete(k);
+		for (const [k, b] of buckets) if (now - b.ts > idleMs) buckets.delete(k);
 	};
 	return {
 		consume(key, rule, now) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ayepi/rate",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Rate-limiting middleware for @ayepi/core — pluggable stores, multiple algorithms, customizable 429 responses",
   "license": "MIT",
   "publishConfig": {
@@ -59,7 +59,7 @@
   },
   "peerDependencies": {
     "ioredis": "^5",
-    "@ayepi/core": "^0.3.0"
+    "@ayepi/core": "^0.4.0"
   },
   "peerDependenciesMeta": {
     "ioredis": {
@@ -74,7 +74,7 @@
     "tsdown": "^0.12.0",
     "vitest": "^2.1.8",
     "zod": "^4.4.3",
-    "@ayepi/core": "0.3.0"
+    "@ayepi/core": "0.4.0"
   },
   "keywords": [
     "ayepi",