@chaos-overlords/worker 0.9.9 → 0.9.10

package/README.md

@@ -9,12 +9,15 @@ This package ships TypeScript sources rather than a bundle, because wrangler bun
 itself. A deployment re-exports both the handler and the Durable Object class:
 
 ```ts
-export { MatchHub } from '@chaos-overlords/worker'
+export { MatchHub, RateLimitCounter } from '@chaos-overlords/worker'
 export { default } from '@chaos-overlords/worker'
 ```
 
-It needs two D1 bindings (`DB`, `BUG_DB`), an R2 bucket (`BUG_BLOBS`) and the `MATCH_HUB` Durable
-Object namespace, whose migration lineage starts at tag `v1` with `new_sqlite_classes = ["MatchHub"]`.
+It needs two D1 bindings (`DB`, `BUG_DB`), an R2 bucket (`BUG_BLOBS`) and two Durable Object
+namespaces: `MATCH_HUB`, whose migration lineage starts at tag `v1` with
+`new_sqlite_classes = ["MatchHub"]`, and `RATE_LIMITS`, the `RateLimitCounter` class added at tag
+`v2` with `new_sqlite_classes = ["RateLimitCounter"]`. `RATE_LIMITS` is what makes the rate limits
+global; without it every isolate counts on its own, and the Worker logs `RATE_LIMITS is not bound`.
 The D1 migration lineages ship in `@chaos-overlords/storage` and `@chaos-overlords/bug-reports`; point
 `wrangler d1 migrations apply` at them rather than copying them.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chaos-overlords/worker",
-  "version": "0.9.9",
+  "version": "0.9.10",
   "description": "Cloudflare Workers runtime facade: the shared Hono app over D1, with one Durable Object per match for SSE fan-out and turn-deadline alarms.",
   "keywords": [
     "chaos-overlords",
@@ -47,17 +47,17 @@
     "provenance": true
   },
   "dependencies": {
-    "@chaos-overlords/bug-reports": "0.9.9",
-    "@chaos-overlords/contracts": "0.9.9",
-    "@chaos-overlords/kernel": "0.9.9",
-    "@chaos-overlords/server": "0.9.9",
-    "@chaos-overlords/storage": "0.9.9",
+    "@chaos-overlords/bug-reports": "0.9.10",
+    "@chaos-overlords/contracts": "0.9.10",
+    "@chaos-overlords/kernel": "0.9.10",
+    "@chaos-overlords/server": "0.9.10",
+    "@chaos-overlords/storage": "0.9.10",
     "drizzle-orm": "^0.45.3",
     "hono": "^4.13.9"
   },
   "devDependencies": {
-    "@chaos-overlords/client": "0.9.9",
-    "@chaos-overlords/conformance": "0.9.9",
+    "@chaos-overlords/client": "0.9.10",
+    "@chaos-overlords/conformance": "0.9.10",
     "@cloudflare/vitest-pool-workers": "^0.22.0",
     "@cloudflare/workers-types": "^5.20260927.1",
     "typescript": "^7.0.2",

package/src/RateLimitCounter.ts

@@ -0,0 +1,163 @@
+import {
+  consumeWindow,
+  type RateLimitPolicy,
+  type RateLimitStore,
+  type RateLimitWindow,
+} from '@chaos-overlords/kernel'
+import type { DurableObjectNamespace, DurableObjectState } from '@cloudflare/workers-types'
+
+interface StoredWindow {
+  windowStart: number
+  resetAt: number
+  count: number
+}
+
+const WINDOW_KEY = 'window'
+
+/**
+ * The longest window kept in memory alone.
+ *
+ * Cloudflare evicts an idle object after 70 to 140 seconds without a request, so a minute-long
+ * window has rolled before its object can be evicted for idleness, and keeping it in memory loses
+ * nothing but the rare window a deployment or a runtime restart interrupts, which forgives that
+ * caller the rest of one minute. A longer window (the day-long journal budget) would be forgiven
+ * by simply waiting two minutes, so it is written to the object's storage and deleted by an alarm
+ * when it rolls.
+ */
+const MEMORY_ONLY_WINDOW_MS = 60_000
+
+const PATHS = { consume: '/consume', inspect: '/inspect', refund: '/refund' } as const
+
+/**
+ * One rate limit window, for one budget and one caller, wherever in the world the caller's
+ * requests land.
+ *
+ * Each key gets an object of its own (`idFromName(key)`), so every isolate that counts a caller
+ * reaches the same counter and a budget holds across the whole deployment; the object is created
+ * near the caller's first request, so the round trip stays short. Cloudflare's rate limiting
+ * binding was not usable for this: it counts per Cloudflare location, only over ten or sixty
+ * seconds, and answers only yes or no, with no time to wait, no inspection and no refund, which the
+ * day-long journal budget, `Retry-After`, the match-creation peek and the journal reservation need.
+ *
+ * An object processes one event at a time and its storage calls hold its input gate, so the read,
+ * decision and write of a `consume` cannot interleave with another caller's.
+ */
+export class RateLimitCounter {
+  private current: StoredWindow | undefined
+  /** Whether `current` is also in storage, so a refund knows to write it back. */
+  private persisted = false
+  /**
+   * The one read of storage, shared by every call that arrives before it settles, so a call that
+   * reads late never overwrites a window another call has already counted in.
+   */
+  private loading: Promise<void> | undefined
+
+  constructor(
+    private readonly state: DurableObjectState,
+    _env: unknown,
+  ) {}
+
+  async fetch(request: Request): Promise<Response> {
+    const path = new URL(request.url).pathname
+    const body = (await request.json()) as {
+      policy?: RateLimitPolicy
+      now?: number
+      windowStart?: number
+    }
+    await this.load()
+    if (path === PATHS.consume && body.policy && typeof body.now === 'number') {
+      const window = consumeWindow(this.current, body.policy, body.now)
+      // A refused call leaves the window as it was, so there is nothing to write.
+      if (window.allowed) {
+        await this.save(
+          { windowStart: window.windowStart, resetAt: window.resetAt, count: window.count },
+          body.policy.windowMs > MEMORY_ONLY_WINDOW_MS,
+        )
+      }
+      return Response.json(window)
+    }
+    if (path === PATHS.inspect && typeof body.now === 'number') {
+      const live = this.current && this.current.resetAt > body.now ? this.current : undefined
+      return Response.json(live ? { count: live.count, resetAt: live.resetAt } : null)
+    }
+    if (path === PATHS.refund && typeof body.windowStart === 'number') {
+      const window = this.current
+      if (window && window.windowStart === body.windowStart && window.count > 0) {
+        await this.save({ ...window, count: window.count - 1 }, this.persisted)
+      }
+      return Response.json(null)
+    }
+    return new Response('not found', { status: 404 })
+  }
+
+  /** Forgets a persisted window once it has rolled; a fresh one is opened by the next call. */
+  async alarm(): Promise<void> {
+    await this.load()
+    if (!this.persisted) return
+    if (this.current && this.current.resetAt > Date.now()) {
+      await this.state.storage.setAlarm(this.current.resetAt)
+      return
+    }
+    this.current = undefined
+    this.persisted = false
+    await this.state.storage.deleteAll()
+  }
+
+  private load(): Promise<void> {
+    this.loading ??= this.state.storage.get<StoredWindow>(WINDOW_KEY).then(
+      (stored) => {
+        this.current = stored ?? undefined
+        this.persisted = this.current !== undefined
+      },
+      (error: unknown) => {
+        // A failed read is retried by the next call rather than remembered.
+        this.loading = undefined
+        throw error
+      },
+    )
+    return this.loading
+  }
+
+  private async save(window: StoredWindow, persist: boolean): Promise<void> {
+    const opened = this.current?.windowStart !== window.windowStart
+    this.current = window
+    if (!persist) {
+      // A minute window that replaced a persisted day window: drop the stored one and its alarm.
+      if (this.persisted) await this.state.storage.deleteAll()
+      this.persisted = false
+      return
+    }
+    await this.state.storage.put(WINDOW_KEY, window)
+    if (opened || !this.persisted) await this.state.storage.setAlarm(window.resetAt)
+    this.persisted = true
+  }
+}
+
+/**
+ * How long a Worker waits on a counter before letting the call through. A counter that does not
+ * answer must not hold every request behind it; see `sharedRateLimiters` for why a failure lets
+ * the call through.
+ */
+const COUNTER_TIMEOUT_MS = 2_000
+
+/** The {@link RateLimitStore} the Worker counts in: one {@link RateLimitCounter} per key. */
+export function durableObjectRateLimitStore(namespace: DurableObjectNamespace): RateLimitStore {
+  const call = async <T>(key: string, path: string, body: unknown): Promise<T> => {
+    const stub = namespace.get(namespace.idFromName(key))
+    const response = await stub.fetch(`https://counter${path}`, {
+      method: 'POST',
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(COUNTER_TIMEOUT_MS),
+    })
+    if (!response.ok) throw new Error(`rate limit counter answered ${response.status}`)
+    return (await response.json()) as T
+  }
+  return {
+    consume: (key, policy, now) => call<RateLimitWindow>(key, PATHS.consume, { policy, now }),
+    inspect: (key, now) =>
+      call<{ count: number; resetAt: number } | null>(key, PATHS.inspect, { now }),
+    refund: async (key, windowStart) => {
+      await call<null>(key, PATHS.refund, { windowStart })
+    },
+  }
+}

package/src/env.ts

@@ -3,6 +3,12 @@ import type { D1Database, DurableObjectNamespace, R2Bucket } from '@cloudflare/w
 export interface Env {
   DB: D1Database
   MATCH_HUB: DurableObjectNamespace
+  /**
+   * The rate limit counters, one `RateLimitCounter` object per budget and caller, so every isolate
+   * spends the same budgets. Unbound, each isolate counts on its own and the Worker logs a warning:
+   * a budget is then multiplied by however many isolates a caller's requests reach.
+   */
+  RATE_LIMITS?: DurableObjectNamespace
   /**
    * Bug reports, in a D1 instance of their own.
    *
@@ -29,7 +35,7 @@ export interface Env {
   MEMBER_RATE_LIMIT_PER_MINUTE?: string
   UPLOAD_RATE_LIMIT_PER_MINUTE?: string
   BUG_REPORT_RATE_LIMIT_PER_MINUTE?: string
-  /** Matches created per minute across every caller, per isolate. */
+  /** Matches created per minute across every caller, counted for the whole deployment when `RATE_LIMITS` is bound. */
   MATCH_CREATION_RATE_LIMIT_PER_MINUTE?: string
   /** Days before a finished or abandoned match is deleted. 0 keeps them forever. */
   RETENTION_DAYS?: string

package/src/index.ts

@@ -1,10 +1,16 @@
-import { RateLimitedError, RateLimiter } from '@chaos-overlords/kernel'
+import {
+  memoryRateLimiters,
+  RateLimitedError,
+  type RateLimiterFactory,
+  sharedRateLimiters,
+} from '@chaos-overlords/kernel'
 import {
   type AppEnv,
   configFlag,
   configInteger,
   configList,
   createApp,
+  createRateLimiters,
   DEFAULT_RATE_LIMITS,
   DEFAULT_SERVER_CONFIG,
   defaultClientAddress,
@@ -14,8 +20,10 @@ import type { ExecutionContext, ScheduledController } from '@cloudflare/workers-
 import type { Hono } from 'hono'
 import type { Env } from './env'
 import { buildBugReports, buildKernel, HUB_PATHS, hubFor, workerLogger } from './kernel'
+import { durableObjectRateLimitStore } from './RateLimitCounter'
 
 export { MatchHub } from './MatchHub'
+export { RateLimitCounter } from './RateLimitCounter'
 
 type Built = { container: ServerContainer; app: Hono<AppEnv> }
 
@@ -29,11 +37,10 @@ type RateLimitVar =
 /**
  * One container per isolate, held in module scope.
  *
- * It has to be cached: a rate limiter counts requests within a window, so building a fresh one per
- * request would reset the window every time and limit nothing. The router and the D1-backed kernel
- * are per-isolate state for the same reason a server builds them once at startup — there is nothing
- * request-specific in either. Cloudflare's own rate limiting rules still belong in front of a public
- * deployment, because an isolate is not the whole world.
+ * It has to be cached: without the `RATE_LIMITS` binding a rate limiter counts requests within a
+ * window in the isolate, so building a fresh one per request would reset the window every time and
+ * limit nothing. The router and the D1-backed kernel are per-isolate state for the same reason a
+ * server builds them once at startup: there is nothing request-specific in either.
  *
  * A module-scoped singleton rather than a `WeakMap` keyed on `env`: the bindings object being the
  * same identity on every request is not a documented guarantee, and if it ever stopped being one the
@@ -61,10 +68,11 @@ export function buildContainer(env: Env): ServerContainer {
   // A misconfigured var refuses every request and every cron run until it is fixed, so the error
   // names the variable: a Worker has no startup log for it to go unnoticed in, only request errors.
   const perMinute = (name: RateLimitVar, fallback: number) =>
-    new RateLimiter(clock, { limit: configInteger(env[name], fallback, 1, name), windowMs: 60_000 })
+    configInteger(env[name], fallback, 1, name)
+  const rateLimits = rateLimitersFor(env, clock)
   const bugReports = buildBugReports(env)
   return {
-    kernel: buildKernel(env),
+    kernel: buildKernel(env, { rateLimits }),
     ...(bugReports ? { bugReports } : {}),
     eventStream: {
       open: async ({ matchId, playerId, afterSeq, signal }) => {
@@ -88,23 +96,29 @@ export function buildContainer(env: Env): ServerContainer {
         })
       },
     },
-    rateLimiters: {
-      anonymous: perMinute('RATE_LIMIT_PER_MINUTE', DEFAULT_RATE_LIMITS.anonymousPerMinute),
-      member: perMinute('MEMBER_RATE_LIMIT_PER_MINUTE', DEFAULT_RATE_LIMITS.memberPerMinute),
-      upload: perMinute('UPLOAD_RATE_LIMIT_PER_MINUTE', DEFAULT_RATE_LIMITS.uploadPerMinute),
-      bugReport: perMinute(
+    rateLimiters: createRateLimiters(rateLimits, {
+      anonymousPerMinute: perMinute(
+        'RATE_LIMIT_PER_MINUTE',
+        DEFAULT_RATE_LIMITS.anonymousPerMinute,
+      ),
+      memberPerMinute: perMinute(
+        'MEMBER_RATE_LIMIT_PER_MINUTE',
+        DEFAULT_RATE_LIMITS.memberPerMinute,
+      ),
+      uploadPerMinute: perMinute(
+        'UPLOAD_RATE_LIMIT_PER_MINUTE',
+        DEFAULT_RATE_LIMITS.uploadPerMinute,
+      ),
+      bugReportPerMinute: perMinute(
         'BUG_REPORT_RATE_LIMIT_PER_MINUTE',
         DEFAULT_RATE_LIMITS.bugReportPerMinute,
       ),
-      bugReportState: new RateLimiter(clock, {
-        limit: DEFAULT_RATE_LIMITS.bugReportStatePerDay,
-        windowMs: 24 * 60 * 60 * 1000,
-      }),
-      matchCreation: perMinute(
+      bugReportStatePerDay: DEFAULT_RATE_LIMITS.bugReportStatePerDay,
+      matchCreationPerMinute: perMinute(
         'MATCH_CREATION_RATE_LIMIT_PER_MINUTE',
         DEFAULT_RATE_LIMITS.matchCreationPerMinute,
       ),
-    },
+    }),
     // Listing is on unless a deployment turns it off: an unset var means the Browse screen works,
     // rather than every client being told the server lists nothing.
     config: {
@@ -119,6 +133,24 @@ export function buildContainer(env: Env): ServerContainer {
   }
 }
 
+/**
+ * Where this Worker's budgets are counted: in a `RateLimitCounter` per key when `RATE_LIMITS` is
+ * bound, so they hold across every isolate and location, and in the isolate otherwise.
+ *
+ * The fallback keeps a deployment that has not added the binding yet serving, and says so once per
+ * isolate, at the point the container is built: a Worker has no startup to fail, and refusing every
+ * request over a missing binding would take the game down for a limit that is only weaker.
+ */
+function rateLimitersFor(env: Env, clock: { now(): Date }): RateLimiterFactory {
+  if (env.RATE_LIMITS) {
+    return sharedRateLimiters(durableObjectRateLimitStore(env.RATE_LIMITS), clock, workerLogger)
+  }
+  workerLogger.warn('RATE_LIMITS is not bound: rate limits count per isolate', {
+    hint: 'bind the RateLimitCounter Durable Object as RATE_LIMITS in wrangler.toml; see multiplayer/README.md',
+  })
+  return memoryRateLimiters(clock)
+}
+
 /**
  * Whether this isolate has ever seen the cron fire, and when it started.
  *

package/src/kernel.ts

@@ -14,6 +14,7 @@ import {
   type EventNotifier,
   type Kernel,
   type Logger,
+  type RateLimiterFactory,
   type RetentionPolicy,
   retentionPolicyFromDays,
   type StreamCloser,
@@ -103,18 +104,26 @@ const HUB_CALL_TIMEOUT_MS = 5_000
  *
  * The step it belongs to has already committed whatever was durable about it, so the only thing a
  * failure here can still cost is latency somebody else's retry or the sweep already covers.
+ *
+ * An answer with an error status is a failure too: the object was reached but did not do what it
+ * was asked, for example a path missing from its routes answers 404. It gets its own warning, so a
+ * hub that refuses every call shows up in the log instead of passing as a delivered one.
  */
-async function tellHub(
+export async function tellHub(
   env: Env,
   call: { matchId: string; path: string; body: unknown },
 ): Promise<void> {
   const { matchId, path, body } = call
   try {
-    await hubFor(env, matchId).fetch(`https://hub${path}`, {
+    const response = await hubFor(env, matchId).fetch(`https://hub${path}`, {
       method: 'POST',
       body: JSON.stringify(body),
       signal: AbortSignal.timeout(HUB_CALL_TIMEOUT_MS),
     })
+    if (!response.ok) {
+      workerLogger.warn('the match hub refused a call', { matchId, path, status: response.status })
+    }
+    await response.body?.cancel()
   } catch (error) {
     workerLogger.warn('could not reach the match hub', { matchId, path, error: String(error) })
   }
@@ -130,6 +139,8 @@ export function buildKernel(
     notifier: EventNotifier
     scheduler: DeadlineScheduler
     streams: StreamCloser
+    /** Where the kernel's password budgets count; the isolate when absent. */
+    rateLimits: RateLimiterFactory
   }> = {},
 ): Kernel {
   const storage = createSqliteStorage(drizzle(env.DB, { schema: sqliteSchema }))
@@ -162,6 +173,7 @@ export function buildKernel(
       streams,
       clock: { now: () => new Date() },
       logger: workerLogger,
+      ...(overrides.rateLimits ? { rateLimits: overrides.rateLimits } : {}),
     },
     { retention: retentionPolicyFor(env) },
   )