@chaos-overlords/worker 0.8.1 → 0.9.0

package/README.md

@@ -23,6 +23,21 @@ The D1 migration lineages ship in `@chaos-overlords/storage` and `@chaos-overlor
 `ABANDONED_RETENTION_DAYS`, `BUG_REPORT_RETENTION_DAYS` and `BUG_REPORT_DAILY_STATE_MB` are vars. The
 `scheduled` handler expects a cron trigger; five minutes is the interval its sweeper is written for.
 
+**A deployment without a cron trigger has no safety net.** The sweeper is what seals a turn whose
+Durable Object alarm never fired, finishes a seal whose isolate died halfway through, re-runs the
+verdict of a match left paused by an interrupted one, and collects retention. Nothing else does any
+of it, and nothing fails loudly when it is missing: matches simply stop advancing for the players in
+them. `wrangler.dev.toml` deliberately carries no `[triggers]` block, because it is not a
+deployment — so the first thing to add to a real one is:
+
+```toml
+[triggers]
+crons = ["*/5 * * * *"]
+```
+
+The Worker logs `cron trigger has not fired` on a request once it has been up for an hour without
+one, which is how a deployment that forgot finds out from its own logs rather than from a player.
+
 ## Install
 
 ```sh

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chaos-overlords/worker",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "description": "Cloudflare Workers runtime facade: the shared Hono app over D1, with one Durable Object per match for SSE fan-out and turn-deadline alarms.",
   "keywords": [
     "chaos-overlords",
@@ -47,22 +47,22 @@
     "provenance": true
   },
   "dependencies": {
-    "@chaos-overlords/bug-reports": "0.8.1",
-    "@chaos-overlords/contracts": "0.8.1",
-    "@chaos-overlords/kernel": "0.8.1",
-    "@chaos-overlords/server": "0.8.1",
-    "@chaos-overlords/storage": "0.8.1",
+    "@chaos-overlords/bug-reports": "0.9.0",
+    "@chaos-overlords/contracts": "0.9.0",
+    "@chaos-overlords/kernel": "0.9.0",
+    "@chaos-overlords/server": "0.9.0",
+    "@chaos-overlords/storage": "0.9.0",
     "drizzle-orm": "^0.45.2",
-    "hono": "^4.13.7"
+    "hono": "^4.13.8"
   },
   "devDependencies": {
-    "@chaos-overlords/client": "0.8.1",
-    "@chaos-overlords/conformance": "0.8.1",
+    "@chaos-overlords/client": "0.9.0",
+    "@chaos-overlords/conformance": "0.9.0",
     "@cloudflare/vitest-pool-workers": "^0.22.0",
-    "@cloudflare/workers-types": "^5.20260910.1",
-    "typescript": "^5.9.3",
+    "@cloudflare/workers-types": "^5.20260920.1",
+    "typescript": "^7.0.2",
     "vitest": "^4.1.11",
-    "wrangler": "^4.131.0"
+    "wrangler": "^4.135.0"
   },
   "scripts": {
     "build": "tsc -p tsconfig.json --noEmit",

package/src/MatchHub.ts

@@ -1,4 +1,8 @@
-import { isDomainError } from '@chaos-overlords/kernel'
+import {
+  isDomainError,
+  type MultiplayerStorage,
+  type PersistedEvent,
+} from '@chaos-overlords/kernel'
 import { DEFAULT_SERVER_CONFIG, LocalEventHub } from '@chaos-overlords/server'
 import { createSqliteStorage, sqliteSchema } from '@chaos-overlords/storage/sqlite'
 import type { DurableObjectState } from '@cloudflare/workers-types'
@@ -20,30 +24,30 @@ const DEADLINE_KEY = 'deadline'
  */
 export class MatchHub {
   private readonly hub: LocalEventHub
+  private readonly repositories: MultiplayerStorage
 
   constructor(
     private readonly state: DurableObjectState,
     private readonly env: Env,
   ) {
-    const storage = createSqliteStorage(drizzle(env.DB, { schema: sqliteSchema }))
-    this.hub = new LocalEventHub(storage.events, DEFAULT_SERVER_CONFIG.sseHeartbeatMs)
+    this.repositories = createSqliteStorage(drizzle(env.DB, { schema: sqliteSchema }))
+    this.hub = new LocalEventHub(this.repositories.events, DEFAULT_SERVER_CONFIG.sseHeartbeatMs)
   }
 
   async fetch(request: Request): Promise<Response> {
     const url = new URL(request.url)
     switch (url.pathname) {
       case HUB_PATHS.notify: {
-        const { matchId } = (await request.json()) as { matchId: string }
-        this.hub.wake(matchId)
+        const event = (await request.json()) as PersistedEvent & { matchId?: string }
+        // The event body when the caller sent one; a bare match id is still accepted, so an isolate
+        // running an older build cannot silence this object's streams.
+        if (typeof event.seq === 'number') await this.hub.notify(event)
+        else if (event.matchId) this.hub.wake(event.matchId)
         return new Response(null, { status: 204 })
       }
       case HUB_PATHS.schedule: {
         const body = (await request.json()) as PendingDeadline & { dueAt: string }
-        await this.state.storage.put<PendingDeadline>(DEADLINE_KEY, {
-          matchId: body.matchId,
-          turn: body.turn,
-        })
-        await this.state.storage.setAlarm(new Date(body.dueAt).getTime())
+        await this.arm(body.matchId, body.turn, new Date(body.dueAt).getTime())
         return new Response(null, { status: 204 })
       }
       case HUB_PATHS.subscribe: {
@@ -78,16 +82,10 @@ export class MatchHub {
     if (!pending) return
     const kernel = buildKernel(this.env, {
       // Already inside the hub: wake local streams directly instead of calling ourselves.
-      notifier: { notify: async (event) => this.hub.wake(event.matchId) },
+      notifier: { notify: async (event) => this.hub.notify(event) },
       streams: this.hub,
       scheduler: {
-        schedule: async (input) => {
-          await this.state.storage.put<PendingDeadline>(DEADLINE_KEY, {
-            matchId: input.matchId,
-            turn: input.turn,
-          })
-          await this.state.storage.setAlarm(input.dueAt.getTime())
-        },
+        schedule: async (input) => this.arm(input.matchId, input.turn, input.dueAt.getTime()),
       },
     })
     const sealed = await kernel.turns.trySeal(pending.matchId, pending.turn, 'deadline')
@@ -95,6 +93,12 @@ export class MatchHub {
     await this.forgetSpentDeadline(pending)
   }
 
+  /** Record which turn the alarm is for, then set the alarm. One deadline is pending at a time. */
+  private async arm(matchId: string, turn: number, dueAtMs: number): Promise<void> {
+    await this.state.storage.put<PendingDeadline>(DEADLINE_KEY, { matchId, turn })
+    await this.state.storage.setAlarm(dueAtMs)
+  }
+
   /**
    * Drop the pending deadline once nothing is left to fire for. A seal on readiness, a finished
    * match and a match retention deleted all leave the alarm armed for a turn that is no longer
@@ -105,10 +109,9 @@ export class MatchHub {
   private async forgetSpentDeadline(fired: PendingDeadline): Promise<void> {
     const current = await this.state.storage.get<PendingDeadline>(DEADLINE_KEY)
     if (!current || current.matchId !== fired.matchId || current.turn !== fired.turn) return
-    const storage = createSqliteStorage(drizzle(this.env.DB, { schema: sqliteSchema }))
     const [match, turn] = await Promise.all([
-      storage.matches.get(fired.matchId),
-      storage.turns.get(fired.matchId, fired.turn),
+      this.repositories.matches.get(fired.matchId),
+      this.repositories.turns.get(fired.matchId, fired.turn),
     ])
     const stillDue =
       match?.status === 'running' && turn?.status === 'open' && turn.deadlineAt !== null

package/src/index.ts

@@ -87,6 +87,10 @@ export function buildContainer(env: Env): ServerContainer {
         env.BUG_REPORT_RATE_LIMIT_PER_MINUTE,
         DEFAULT_RATE_LIMITS.bugReportPerMinute,
       ),
+      bugReportState: new RateLimiter(clock, {
+        limit: DEFAULT_RATE_LIMITS.bugReportStatePerDay,
+        windowMs: 24 * 60 * 60 * 1000,
+      }),
     },
     // Listing is on unless a deployment turns it off: an unset var means the Browse screen works,
     // rather than every client being told the server lists nothing.
@@ -98,8 +102,37 @@ export function buildContainer(env: Env): ServerContainer {
   }
 }
 
+/**
+ * Whether this isolate has ever seen the cron fire, and when it started.
+ *
+ * A deployment with no cron trigger loses every safety net the `scheduled` handler is: the seal of
+ * a turn whose Durable Object alarm never fired, the repair of a seal an isolate died in the
+ * middle of, the re-run of a verdict cut short, and all of retention. None of that fails loudly —
+ * matches just stop advancing for the people in them — so it is worth one log line. Per isolate,
+ * which means a busy Worker says it a few times and then never again; that is the right volume for
+ * something whose remedy is four lines of `wrangler.toml`.
+ */
+let cronSeen = false
+let cronWatchStartedAt = 0
+const CRON_GRACE_MS = 60 * 60 * 1000
+
+function warnIfCronIsMissing(): void {
+  if (cronSeen) return
+  const now = Date.now()
+  if (cronWatchStartedAt === 0) {
+    cronWatchStartedAt = now
+    return
+  }
+  if (now - cronWatchStartedAt < CRON_GRACE_MS) return
+  cronWatchStartedAt = now
+  workerLogger.warn('cron trigger has not fired', {
+    hint: 'add [triggers] crons = ["*/5 * * * *"] to wrangler.toml; without it nothing seals a missed deadline, finishes an interrupted seal or collects retention',
+  })
+}
+
 export default {
   async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
+    warnIfCronIsMissing()
     return containerFor(env).app.fetch(request, env, ctx)
   },
   /** The cron safety net: expired deadlines, interrupted seals, and retention. */
@@ -108,21 +141,33 @@ export default {
     env: Env,
     ctx: ExecutionContext,
   ): Promise<void> {
+    cronSeen = true
     const { container } = containerFor(env)
     const { kernel } = container
+    // Each step in its own `try`, as the Node sweeper already does. One shared `catch` meant a
+    // throw inside the turn sweep also stopped both retention sweeps, every time the cron ran.
+    const step = async (name: string, run: () => Promise<void>): Promise<void> => {
+      try {
+        await run()
+      } catch (error: unknown) {
+        workerLogger.error('cron step failed', { step: name, error: String(error) })
+      }
+    }
     ctx.waitUntil(
       (async () => {
-        const { sealed, repaired } = await kernel.turns.sweep()
-        if (sealed > 0 || repaired > 0) {
-          workerLogger.info('cron advanced turns', { sealed, repaired })
-        }
-        await kernel.retention.collect()
+        await step('turns', async () => {
+          const { sealed, repaired } = await kernel.turns.sweep()
+          if (sealed > 0 || repaired > 0) {
+            workerLogger.info('cron advanced turns', { sealed, repaired })
+          }
+        })
+        await step('retention', () => kernel.retention.collect().then(() => undefined))
         // A separate database with a separate window; nothing about a match's retention decides
         // when a bug report and its R2 object go.
-        await container.bugReports?.collect()
-      })().catch((error: unknown) => {
-        workerLogger.error('cron sweep failed', { error: String(error) })
-      }),
+        await step('bugReports', async () => {
+          await container.bugReports?.collect()
+        })
+      })(),
     )
   },
 }

package/src/kernel.ts

@@ -30,6 +30,12 @@ const DAY_MS = 24 * 60 * 60 * 1000
 const DEFAULT_RETENTION_DAYS = 30
 const DEFAULT_ABANDONED_RETENTION_DAYS = 90
 
+/** A non-negative whole number from an environment variable; anything else reads as `fallback`. */
+function wholeNumber(raw: string | undefined, fallback: number): number {
+  const value = Number(raw ?? fallback)
+  return Number.isInteger(value) && value >= 0 ? value : fallback
+}
+
 export const HUB_PATHS = {
   notify: '/notify',
   schedule: '/schedule',
@@ -43,6 +49,39 @@ export function hubFor(env: Env, matchId: string) {
   return env.MATCH_HUB.get(env.MATCH_HUB.idFromName(matchId))
 }
 
+/**
+ * How long a call into a match's Durable Object may take before the caller gives up on it.
+ *
+ * Every one of them is best effort by design — fan-out, arming a deadline, hanging up a revoked
+ * membership — and each has a safety net behind it: the stream's own catch-up drain, the sweep's
+ * `listExpiredOpen`, and the revoked token itself. What they did not have was an end. An object
+ * that is slow to wake, being relocated, or holding a lock kept the REQUEST waiting, so a seal
+ * completed in the database and the submitter saw a timeout.
+ */
+const HUB_CALL_TIMEOUT_MS = 5_000
+
+/**
+ * A best-effort call into a hub: bounded in time, and its failure logged rather than raised.
+ *
+ * The step it belongs to has already committed whatever was durable about it, so the only thing a
+ * failure here can still cost is latency somebody else's retry or the sweep already covers.
+ */
+async function tellHub(
+  env: Env,
+  call: { matchId: string; path: string; body: unknown },
+): Promise<void> {
+  const { matchId, path, body } = call
+  try {
+    await hubFor(env, matchId).fetch(`https://hub${path}`, {
+      method: 'POST',
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(HUB_CALL_TIMEOUT_MS),
+    })
+  } catch (error) {
+    workerLogger.warn('could not reach the match hub', { matchId, path, error: String(error) })
+  }
+}
+
 /**
  * Builds the kernel over D1 for one invocation. Fan-out and alarms cross into the match's Durable
  * Object; everything else is plain D1 through the shared SQLite repositories.
@@ -57,32 +96,25 @@ export function buildKernel(
 ): Kernel {
   const storage = createSqliteStorage(drizzle(env.DB, { schema: sqliteSchema }))
   const notifier: EventNotifier = overrides.notifier ?? {
-    notify: async (event) => {
-      await hubFor(env, event.matchId).fetch(`https://hub${HUB_PATHS.notify}`, {
-        method: 'POST',
-        body: JSON.stringify({ matchId: event.matchId }),
-      })
-    },
+    // The whole durable row, not just the match id. The object formats it once and hands the frame
+    // to every stream of the match, so a seal's burst costs it no reads at all; being told only
+    // which match had changed meant each subscriber queried D1 for rows the notification was
+    // already carrying. The event has been persisted before this runs, so the body crossing the
+    // isolate boundary is a copy of a fact, never a substitute for one.
+    notify: async (event) =>
+      tellHub(env, { matchId: event.matchId, path: HUB_PATHS.notify, body: event }),
   }
   const streams: StreamCloser = overrides.streams ?? {
-    close: async (input) => {
-      await hubFor(env, input.matchId).fetch(`https://hub${HUB_PATHS.disconnect}`, {
-        method: 'POST',
-        body: JSON.stringify(input),
-      })
-    },
+    close: async (input) =>
+      tellHub(env, { matchId: input.matchId, path: HUB_PATHS.disconnect, body: input }),
   }
   const scheduler: DeadlineScheduler = overrides.scheduler ?? {
-    schedule: async (input) => {
-      await hubFor(env, input.matchId).fetch(`https://hub${HUB_PATHS.schedule}`, {
-        method: 'POST',
-        body: JSON.stringify({ ...input, dueAt: input.dueAt.toISOString() }),
-      })
-    },
-  }
-  const days = (raw: string | undefined, fallback: number): number => {
-    const value = Number(raw ?? fallback)
-    return Number.isInteger(value) && value >= 0 ? value : fallback
+    schedule: async (input) =>
+      tellHub(env, {
+        matchId: input.matchId,
+        path: HUB_PATHS.schedule,
+        body: { ...input, dueAt: input.dueAt.toISOString() },
+      }),
   }
   return createKernel(
     {
@@ -95,9 +127,13 @@ export function buildKernel(
     },
     {
       retention: {
-        maxAgeMs: days(env.RETENTION_DAYS, DEFAULT_RETENTION_DAYS) * DAY_MS,
+        maxAgeMs: wholeNumber(env.RETENTION_DAYS, DEFAULT_RETENTION_DAYS) * DAY_MS,
         abandonedLiveMaxAgeMs:
-          days(env.ABANDONED_RETENTION_DAYS, DEFAULT_ABANDONED_RETENTION_DAYS) * DAY_MS,
+          wholeNumber(env.ABANDONED_RETENTION_DAYS, DEFAULT_ABANDONED_RETENTION_DAYS) * DAY_MS,
+        // Twice the abandoned window, and without its roster test, which never collects an untimed
+        // match whose players' clients died without a `leave`.
+        silentLiveMaxAgeMs:
+          wholeNumber(env.ABANDONED_RETENTION_DAYS, DEFAULT_ABANDONED_RETENTION_DAYS) * 2 * DAY_MS,
         batchSize: 50,
       },
     },
@@ -119,10 +155,6 @@ export function buildBugReports(env: Env): BugReportService | undefined {
   if (!env.BUG_DB) return undefined
   const repository = createBugReportRepository(drizzle(env.BUG_DB, { schema: bugReportSchema }))
   const blobs = env.BUG_BLOBS ? createR2BlobStore(env.BUG_BLOBS) : undefined
-  const number = (raw: string | undefined, fallback: number): number => {
-    const value = Number(raw ?? fallback)
-    return Number.isInteger(value) && value >= 0 ? value : fallback
-  }
   return createBugReportService({
     repository,
     clock: { now: () => new Date() },
@@ -130,14 +162,14 @@ export function buildBugReports(env: Env): BugReportService | undefined {
     retention: {
       ...DEFAULT_BUG_REPORT_RETENTION,
       dailyStateBytes:
-        number(
+        wholeNumber(
           env.BUG_REPORT_DAILY_STATE_MB,
           DEFAULT_BUG_REPORT_RETENTION.dailyStateBytes / (1024 * 1024),
         ) *
         1024 *
         1024,
       maxAgeMs:
-        number(env.BUG_REPORT_RETENTION_DAYS, DEFAULT_BUG_REPORT_RETENTION.maxAgeMs / DAY_MS) *
+        wholeNumber(env.BUG_REPORT_RETENTION_DAYS, DEFAULT_BUG_REPORT_RETENTION.maxAgeMs / DAY_MS) *
         DAY_MS,
     },
     ...(blobs ? { blobs } : {}),