@chaos-overlords/worker 0.8.1 → 0.8.2

package/README.md

@@ -23,6 +23,21 @@ The D1 migration lineages ship in `@chaos-overlords/storage` and `@chaos-overlor
 `ABANDONED_RETENTION_DAYS`, `BUG_REPORT_RETENTION_DAYS` and `BUG_REPORT_DAILY_STATE_MB` are vars. The
 `scheduled` handler expects a cron trigger; five minutes is the interval its sweeper is written for.
 
+**A deployment without a cron trigger has no safety net.** The sweeper is what seals a turn whose
+Durable Object alarm never fired, finishes a seal whose isolate died halfway through, re-runs the
+verdict of a match left paused by an interrupted one, and collects retention. Nothing else does any
+of it, and nothing fails loudly when it is missing: matches simply stop advancing for the players in
+them. `wrangler.dev.toml` deliberately carries no `[triggers]` block, because it is not a
+deployment — so the first thing to add to a real one is:
+
+```toml
+[triggers]
+crons = ["*/5 * * * *"]
+```
+
+The Worker logs `cron trigger has not fired` on a request once it has been up for an hour without
+one, which is how a deployment that forgot finds out from its own logs rather than from a player.
+
 ## Install
 
 ```sh

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chaos-overlords/worker",
-  "version": "0.8.1",
+  "version": "0.8.2",
   "description": "Cloudflare Workers runtime facade: the shared Hono app over D1, with one Durable Object per match for SSE fan-out and turn-deadline alarms.",
   "keywords": [
     "chaos-overlords",
@@ -47,22 +47,22 @@
     "provenance": true
   },
   "dependencies": {
-    "@chaos-overlords/bug-reports": "0.8.1",
-    "@chaos-overlords/contracts": "0.8.1",
-    "@chaos-overlords/kernel": "0.8.1",
-    "@chaos-overlords/server": "0.8.1",
-    "@chaos-overlords/storage": "0.8.1",
+    "@chaos-overlords/bug-reports": "0.8.2",
+    "@chaos-overlords/contracts": "0.8.2",
+    "@chaos-overlords/kernel": "0.8.2",
+    "@chaos-overlords/server": "0.8.2",
+    "@chaos-overlords/storage": "0.8.2",
     "drizzle-orm": "^0.45.2",
-    "hono": "^4.13.7"
+    "hono": "^4.13.8"
   },
   "devDependencies": {
-    "@chaos-overlords/client": "0.8.1",
-    "@chaos-overlords/conformance": "0.8.1",
+    "@chaos-overlords/client": "0.8.2",
+    "@chaos-overlords/conformance": "0.8.2",
     "@cloudflare/vitest-pool-workers": "^0.22.0",
-    "@cloudflare/workers-types": "^5.20260910.1",
-    "typescript": "^5.9.3",
+    "@cloudflare/workers-types": "^5.20260920.1",
+    "typescript": "^7.0.2",
     "vitest": "^4.1.11",
-    "wrangler": "^4.131.0"
+    "wrangler": "^4.135.0"
   },
   "scripts": {
     "build": "tsc -p tsconfig.json --noEmit",

package/src/MatchHub.ts

@@ -1,4 +1,4 @@
-import { isDomainError } from '@chaos-overlords/kernel'
+import { isDomainError, type PersistedEvent } from '@chaos-overlords/kernel'
 import { DEFAULT_SERVER_CONFIG, LocalEventHub } from '@chaos-overlords/server'
 import { createSqliteStorage, sqliteSchema } from '@chaos-overlords/storage/sqlite'
 import type { DurableObjectState } from '@cloudflare/workers-types'
@@ -33,8 +33,11 @@ export class MatchHub {
     const url = new URL(request.url)
     switch (url.pathname) {
       case HUB_PATHS.notify: {
-        const { matchId } = (await request.json()) as { matchId: string }
-        this.hub.wake(matchId)
+        const event = (await request.json()) as PersistedEvent & { matchId?: string }
+        // The event body when the caller sent one; a bare match id is still accepted, so an isolate
+        // running an older build cannot silence this object's streams.
+        if (typeof event.seq === 'number') await this.hub.notify(event)
+        else if (event.matchId) this.hub.wake(event.matchId)
         return new Response(null, { status: 204 })
       }
       case HUB_PATHS.schedule: {
@@ -78,7 +81,7 @@ export class MatchHub {
     if (!pending) return
     const kernel = buildKernel(this.env, {
       // Already inside the hub: wake local streams directly instead of calling ourselves.
-      notifier: { notify: async (event) => this.hub.wake(event.matchId) },
+      notifier: { notify: async (event) => this.hub.notify(event) },
       streams: this.hub,
       scheduler: {
         schedule: async (input) => {

package/src/index.ts

@@ -87,6 +87,10 @@ export function buildContainer(env: Env): ServerContainer {
         env.BUG_REPORT_RATE_LIMIT_PER_MINUTE,
         DEFAULT_RATE_LIMITS.bugReportPerMinute,
       ),
+      bugReportState: new RateLimiter(clock, {
+        limit: DEFAULT_RATE_LIMITS.bugReportStatePerDay,
+        windowMs: 24 * 60 * 60 * 1000,
+      }),
     },
     // Listing is on unless a deployment turns it off: an unset var means the Browse screen works,
     // rather than every client being told the server lists nothing.
@@ -98,8 +102,37 @@ export function buildContainer(env: Env): ServerContainer {
   }
 }
 
+/**
+ * Whether this isolate has ever seen the cron fire, and when it started.
+ *
+ * A deployment with no cron trigger loses every safety net the `scheduled` handler is: the seal of
+ * a turn whose Durable Object alarm never fired, the repair of a seal an isolate died in the
+ * middle of, the re-run of a verdict cut short, and all of retention. None of that fails loudly —
+ * matches just stop advancing for the people in them — so it is worth one log line. Per isolate,
+ * which means a busy Worker says it a few times and then never again; that is the right volume for
+ * something whose remedy is four lines of `wrangler.toml`.
+ */
+let cronSeen = false
+let cronWatchStartedAt = 0
+const CRON_GRACE_MS = 60 * 60 * 1000
+
+function warnIfCronIsMissing(): void {
+  if (cronSeen) return
+  const now = Date.now()
+  if (cronWatchStartedAt === 0) {
+    cronWatchStartedAt = now
+    return
+  }
+  if (now - cronWatchStartedAt < CRON_GRACE_MS) return
+  cronWatchStartedAt = now
+  workerLogger.warn('cron trigger has not fired', {
+    hint: 'add [triggers] crons = ["*/5 * * * *"] to wrangler.toml; without it nothing seals a missed deadline, finishes an interrupted seal or collects retention',
+  })
+}
+
 export default {
   async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
+    warnIfCronIsMissing()
     return containerFor(env).app.fetch(request, env, ctx)
   },
   /** The cron safety net: expired deadlines, interrupted seals, and retention. */
@@ -108,21 +141,33 @@ export default {
     env: Env,
     ctx: ExecutionContext,
   ): Promise<void> {
+    cronSeen = true
     const { container } = containerFor(env)
     const { kernel } = container
+    // Each step in its own `try`, as the Node sweeper already does. One shared `catch` meant a
+    // throw inside the turn sweep also stopped both retention sweeps, every time the cron ran.
+    const step = async (name: string, run: () => Promise<void>): Promise<void> => {
+      try {
+        await run()
+      } catch (error: unknown) {
+        workerLogger.error('cron step failed', { step: name, error: String(error) })
+      }
+    }
     ctx.waitUntil(
       (async () => {
-        const { sealed, repaired } = await kernel.turns.sweep()
-        if (sealed > 0 || repaired > 0) {
-          workerLogger.info('cron advanced turns', { sealed, repaired })
-        }
-        await kernel.retention.collect()
+        await step('turns', async () => {
+          const { sealed, repaired } = await kernel.turns.sweep()
+          if (sealed > 0 || repaired > 0) {
+            workerLogger.info('cron advanced turns', { sealed, repaired })
+          }
+        })
+        await step('retention', () => kernel.retention.collect().then(() => undefined))
         // A separate database with a separate window; nothing about a match's retention decides
         // when a bug report and its R2 object go.
-        await container.bugReports?.collect()
-      })().catch((error: unknown) => {
-        workerLogger.error('cron sweep failed', { error: String(error) })
-      }),
+        await step('bugReports', async () => {
+          await container.bugReports?.collect()
+        })
+      })(),
     )
   },
 }

package/src/kernel.ts

@@ -43,6 +43,39 @@ export function hubFor(env: Env, matchId: string) {
   return env.MATCH_HUB.get(env.MATCH_HUB.idFromName(matchId))
 }
 
+/**
+ * How long a call into a match's Durable Object may take before the caller gives up on it.
+ *
+ * Every one of them is best effort by design — fan-out, arming a deadline, hanging up a revoked
+ * membership — and each has a safety net behind it: the stream's own catch-up drain, the sweep's
+ * `listExpiredOpen`, and the revoked token itself. What they did not have was an end. An object
+ * that is slow to wake, being relocated, or holding a lock kept the REQUEST waiting, so a seal
+ * completed in the database and the submitter saw a timeout.
+ */
+const HUB_CALL_TIMEOUT_MS = 5_000
+
+/**
+ * A best-effort call into a hub: bounded in time, and its failure logged rather than raised.
+ *
+ * The step it belongs to has already committed whatever was durable about it, so the only thing a
+ * failure here can still cost is latency somebody else's retry or the sweep already covers.
+ */
+async function tellHub(
+  env: Env,
+  call: { matchId: string; path: string; body: unknown },
+): Promise<void> {
+  const { matchId, path, body } = call
+  try {
+    await hubFor(env, matchId).fetch(`https://hub${path}`, {
+      method: 'POST',
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(HUB_CALL_TIMEOUT_MS),
+    })
+  } catch (error) {
+    workerLogger.warn('could not reach the match hub', { matchId, path, error: String(error) })
+  }
+}
+
 /**
  * Builds the kernel over D1 for one invocation. Fan-out and alarms cross into the match's Durable
  * Object; everything else is plain D1 through the shared SQLite repositories.
@@ -57,28 +90,25 @@ export function buildKernel(
 ): Kernel {
   const storage = createSqliteStorage(drizzle(env.DB, { schema: sqliteSchema }))
   const notifier: EventNotifier = overrides.notifier ?? {
-    notify: async (event) => {
-      await hubFor(env, event.matchId).fetch(`https://hub${HUB_PATHS.notify}`, {
-        method: 'POST',
-        body: JSON.stringify({ matchId: event.matchId }),
-      })
-    },
+    // The whole durable row, not just the match id. The object formats it once and hands the frame
+    // to every stream of the match, so a seal's burst costs it no reads at all; being told only
+    // which match had changed meant each subscriber queried D1 for rows the notification was
+    // already carrying. The event has been persisted before this runs, so the body crossing the
+    // isolate boundary is a copy of a fact, never a substitute for one.
+    notify: async (event) =>
+      tellHub(env, { matchId: event.matchId, path: HUB_PATHS.notify, body: event }),
   }
   const streams: StreamCloser = overrides.streams ?? {
-    close: async (input) => {
-      await hubFor(env, input.matchId).fetch(`https://hub${HUB_PATHS.disconnect}`, {
-        method: 'POST',
-        body: JSON.stringify(input),
-      })
-    },
+    close: async (input) =>
+      tellHub(env, { matchId: input.matchId, path: HUB_PATHS.disconnect, body: input }),
   }
   const scheduler: DeadlineScheduler = overrides.scheduler ?? {
-    schedule: async (input) => {
-      await hubFor(env, input.matchId).fetch(`https://hub${HUB_PATHS.schedule}`, {
-        method: 'POST',
-        body: JSON.stringify({ ...input, dueAt: input.dueAt.toISOString() }),
-      })
-    },
+    schedule: async (input) =>
+      tellHub(env, {
+        matchId: input.matchId,
+        path: HUB_PATHS.schedule,
+        body: { ...input, dueAt: input.dueAt.toISOString() },
+      }),
   }
   const days = (raw: string | undefined, fallback: number): number => {
     const value = Number(raw ?? fallback)
@@ -98,6 +128,10 @@ export function buildKernel(
         maxAgeMs: days(env.RETENTION_DAYS, DEFAULT_RETENTION_DAYS) * DAY_MS,
         abandonedLiveMaxAgeMs:
           days(env.ABANDONED_RETENTION_DAYS, DEFAULT_ABANDONED_RETENTION_DAYS) * DAY_MS,
+        // Twice the abandoned window, and without its roster test, which never collects an untimed
+        // match whose players' clients died without a `leave`.
+        silentLiveMaxAgeMs:
+          days(env.ABANDONED_RETENTION_DAYS, DEFAULT_ABANDONED_RETENTION_DAYS) * 2 * DAY_MS,
         batchSize: 50,
       },
     },