@hadoku/game-server 0.0.0-stage → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/base-path.d.ts +29 -0
- package/dist/base-path.js +30 -0
- package/dist/busy.d.ts +40 -0
- package/dist/busy.js +23 -0
- package/dist/capacity.d.ts +9 -0
- package/dist/capacity.js +6 -0
- package/dist/config.d.ts +247 -0
- package/dist/config.js +271 -0
- package/dist/edge-auth.d.ts +43 -0
- package/dist/edge-auth.js +47 -0
- package/dist/erase/anonymise.d.ts +7 -0
- package/dist/erase/anonymise.js +7 -0
- package/dist/erase/index.d.ts +18 -0
- package/dist/erase/index.js +20 -0
- package/dist/erase/legacy-ids.d.ts +7 -0
- package/dist/erase/legacy-ids.js +24 -0
- package/dist/erase/recordings.d.ts +26 -0
- package/dist/erase/recordings.js +128 -0
- package/dist/erase/rooms.d.ts +63 -0
- package/dist/erase/rooms.js +67 -0
- package/dist/gate.d.ts +27 -0
- package/dist/gate.js +15 -0
- package/dist/http/client-events.d.ts +25 -0
- package/dist/http/client-events.js +15 -0
- package/dist/http/page.d.ts +26 -0
- package/dist/http/page.js +35 -0
- package/dist/http/prepared.d.ts +14 -0
- package/dist/http/prepared.js +16 -0
- package/dist/http/request.d.ts +59 -0
- package/dist/http/request.js +37 -0
- package/dist/http/room-core.d.ts +160 -0
- package/dist/http/room-core.js +278 -0
- package/dist/http/trim.d.ts +23 -0
- package/dist/http/trim.js +76 -0
- package/dist/http/usage.d.ts +22 -0
- package/dist/http/usage.js +16 -0
- package/dist/kernel.d.ts +4 -0
- package/dist/kernel.js +26 -0
- package/dist/log.d.ts +9 -0
- package/dist/log.js +9 -0
- package/dist/people.d.ts +105 -0
- package/dist/people.js +145 -0
- package/dist/play/clock.d.ts +39 -0
- package/dist/play/clock.js +81 -0
- package/dist/play/diagnostics.d.ts +21 -0
- package/dist/play/diagnostics.js +35 -0
- package/dist/play/engine-runtime.d.ts +200 -0
- package/dist/play/engine-runtime.js +309 -0
- package/dist/play/game-adapter.d.ts +158 -0
- package/dist/play/game-adapter.js +1 -0
- package/dist/play/host.d.ts +482 -0
- package/dist/play/host.js +950 -0
- package/dist/play/jvm-transport.d.ts +25 -0
- package/dist/play/jvm-transport.js +1 -0
- package/dist/play/local-jvm.d.ts +325 -0
- package/dist/play/local-jvm.js +829 -0
- package/dist/play/match/hosted.d.ts +92 -0
- package/dist/play/match/hosted.js +151 -0
- package/dist/play/match/outcome.d.ts +26 -0
- package/dist/play/match/outcome.js +21 -0
- package/dist/play/match/shell.d.ts +378 -0
- package/dist/play/match/shell.js +613 -0
- package/dist/play/recording/retention.d.ts +36 -0
- package/dist/play/recording/retention.js +89 -0
- package/dist/play/recording/writer.d.ts +185 -0
- package/dist/play/recording/writer.js +90 -0
- package/dist/play/relay-socket.d.ts +128 -0
- package/dist/play/relay-socket.js +194 -0
- package/dist/play/runtimes/in-process.d.ts +47 -0
- package/dist/play/runtimes/in-process.js +43 -0
- package/dist/play/runtimes/relay-match.d.ts +166 -0
- package/dist/play/runtimes/relay-match.js +240 -0
- package/dist/play/runtimes/relay.d.ts +128 -0
- package/dist/play/runtimes/relay.js +183 -0
- package/dist/play/runtimes/streamed.d.ts +116 -0
- package/dist/play/runtimes/streamed.js +148 -0
- package/dist/play/start-queue.d.ts +65 -0
- package/dist/play/start-queue.js +187 -0
- package/dist/room/avatar.d.ts +1 -0
- package/dist/room/avatar.js +28 -0
- package/dist/room/caller.d.ts +21 -0
- package/dist/room/caller.js +1 -0
- package/dist/room/constants.d.ts +27 -0
- package/dist/room/constants.js +27 -0
- package/dist/room/events.d.ts +59 -0
- package/dist/room/events.js +174 -0
- package/dist/room/identity.d.ts +89 -0
- package/dist/room/identity.js +98 -0
- package/dist/room/lobby.d.ts +426 -0
- package/dist/room/lobby.js +880 -0
- package/dist/room/membership.d.ts +64 -0
- package/dist/room/membership.js +76 -0
- package/dist/room/provisioning.d.ts +22 -0
- package/dist/room/provisioning.js +46 -0
- package/dist/room/record.d.ts +63 -0
- package/dist/room/record.js +39 -0
- package/dist/room/room-list.d.ts +67 -0
- package/dist/room/room-list.js +66 -0
- package/dist/room/seats.d.ts +39 -0
- package/dist/room/seats.js +58 -0
- package/dist/room/session.d.ts +91 -0
- package/dist/room/session.js +156 -0
- package/dist/room/socket-auth.d.ts +2 -0
- package/dist/room/socket-auth.js +10 -0
- package/dist/room/watching.d.ts +35 -0
- package/dist/room/watching.js +72 -0
- package/dist/room/watchparty-link.d.ts +1 -0
- package/dist/room/watchparty-link.js +17 -0
- package/dist/stream/alloc.d.ts +115 -0
- package/dist/stream/alloc.js +368 -0
- package/dist/stream/index.d.ts +66 -0
- package/dist/stream/index.js +293 -0
- package/dist/telemetry.d.ts +46 -0
- package/dist/telemetry.js +25 -0
- package/dist/testPort.d.ts +1 -0
- package/dist/testPort.js +28 -0
- package/package.json +36 -4
- package/README.md +0 -3
|
@@ -0,0 +1,950 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a match's Forge runs: matches placed into shared `--host` bridge JVMs.
|
|
3
|
+
*
|
|
4
|
+
* ## Many matches per JVM
|
|
5
|
+
*
|
|
6
|
+
* Every bridge JVM is a `PlayBridge --host <hostId>` (`docs/spikes/multi-match-jvm.md`): it boots
|
|
7
|
+
* Forge's model once, says `warm`, and then takes any number of
|
|
8
|
+
* `{"t":"start","match":<hostId>,"config":{…}}` lines, each starting one match on a Forge server
|
|
9
|
+
* of its own. Every line it prints carries its match's id, commands are routed by it, and a
|
|
10
|
+
* match's `stop` ends that match alone, answered by `{"t":"stopped"}`. A `stop` addressed to the
|
|
11
|
+
* host id ends the JVM.
|
|
12
|
+
*
|
|
13
|
+
* This file is the coordinator. `jvmHostFactory` places each match into a JVM with room — the
|
|
14
|
+
* fullest warm one first, so idle JVMs empty out and retire — up to `perJvm` matches each
|
|
15
|
+
* (`MATCHES_PER_JVM`, 25), and opens a new JVM when every one is full. Where a JVM runs and how
|
|
16
|
+
* its line stream is reached is a `JvmLauncher`'s (`jvm-transport.ts`): this machine's (`local-jvm.ts`), by
|
|
17
|
+
* default; a claw's, through a remote agent.
|
|
18
|
+
*
|
|
19
|
+
* - **A match's exit is its `stopped`.** A match asked to stop whose `stopped` never comes is let
|
|
20
|
+
* go after `MATCH_STOP_MS` and logged (`match-stop-unanswered`); its JVM keeps the others.
|
|
21
|
+
* - **A JVM's death fails EVERY match on it**, with the JVM's reason (`jvm-lost`). Nothing on
|
|
22
|
+
* that JVM retries in place: a failed claw always fails over (the hosting rulings), and the
|
|
23
|
+
* failover acts on this signal.
|
|
24
|
+
* - **A JVM with no matches for `IDLE_JVM_MS` is retired**, unless it is the one JVM left with
|
|
25
|
+
* room (the warm one).
|
|
26
|
+
*
|
|
27
|
+
* ## The warm JVM
|
|
28
|
+
*
|
|
29
|
+
* One JVM with room is kept booted ahead of the next match (unless `warmJvm` is off): when the
|
|
30
|
+
* last JVM with room fills, another is launched behind it. A match placed into a warm JVM starts
|
|
31
|
+
* at once; one placed into a JVM still booting starts the moment it is warm (its `start` waits in
|
|
32
|
+
* the fifo). A JVM that dies before `warm` is a failed boot: the next is tried after
|
|
33
|
+
* `WARM_RETRY_MS`, and after `WARM_MAX_FAILURES` in a row no JVM is kept warm — a match still
|
|
34
|
+
* launches one when it needs one — which is logged.
|
|
35
|
+
*
|
|
36
|
+
* ## The ceiling: at most `MAX_CONCURRENT_MATCHES` matches being played
|
|
37
|
+
*
|
|
38
|
+
* **50** since 2026-10-28 (the operator: "if we hit just 50 concurrent matches that'd be fine"):
|
|
39
|
+
* two claws at `MATCHES_PER_JVM` each. It was 10 while every match was its own ~0.7–1 GB JVM.
|
|
40
|
+
* Counted over the matches still `starting` or `running` (a finished match keeps its slot in its
|
|
41
|
+
* JVM for `overGraceMs`, not a place in the ceiling). `socket.ts` refuses one more with a 503
|
|
42
|
+
* before any JVM is spent. Each person may play at most 5 at once
|
|
43
|
+
* (`http/routes.ts`). The JVM count is bounded separately (`JvmPoolOptions.maxJvms`). A JVM's
|
|
44
|
+
* heap is capped (`JVM_HEAP_MB` for `MATCHES_PER_JVM` matches, `local-jvm.ts`) so a runaway
|
|
45
|
+
* cannot take the box.
|
|
46
|
+
*
|
|
47
|
+
* Nothing outside this file knows a JVM exists: `match.ts` sees a `MatchHost` — send a command,
|
|
48
|
+
* get lines back, stop — and a `MatchHostFactory` makes one.
|
|
49
|
+
*
|
|
50
|
+
* ## A match outlives its server (Wave 22, §E)
|
|
51
|
+
*
|
|
52
|
+
* A deploy must not end live games. The JVM survives the server (`local-jvm.ts`, "A JVM
|
|
53
|
+
* outlives its server"), and the next server has to be able to rebuild each match on it: a
|
|
54
|
+
* re-attached JVM keeps Forge's games, but `Match`'s per-seat projectors, histories, cursors and
|
|
55
|
+
* parked asks lived in the process that died. So:
|
|
56
|
+
*
|
|
57
|
+
* - **Every protocol line and every command of a match is appended to its own stream**,
|
|
58
|
+
* `<dir>/<matchId>.jsonl`, as it is read or sent; a re-attach replays that file into the new
|
|
59
|
+
* `Match` before a single live line reaches it. Replay is deterministic, so each seat's `seq`
|
|
60
|
+
* comes back identical. A line is kept as the bridge wrote it, `match` stamp and all, through
|
|
61
|
+
* a descriptor held open for the match's life; each append is synchronous, so the file is
|
|
62
|
+
* whole for any reader the moment `persist` returns — this process's own `attach`, or the next
|
|
63
|
+
* server's after a crash.
|
|
64
|
+
* - **One record per JVM**, `<dir>/jvm.json` (`JvmRecord`): its host id, when it started, when it
|
|
65
|
+
* was handed over, and every match on it with its epoch, its seats' owners and its spec. A
|
|
66
|
+
* client therefore resyncs on the cursor it already held, and seat 2 cannot come back as seat 1.
|
|
67
|
+
* - **At boot** the launcher offers back every JVM still running (`running()`); each is attached
|
|
68
|
+
* at once, and every match its record lists is offered through `orphans()`. Lines arriving for
|
|
69
|
+
* a match not yet claimed are kept in its stream and replayed when `attach` claims it.
|
|
70
|
+
* - **A degraded match is never handed over** (hosting-test-plan H4): one whose stream could not
|
|
71
|
+
* be written has a hole a replay would turn into a board Forge does not have. Its stream is
|
|
72
|
+
* renamed out of the way (a rename needs no space on a disk that just filled), `handOver` refuses
|
|
73
|
+
* in words, and a later boot stops it instead of offering it.
|
|
74
|
+
*
|
|
75
|
+
* **Nothing keeps an orphan for ever**: a JVM `ORPHAN_MAX_MS` past its `orphanedAt` is retired,
|
|
76
|
+
* not re-attached; a match nobody claims within `ORPHAN_CLAIM_MS` of boot is stopped; and a
|
|
77
|
+
* re-attached match whose seats never come back is stopped by `Match.armRejoinGrace`.
|
|
78
|
+
*
|
|
79
|
+
* ## Going down: what must die before we do (Wave 22, lane U)
|
|
80
|
+
*
|
|
81
|
+
* A JVM with a match that was handed over outlives us on purpose; every other must not. Measured
|
|
82
|
+
* against a real match JVM (lane U): `{"t":"stop"}` → gone 9355 ms, SIGTERM 4322 ms, SIGKILL
|
|
83
|
+
* 0 ms; an idle JVM's host stop 314 ms. pm2 SIGKILLs us after 1.6 s, so there is no budget for a
|
|
84
|
+
* polite end, and `reapOnExit()` is the last word, **synchronous and immediate**: every JVM with
|
|
85
|
+
* no handed-over match is asked to go and killed now (confirmed gone, so its run directory is
|
|
86
|
+
* settled), and every JVM with one is let go with its `orphanedAt` stamped.
|
|
87
|
+
*
|
|
88
|
+
* ## Liveness
|
|
89
|
+
*
|
|
90
|
+
* A JVM's pid says it exists, not that it runs: a SIGSTOPped bridge kept its match `running` for
|
|
91
|
+
* good (hosting-test-plan C2). So the bridge says `alive` every 2 s (`PlayBridge.ALIVE_MS`) from a
|
|
92
|
+
* thread of its own, and a JVM this file has heard nothing from for `UNRESPONSIVE_MS` is killed
|
|
93
|
+
* and every match on it failed through the exit path, exactly as a crash.
|
|
94
|
+
*
|
|
95
|
+
* Why 40 s: a 30 s stop-the-world pause must not end a match, and the silence a pause of P makes
|
|
96
|
+
* is at most P + one interval (2 s), so 32 s; 40 s leaves 8 s for scheduling on a loaded host.
|
|
97
|
+
* Forge's own client timeout is raised far past this (`FORGE_HEARTBEAT_TIMEOUT_S`). A JVM
|
|
98
|
+
* re-attached from an earlier server is watched only from its first line.
|
|
99
|
+
*
|
|
100
|
+
* **Per match, there is no watchdog**, because the bridge reports nothing a watchdog could read:
|
|
101
|
+
* `alive` is one line per JVM from a thread of its own (`PlayBridge.startAlive`), and a match's
|
|
102
|
+
* own lines stop for as long as a player thinks, which is not a fault. A game thread that dies is
|
|
103
|
+
* already the bridge's to report (`MatchContext.fail` → `fatal`, then `stopped`). What a per-match
|
|
104
|
+
* watchdog would need is in "What remains" of `docs/spikes/multi-match-jvm.md`.
|
|
105
|
+
*
|
|
106
|
+
* ## AI cutoffs
|
|
107
|
+
*
|
|
108
|
+
* Forge gives each AI decision a 5 s wall-clock cutoff, and past it the AI plays its best so far:
|
|
109
|
+
* on a host short of CPU the AI plays worse, not slower. The bridge says `cutoff` each time
|
|
110
|
+
* (`PlayBridge.CutoffWatch`), for the match whose game was cut, or for the JVM when no match owns
|
|
111
|
+
* the thread. Like `alive` it is the pool's, never a match's: it is logged (`ai-cutoff`) and handed
|
|
112
|
+
* to `JvmPoolOptions.onCutoff`, which placement counts against the host, and it is neither kept in
|
|
113
|
+
* a stream nor forwarded, so a replay never counts one twice.
|
|
114
|
+
*
|
|
115
|
+
* ## The line protocol
|
|
116
|
+
*
|
|
117
|
+
* `bridge/PlayBridge.java` writes one JSON object per line on stdout and reads one per line on
|
|
118
|
+
* stdin. Its class comment has the list. Stdout carries nothing else, so a line that is not JSON
|
|
119
|
+
* is a bug and is logged, not parsed.
|
|
120
|
+
*/
|
|
121
|
+
import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync } from 'node:fs';
|
|
122
|
+
import { randomBytes } from 'node:crypto';
|
|
123
|
+
import { join } from 'node:path';
|
|
124
|
+
import { jsonLogger } from '@wolffm/service-kernel/log';
|
|
125
|
+
export { bridgeDiagnostics } from './diagnostics.js';
|
|
126
|
+
import { JVM_HEAP_MB, MATCHES_PER_JVM, localJvmLauncher, writeFileAtomic } from './local-jvm.js';
|
|
127
|
+
export { BRIDGE_CACHE_MAX_AGE_MS, BRIDGE_JVM_FLAGS, CRASH_RECORD_FILES, CRASH_RECORD_MAX_AGE_MS, CRASH_RECORDS_KEPT, FORGE_HEARTBEAT_TIMEOUT_S, FORGE_PREFERENCES, JVM_HEAP_MB, MATCHES_PER_JVM, bridgeLaunch, bridgeVersion, jvmHostConfig, liveJvmCount, stageRunDir, sweepRunDirs, writeFileAtomic, } from './local-jvm.js';
|
|
128
|
+
const LIFECYCLE_T = new Set(['ready', 'fatal', 'frame', 'snapshot']);
|
|
129
|
+
/** Narrows any engine's line onto the four the host itself understands. See {@link HostLifecycleLine}. */
|
|
130
|
+
export function isLifecycle(line) {
|
|
131
|
+
return LIFECYCLE_T.has(line.t);
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* A bot game's (`MatchSpec.headless`) exit when Forge ended it on its own: the bridge plays the whole
|
|
135
|
+
* best-of and stops, so this is the game finishing, never its host failing.
|
|
136
|
+
*/
|
|
137
|
+
export const PLAYED_OUT = 'played out';
|
|
138
|
+
/** See "The ceiling" in the module header. Also `PlayDeps.maxMatches`'s default. */
|
|
139
|
+
export const MAX_CONCURRENT_MATCHES = 50;
|
|
140
|
+
/** See "Liveness" in the module header. */
|
|
141
|
+
export const UNRESPONSIVE_MS = 40_000;
|
|
142
|
+
const LIVENESS_CHECK_MS = 1000;
|
|
143
|
+
/**
|
|
144
|
+
* A match asked to stop is let go after this long without its `stopped`. The bridge concedes,
|
|
145
|
+
* answers parked asks as aborted until the game is over (up to 10 s), then tears the match down:
|
|
146
|
+
* a real match took ~9.3 s (lane U).
|
|
147
|
+
*/
|
|
148
|
+
export const MATCH_STOP_MS = 30_000;
|
|
149
|
+
/** A JVM with no matches for this long is retired, unless it is the warm one. */
|
|
150
|
+
export const IDLE_JVM_MS = 10 * 60 * 1000;
|
|
151
|
+
/** After a JVM fails to boot, the next warm one is tried this much later… */
|
|
152
|
+
const WARM_RETRY_MS = 5000;
|
|
153
|
+
/** …and after this many failed boots in a row, no JVM is kept warm. */
|
|
154
|
+
export const WARM_MAX_FAILURES = 3;
|
|
155
|
+
/** `JvmRecord`'s file, in each JVM's directory. */
|
|
156
|
+
export const JVM_RECORD_FILE = 'jvm.json';
|
|
157
|
+
/** A JVM orphaned longer than this is retired, not re-attached. */
|
|
158
|
+
export const ORPHAN_MAX_MS = 12 * 60 * 60 * 1000;
|
|
159
|
+
/** A match nobody claimed this long after boot is stopped, so ignoring `orphans()` leaks nothing. */
|
|
160
|
+
export const ORPHAN_CLAIM_MS = 60_000;
|
|
161
|
+
/** A match's stream: every line and command, in order, for a re-attach to replay. */
|
|
162
|
+
export function streamFile(dir, matchId) {
|
|
163
|
+
return join(dir, `${matchId}.jsonl`);
|
|
164
|
+
}
|
|
165
|
+
/** Where a degraded match's stream is moved: kept for a reader, never replayed. */
|
|
166
|
+
function brokenStreamFile(dir, matchId) {
|
|
167
|
+
return join(dir, `${matchId}.broken.jsonl`);
|
|
168
|
+
}
|
|
169
|
+
const LINE_TYPES = new Set(['ready', 'frame', 'probe', 'gameover', 'log', 'fatal', 'rejected', 'stale', 'unsupported', 'pace', 'resent', 'applied', 'input', 'dropped', 'snapshot', 'impersonated', 'scan-overtaken', 'undo', 'rewind', 'rewound', 'log-truncated']);
|
|
170
|
+
/**
|
|
171
|
+
* The `bridge.stderr.txt` of the local JVM that hosts `matchId`, found by the `match-placed`
|
|
172
|
+
* line the coordinator logged to that match's log. A JVM's run directory is named for the JVM,
|
|
173
|
+
* not the match, so this is how a test or a dump reaches a match's stderr. Null before placement.
|
|
174
|
+
*/
|
|
175
|
+
export function matchStderrPath(workDir, log, matchId) {
|
|
176
|
+
const placed = log.find((l) => l.evt === 'match-placed' && l.matchId === matchId);
|
|
177
|
+
return typeof placed?.hostId === 'string' ? join(workDir, placed.hostId, 'bridge.stderr.txt') : null;
|
|
178
|
+
}
|
|
179
|
+
const isClockMs = (v) => Array.isArray(v) && v.length === 2 && v.every((ms) => Number.isInteger(ms) && ms >= 0);
|
|
180
|
+
/** A record's `snapshot` as a server actually wrote it: this was never checked, so it was never decoded either. */
|
|
181
|
+
const isTurnSnapshot = (v) => typeof v === 'object' && v !== null && typeof v.turn === 'number' && typeof v.state === 'string';
|
|
182
|
+
/** `<dir>/jvm.json`, or null when it is missing, unreadable or not a record. */
|
|
183
|
+
export function readJvmRecord(dir) {
|
|
184
|
+
let parsed;
|
|
185
|
+
try {
|
|
186
|
+
parsed = JSON.parse(readFileSync(join(dir, JVM_RECORD_FILE), 'utf8'));
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
if (typeof parsed !== 'object' || parsed === null)
|
|
192
|
+
return null;
|
|
193
|
+
const r = parsed;
|
|
194
|
+
if (typeof r.hostId !== 'string' || !Array.isArray(r.matches))
|
|
195
|
+
return null;
|
|
196
|
+
const matches = r.matches.filter((m) => typeof m === 'object' && m !== null && typeof m.matchId === 'string' && Array.isArray(m.spec?.seats) && m.spec.seats.length === 2);
|
|
197
|
+
return {
|
|
198
|
+
hostId: r.hostId,
|
|
199
|
+
startedAt: typeof r.startedAt === 'string' ? r.startedAt : new Date(0).toISOString(),
|
|
200
|
+
...(typeof r.orphanedAt === 'string' ? { orphanedAt: r.orphanedAt } : {}),
|
|
201
|
+
matches: matches.map((m) => ({
|
|
202
|
+
matchId: m.matchId,
|
|
203
|
+
startedAt: typeof m.startedAt === 'string' ? m.startedAt : new Date(0).toISOString(),
|
|
204
|
+
epoch: typeof m.epoch === 'string' ? m.epoch : null,
|
|
205
|
+
owners: Array.isArray(m.owners) ? m.owners.map((o) => (typeof o === 'string' ? o : null)) : [],
|
|
206
|
+
// Durable's own field: opaque to this file, so it is passed through as whatever the
|
|
207
|
+
// record holds — absent on a record written before checkpoints existed decodes as
|
|
208
|
+
// null, never dropped.
|
|
209
|
+
checkpoint: m.checkpoint ?? null,
|
|
210
|
+
// `remember` has written this since e308cdf9b, but nothing read it back until now — an
|
|
211
|
+
// orphan re-attached after a restart lost its turn-start snapshot.
|
|
212
|
+
...(isTurnSnapshot(m.snapshot) ? { snapshot: m.snapshot } : {}),
|
|
213
|
+
spec: m.spec,
|
|
214
|
+
...(isClockMs(m.clockMs) ? { clockMs: m.clockMs } : {}),
|
|
215
|
+
})),
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
/** The config object the bridge reads from a `start` line. `matchId` first: a refusal quotes the line's first 200 characters. */
|
|
219
|
+
function bridgeConfig(matchId, spec, aiFlatMc) {
|
|
220
|
+
return { matchId, ...spec, aiFlatMc };
|
|
221
|
+
}
|
|
222
|
+
/** The match id a refused `start` named, from the `rejected` line's quote of it. */
|
|
223
|
+
function refusedStart(command) {
|
|
224
|
+
if (typeof command !== 'string' || !command.startsWith('{"t":"start"'))
|
|
225
|
+
return null;
|
|
226
|
+
return /"matchId":"([^"]+)"/.exec(command)?.[1] ?? null;
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Matches placed into shared `--host` JVMs. A match's own `HostEvents.log` receives what is about
|
|
230
|
+
* that match; what is about a JVM goes to `options.log`.
|
|
231
|
+
*/
|
|
232
|
+
export function jvmHostFactory(config, options = {}) {
|
|
233
|
+
const warmWanted = options.warm ?? config.warmJvm;
|
|
234
|
+
const maxJvmsOption = options.maxJvms ?? MAX_CONCURRENT_MATCHES;
|
|
235
|
+
const maxJvms = () => (typeof maxJvmsOption === 'number' ? maxJvmsOption : maxJvmsOption());
|
|
236
|
+
const perJvm = config.perJvm ?? MATCHES_PER_JVM;
|
|
237
|
+
const idleMs = options.idleMs ?? IDLE_JVM_MS;
|
|
238
|
+
const matchStopMs = options.matchStopMs ?? MATCH_STOP_MS;
|
|
239
|
+
const unresponsiveMs = config.unresponsiveMs ?? UNRESPONSIVE_MS;
|
|
240
|
+
const poolLog = options.log ?? jsonLogger('cardtable-play');
|
|
241
|
+
const launcher = options.launcher ?? localJvmLauncher(config, { sweep: options.sweep ?? true, log: poolLog });
|
|
242
|
+
const jvms = new Set();
|
|
243
|
+
/** Orphans from an earlier server, not yet claimed. */
|
|
244
|
+
const pending = new Map();
|
|
245
|
+
const roomWatchers = [];
|
|
246
|
+
let bootFailures = 0;
|
|
247
|
+
let retry = null;
|
|
248
|
+
let claimDeadline = null;
|
|
249
|
+
let closed = false;
|
|
250
|
+
/** `reapOnExit` is running: a JVM's end is not news to a `Match` in a process that is going. */
|
|
251
|
+
let reaping = false;
|
|
252
|
+
const notifyRoom = () => {
|
|
253
|
+
for (const listener of roomWatchers)
|
|
254
|
+
listener();
|
|
255
|
+
};
|
|
256
|
+
const liveJvms = () => [...jvms].filter((j) => !j.exited && !j.retiring && !j.released);
|
|
257
|
+
const hasRoom = (jvm) => jvm.matches.size < perJvm && !jvm.draining;
|
|
258
|
+
/* ── The record and the streams ── */
|
|
259
|
+
function writeRecord(jvm) {
|
|
260
|
+
if (jvm.released || jvm.exited)
|
|
261
|
+
return;
|
|
262
|
+
// A match being stopped is not one a later server should take back: it would replay into a
|
|
263
|
+
// new `Match` only to meet its `stopped`.
|
|
264
|
+
const hosted = [...jvm.matches.values()].filter((h) => !h.ended && !h.stopping && h.degraded === null);
|
|
265
|
+
const record = {
|
|
266
|
+
hostId: jvm.hostId,
|
|
267
|
+
startedAt: jvm.startedAt,
|
|
268
|
+
...(jvm.orphanedAt === null ? {} : { orphanedAt: jvm.orphanedAt }),
|
|
269
|
+
matches: hosted.map((h) => h.entry),
|
|
270
|
+
};
|
|
271
|
+
try {
|
|
272
|
+
writeFileAtomic(join(jvm.dir, JVM_RECORD_FILE), `${JSON.stringify(record, null, 2)}\n`);
|
|
273
|
+
for (const h of hosted)
|
|
274
|
+
h.onDisk = JSON.stringify(h.entry);
|
|
275
|
+
}
|
|
276
|
+
catch (err) {
|
|
277
|
+
// The old record is whole (`writeFileAtomic`), so it names each match as it was. One whose
|
|
278
|
+
// entry changed since cannot be re-attached as it is now, and must not be handed over.
|
|
279
|
+
poolLog({ evt: 'jvm-record-failed', id: jvm.hostId, error: err.message });
|
|
280
|
+
for (const h of hosted)
|
|
281
|
+
if (h.onDisk !== JSON.stringify(h.entry))
|
|
282
|
+
degrade(h, `its record could not be written (${err.message})`);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Keep one entry — a bridge line as it came, or `{ c: command }` — in the match's stream. An
|
|
287
|
+
* entry that cannot be kept degrades the match.
|
|
288
|
+
*/
|
|
289
|
+
function persist(h, json) {
|
|
290
|
+
if (h.degraded !== null)
|
|
291
|
+
return;
|
|
292
|
+
try {
|
|
293
|
+
h.stream ??= openSync(streamFile(h.jvm.dir, h.id), 'a');
|
|
294
|
+
appendFileSync(h.stream, `${json}\n`);
|
|
295
|
+
}
|
|
296
|
+
catch (err) {
|
|
297
|
+
degrade(h, `its stream could not be written (${err.message})`);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
/** Let go of the match's stream descriptor: it has ended, degraded, or been handed over. */
|
|
301
|
+
function closeStream(h) {
|
|
302
|
+
if (h.stream === null)
|
|
303
|
+
return;
|
|
304
|
+
const fd = h.stream;
|
|
305
|
+
h.stream = null;
|
|
306
|
+
try {
|
|
307
|
+
closeSync(fd);
|
|
308
|
+
}
|
|
309
|
+
catch (err) {
|
|
310
|
+
(h.events?.log ?? poolLog)({ evt: 'match-stream-close-failed', matchId: h.id, error: err.message });
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
/** The match has a hole now, so it can never be replayed: it plays on here, but cannot outlive us. */
|
|
314
|
+
function degrade(h, cause) {
|
|
315
|
+
if (h.degraded !== null)
|
|
316
|
+
return;
|
|
317
|
+
h.degraded = `${cause}, so a replay would give the seats a board Forge does not have`;
|
|
318
|
+
const log = h.events?.log ?? poolLog;
|
|
319
|
+
log({ evt: 'match-degraded', level: 'error', matchId: h.id, hostId: h.jvm.hostId, why: h.degraded });
|
|
320
|
+
closeStream(h);
|
|
321
|
+
try {
|
|
322
|
+
renameSync(streamFile(h.jvm.dir, h.id), brokenStreamFile(h.jvm.dir, h.id));
|
|
323
|
+
}
|
|
324
|
+
catch (err) {
|
|
325
|
+
// No stream yet, or the rename failed: either way no replay will find a whole one.
|
|
326
|
+
log({ evt: 'match-stream-rename-failed', matchId: h.id, error: err.message });
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
/** Feed a match's stream back to it. Returns how many entries it had. */
|
|
330
|
+
function replay(h, events) {
|
|
331
|
+
let text;
|
|
332
|
+
try {
|
|
333
|
+
text = readFileSync(streamFile(h.jvm.dir, h.id), 'utf8');
|
|
334
|
+
}
|
|
335
|
+
catch (err) {
|
|
336
|
+
events.log({ evt: 'replay-unreadable', matchId: h.id, error: err.message });
|
|
337
|
+
return 0;
|
|
338
|
+
}
|
|
339
|
+
let replayed = 0;
|
|
340
|
+
for (const raw of text.split('\n')) {
|
|
341
|
+
if (raw === '')
|
|
342
|
+
continue;
|
|
343
|
+
let entry;
|
|
344
|
+
try {
|
|
345
|
+
entry = JSON.parse(raw);
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
// Only ever the last line, and only if the old process died between the write and the
|
|
349
|
+
// newline: the seat is one frame behind, and its next delta will say so.
|
|
350
|
+
events.log({ evt: 'replay-truncated', matchId: h.id, bytes: raw.length });
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
if (typeof entry !== 'object' || entry === null)
|
|
354
|
+
continue;
|
|
355
|
+
const sent = entry.c;
|
|
356
|
+
if (typeof sent === 'object' && sent !== null) {
|
|
357
|
+
events.replayed?.(sent);
|
|
358
|
+
replayed += 1;
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
if (!LINE_TYPES.has(String(entry.t)))
|
|
362
|
+
continue;
|
|
363
|
+
// Kept with the bridge's `match` stamp, which `onText` takes off a live line.
|
|
364
|
+
const { match: _stamp, ...line } = entry;
|
|
365
|
+
events.line(line);
|
|
366
|
+
replayed += 1;
|
|
367
|
+
}
|
|
368
|
+
return replayed;
|
|
369
|
+
}
|
|
370
|
+
/* ── JVMs ── */
|
|
371
|
+
function track(hostId, dir, transport, from) {
|
|
372
|
+
const jvm = {
|
|
373
|
+
hostId,
|
|
374
|
+
dir,
|
|
375
|
+
transport,
|
|
376
|
+
startedAt: from.startedAt,
|
|
377
|
+
orphanedAt: from.orphanedAt,
|
|
378
|
+
warm: from.warm,
|
|
379
|
+
exited: false,
|
|
380
|
+
retiring: false,
|
|
381
|
+
draining: false,
|
|
382
|
+
adopted: from.adopted ?? false,
|
|
383
|
+
released: false,
|
|
384
|
+
matches: new Map(),
|
|
385
|
+
lastHeard: Date.now(),
|
|
386
|
+
lastCheck: Date.now(),
|
|
387
|
+
livenessArmed: !from.warm,
|
|
388
|
+
unresponsive: null,
|
|
389
|
+
liveness: null,
|
|
390
|
+
idle: null,
|
|
391
|
+
};
|
|
392
|
+
jvms.add(jvm);
|
|
393
|
+
transport.onLine((text) => onText(jvm, text));
|
|
394
|
+
transport.onExit((reason) => onJvmExit(jvm, reason));
|
|
395
|
+
watchLiveness(jvm);
|
|
396
|
+
return jvm;
|
|
397
|
+
}
|
|
398
|
+
/** Boot a new JVM. Throws what the launcher throws. */
|
|
399
|
+
function launch() {
|
|
400
|
+
const hostId = `h-${randomBytes(6).toString('hex')}`;
|
|
401
|
+
const transport = launcher.launch(hostId, { xmxMb: config.xmxMb ?? JVM_HEAP_MB });
|
|
402
|
+
const dir = join(config.workDir, hostId);
|
|
403
|
+
mkdirSync(dir, { recursive: true });
|
|
404
|
+
const jvm = track(hostId, dir, transport, { warm: false, startedAt: new Date().toISOString(), orphanedAt: null });
|
|
405
|
+
writeRecord(jvm);
|
|
406
|
+
poolLog({ evt: 'jvm-launched', id: hostId, jvms: jvms.size });
|
|
407
|
+
notifyRoom();
|
|
408
|
+
return jvm;
|
|
409
|
+
}
|
|
410
|
+
/**
|
|
411
|
+
* Kill a JVM that has said nothing for `unresponsiveMs`. A check that itself ran late means
|
|
412
|
+
* THIS process was not reading — its event loop stalled — so the clock restarts rather than
|
|
413
|
+
* blaming the JVM.
|
|
414
|
+
*/
|
|
415
|
+
function watchLiveness(jvm) {
|
|
416
|
+
const timer = setInterval(() => {
|
|
417
|
+
const now = Date.now();
|
|
418
|
+
if (now - jvm.lastCheck > 3 * LIVENESS_CHECK_MS)
|
|
419
|
+
jvm.lastHeard = now;
|
|
420
|
+
jvm.lastCheck = now;
|
|
421
|
+
if (!jvm.livenessArmed || jvm.exited || jvm.released)
|
|
422
|
+
return;
|
|
423
|
+
const silentMs = now - jvm.lastHeard;
|
|
424
|
+
if (silentMs < unresponsiveMs)
|
|
425
|
+
return;
|
|
426
|
+
clearInterval(timer);
|
|
427
|
+
jvm.liveness = null;
|
|
428
|
+
jvm.unresponsive = `the JVM stopped answering (nothing from it for ${Math.round(silentMs / 1000)} s), so it was killed`;
|
|
429
|
+
poolLog({ evt: 'jvm-unresponsive', id: jvm.hostId, silentMs, matches: [...jvm.matches.keys()] });
|
|
430
|
+
jvm.transport.kill();
|
|
431
|
+
}, LIVENESS_CHECK_MS);
|
|
432
|
+
timer.unref();
|
|
433
|
+
jvm.liveness = timer;
|
|
434
|
+
}
|
|
435
|
+
function onText(jvm, text) {
|
|
436
|
+
jvm.lastHeard = Date.now();
|
|
437
|
+
jvm.livenessArmed = true;
|
|
438
|
+
// Liveness only: never kept, never forwarded. The bridge writes `t` first (`PlayBridge.line`).
|
|
439
|
+
if (text.startsWith('{"t":"alive"'))
|
|
440
|
+
return;
|
|
441
|
+
let value;
|
|
442
|
+
try {
|
|
443
|
+
value = text.startsWith('{') ? JSON.parse(text) : null;
|
|
444
|
+
}
|
|
445
|
+
catch {
|
|
446
|
+
value = null;
|
|
447
|
+
}
|
|
448
|
+
if (typeof value !== 'object' || value === null || typeof value.t !== 'string') {
|
|
449
|
+
poolLog({ evt: 'bridge-noise', id: jvm.hostId, text: text.slice(0, 200) });
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
const { match, ...line } = value;
|
|
453
|
+
if (line.t === 'cutoff') {
|
|
454
|
+
// Counted whoever it names: a match that has just ended was still cut short on this JVM.
|
|
455
|
+
aiCutoff(jvm, typeof match === 'string' && match !== jvm.hostId ? match : null, typeof line.where === 'string' ? line.where : null);
|
|
456
|
+
return;
|
|
457
|
+
}
|
|
458
|
+
if (match === jvm.hostId) {
|
|
459
|
+
onHostLine(jvm, line);
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
const h = typeof match === 'string' ? jvm.matches.get(match) : undefined;
|
|
463
|
+
if (h === undefined || h.ended) {
|
|
464
|
+
poolLog({ evt: 'bridge-wrong-match', id: jvm.hostId, got: match, line: line.t });
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
if (line.t === 'stopped') {
|
|
468
|
+
matchEnded(h);
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
471
|
+
if (!LINE_TYPES.has(line.t)) {
|
|
472
|
+
poolLog({ evt: 'bridge-noise', id: jvm.hostId, text: text.slice(0, 200) });
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
475
|
+
// A JVM given up on (abandoned with its claw, or killed as silent) is gone for its matches: they
|
|
476
|
+
// have moved, and its claw's records may already be moved aside (2026-11-09: a late line wrote
|
|
477
|
+
// into a renamed run directory and logged its moved match as degraded).
|
|
478
|
+
if (jvm.unresponsive !== null)
|
|
479
|
+
return;
|
|
480
|
+
if (line.t === 'fatal')
|
|
481
|
+
h.fatal = String(line.message);
|
|
482
|
+
persist(h, text);
|
|
483
|
+
if (h.events !== null && !h.handed)
|
|
484
|
+
h.events.line(line);
|
|
485
|
+
}
|
|
486
|
+
function aiCutoff(jvm, matchId, where) {
|
|
487
|
+
const h = matchId === null ? undefined : jvm.matches.get(matchId);
|
|
488
|
+
(h?.events?.log ?? poolLog)({ evt: 'ai-cutoff', id: jvm.hostId, matchId, where });
|
|
489
|
+
options.onCutoff?.({ hostId: jvm.hostId, matchId, where });
|
|
490
|
+
}
|
|
491
|
+
/** A line about the JVM itself: `warm`, or a `start` it refused. */
|
|
492
|
+
function onHostLine(jvm, line) {
|
|
493
|
+
if (line.t === 'warm') {
|
|
494
|
+
jvm.warm = true;
|
|
495
|
+
bootFailures = 0;
|
|
496
|
+
poolLog({ evt: 'jvm-warm', id: jvm.hostId, bootMs: line.bootMs, matches: jvm.matches.size });
|
|
497
|
+
notifyRoom();
|
|
498
|
+
replenish();
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
const refused = line.t === 'rejected' ? refusedStart(line.command) : null;
|
|
502
|
+
const h = refused === null ? undefined : jvm.matches.get(refused);
|
|
503
|
+
if (h !== undefined && !h.ended) {
|
|
504
|
+
// A one-match JVM said `fatal` and exited for this; here the JVM goes on and only the match ends.
|
|
505
|
+
const message = `Forge refused the match: ${String(line.message)}`;
|
|
506
|
+
h.fatal = message;
|
|
507
|
+
h.events?.line({ t: 'fatal', message });
|
|
508
|
+
matchEnded(h);
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
poolLog({ evt: 'bridge-host-line', id: jvm.hostId, line });
|
|
512
|
+
}
|
|
513
|
+
/** The match is over in Forge: its `stopped` came, or nobody is waiting for it any more. */
|
|
514
|
+
function matchEnded(h) {
|
|
515
|
+
if (h.ended)
|
|
516
|
+
return;
|
|
517
|
+
h.ended = true;
|
|
518
|
+
if (h.stopTimer !== null)
|
|
519
|
+
clearTimeout(h.stopTimer);
|
|
520
|
+
closeStream(h);
|
|
521
|
+
pending.delete(h.id);
|
|
522
|
+
const jvm = h.jvm;
|
|
523
|
+
jvm.matches.delete(h.id);
|
|
524
|
+
writeRecord(jvm);
|
|
525
|
+
if (h.events !== null && !h.handed) {
|
|
526
|
+
h.events.exit(h.stopping
|
|
527
|
+
? 'stopped'
|
|
528
|
+
: h.fatal !== null
|
|
529
|
+
? `Forge failed: ${h.fatal}`
|
|
530
|
+
: h.entry.spec.headless !== undefined
|
|
531
|
+
? PLAYED_OUT
|
|
532
|
+
: 'Forge ended the match without being asked to');
|
|
533
|
+
}
|
|
534
|
+
if (jvm.matches.size === 0)
|
|
535
|
+
idle(jvm);
|
|
536
|
+
notifyRoom();
|
|
537
|
+
replenish();
|
|
538
|
+
}
|
|
539
|
+
function onJvmExit(jvm, reason) {
|
|
540
|
+
if (jvm.exited)
|
|
541
|
+
return;
|
|
542
|
+
jvm.exited = true;
|
|
543
|
+
jvms.delete(jvm);
|
|
544
|
+
if (jvm.liveness !== null)
|
|
545
|
+
clearInterval(jvm.liveness);
|
|
546
|
+
if (jvm.idle !== null)
|
|
547
|
+
clearTimeout(jvm.idle);
|
|
548
|
+
const why = jvm.unresponsive ?? reason;
|
|
549
|
+
const lost = [...jvm.matches.values()].filter((h) => !h.ended);
|
|
550
|
+
for (const h of lost) {
|
|
551
|
+
h.ended = true;
|
|
552
|
+
if (h.stopTimer !== null)
|
|
553
|
+
clearTimeout(h.stopTimer);
|
|
554
|
+
closeStream(h);
|
|
555
|
+
pending.delete(h.id);
|
|
556
|
+
if (!reaping && h.events !== null && !h.handed)
|
|
557
|
+
h.events.exit(h.stopping ? 'stopped' : why);
|
|
558
|
+
}
|
|
559
|
+
jvm.matches.clear();
|
|
560
|
+
if (reaping)
|
|
561
|
+
return;
|
|
562
|
+
if (lost.some((h) => !h.stopping)) {
|
|
563
|
+
poolLog({ evt: 'jvm-lost', level: 'error', id: jvm.hostId, reason: why, matches: lost.map((h) => h.id) });
|
|
564
|
+
}
|
|
565
|
+
else {
|
|
566
|
+
poolLog({ evt: 'jvm-ended', id: jvm.hostId, reason: why });
|
|
567
|
+
}
|
|
568
|
+
if (!jvm.warm && !jvm.retiring) {
|
|
569
|
+
bootFailures += 1;
|
|
570
|
+
poolLog({ evt: 'jvm-boot-failed', id: jvm.hostId, reason: why, failures: bootFailures });
|
|
571
|
+
if (bootFailures === WARM_MAX_FAILURES) {
|
|
572
|
+
poolLog({ evt: 'warm-disabled', reason: `${bootFailures} JVMs died before warm; a match launches its own` });
|
|
573
|
+
}
|
|
574
|
+
notifyRoom();
|
|
575
|
+
retryLater();
|
|
576
|
+
return;
|
|
577
|
+
}
|
|
578
|
+
notifyRoom();
|
|
579
|
+
replenish();
|
|
580
|
+
}
|
|
581
|
+
/** Ask the JVM to go, and make sure it does. Its exit is not a failure of anything. */
|
|
582
|
+
function retire(jvm, why) {
|
|
583
|
+
if (jvm.retiring || jvm.exited || jvm.released)
|
|
584
|
+
return;
|
|
585
|
+
jvm.retiring = true;
|
|
586
|
+
if (jvm.idle !== null)
|
|
587
|
+
clearTimeout(jvm.idle);
|
|
588
|
+
poolLog({ evt: 'jvm-retired', id: jvm.hostId, why });
|
|
589
|
+
jvm.transport.send(JSON.stringify({ t: 'stop', match: jvm.hostId }));
|
|
590
|
+
const kill = setTimeout(() => jvm.transport.kill(), config.stopGraceMs);
|
|
591
|
+
kill.unref();
|
|
592
|
+
notifyRoom();
|
|
593
|
+
}
|
|
594
|
+
/** The JVM has no matches: retire it now, or after `idleMs` unless it is the warm one then. */
|
|
595
|
+
function idle(jvm) {
|
|
596
|
+
if (jvm.matches.size > 0 || jvm.exited || jvm.retiring || jvm.released)
|
|
597
|
+
return;
|
|
598
|
+
if (closed || !warmWanted || jvm.draining) {
|
|
599
|
+
retire(jvm, closed ? 'closing' : jvm.draining ? 'drained' : 'no match left, and no JVM is kept warm');
|
|
600
|
+
return;
|
|
601
|
+
}
|
|
602
|
+
if (jvm.idle !== null)
|
|
603
|
+
clearTimeout(jvm.idle);
|
|
604
|
+
jvm.idle = setTimeout(() => {
|
|
605
|
+
jvm.idle = null;
|
|
606
|
+
if (jvm.matches.size > 0)
|
|
607
|
+
return;
|
|
608
|
+
if (liveJvms().some((other) => other !== jvm && hasRoom(other)))
|
|
609
|
+
retire(jvm, `no match for ${idleMs} ms`);
|
|
610
|
+
else
|
|
611
|
+
idle(jvm);
|
|
612
|
+
}, idleMs);
|
|
613
|
+
jvm.idle.unref();
|
|
614
|
+
}
|
|
615
|
+
function retryLater() {
|
|
616
|
+
if (retry !== null || closed)
|
|
617
|
+
return;
|
|
618
|
+
retry = setTimeout(() => {
|
|
619
|
+
retry = null;
|
|
620
|
+
replenish();
|
|
621
|
+
}, options.warmRetryMs ?? WARM_RETRY_MS);
|
|
622
|
+
retry.unref();
|
|
623
|
+
}
|
|
624
|
+
/**
|
|
625
|
+
* Keep one JVM with room — but launch it only while no JVM is booting. Starts fill a booting JVM
|
|
626
|
+
* as readily as a warm one, so without that bound a burst would launch one warm JVM per
|
|
627
|
+
* `perJvm` starts, all booting at once, past `start-queue.ts`'s limit (a burst of 8 at 2 per
|
|
628
|
+
* JVM had 4 booting together on a claw). It is asked again when a JVM turns warm. Never throws: it runs after a placement, on a JVM's exit and from a
|
|
629
|
+
* timer, and none of those may fail for it (a full disk made staging throw from a timer, which
|
|
630
|
+
* is the whole server). A launch that throws counts as a failed boot.
|
|
631
|
+
*/
|
|
632
|
+
function replenish() {
|
|
633
|
+
if (!warmWanted || closed || retry !== null || bootFailures >= WARM_MAX_FAILURES)
|
|
634
|
+
return;
|
|
635
|
+
if (liveJvms().some((j) => hasRoom(j) || !j.warm) || jvms.size >= maxJvms())
|
|
636
|
+
return;
|
|
637
|
+
try {
|
|
638
|
+
launch();
|
|
639
|
+
}
|
|
640
|
+
catch (err) {
|
|
641
|
+
bootFailures += 1;
|
|
642
|
+
poolLog({ evt: 'jvm-launch-failed', error: err.message, failures: bootFailures });
|
|
643
|
+
if (bootFailures === WARM_MAX_FAILURES)
|
|
644
|
+
poolLog({ evt: 'warm-disabled', reason: `${bootFailures} JVMs could not be launched; a match launches its own` });
|
|
645
|
+
retryLater();
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
/* ── Matches ── */
|
|
649
|
+
function stopMatch(h) {
|
|
650
|
+
if (h.stopping || h.ended)
|
|
651
|
+
return;
|
|
652
|
+
h.stopping = true;
|
|
653
|
+
// Out of the record now, not at its `stopped`: a JVM killed before that (remote, unconfirmed)
|
|
654
|
+
// would otherwise offer the match to the next server.
|
|
655
|
+
writeRecord(h.jvm);
|
|
656
|
+
h.jvm.transport.send(JSON.stringify({ t: 'stop', match: h.id }));
|
|
657
|
+
h.stopTimer = setTimeout(() => {
|
|
658
|
+
(h.events?.log ?? poolLog)({ evt: 'match-stop-unanswered', matchId: h.id, hostId: h.jvm.hostId, afterMs: matchStopMs });
|
|
659
|
+
matchEnded(h);
|
|
660
|
+
}, matchStopMs);
|
|
661
|
+
h.stopTimer.unref();
|
|
662
|
+
}
|
|
663
|
+
function hostFace(h) {
|
|
664
|
+
return {
|
|
665
|
+
send: (command) => {
|
|
666
|
+
if (command.t === 'stop') {
|
|
667
|
+
stopMatch(h);
|
|
668
|
+
return;
|
|
669
|
+
}
|
|
670
|
+
if (h.ended || h.jvm.exited)
|
|
671
|
+
return;
|
|
672
|
+
// Kept in the same stream, in the same order, as the lines that came back: a replay
|
|
673
|
+
// has to know which asks the old server had already answered.
|
|
674
|
+
persist(h, JSON.stringify({ c: command }));
|
|
675
|
+
h.jvm.transport.send(JSON.stringify({ ...command, match: h.id }));
|
|
676
|
+
},
|
|
677
|
+
stop: () => stopMatch(h),
|
|
678
|
+
remember: (state) => {
|
|
679
|
+
h.entry = {
|
|
680
|
+
...h.entry,
|
|
681
|
+
epoch: state.epoch,
|
|
682
|
+
owners: state.owners,
|
|
683
|
+
// Whatever the caller's own game last had of its `{ t: 'checkpoint' }` line — see `Durable`'s own doc.
|
|
684
|
+
checkpoint: state.checkpoint ?? null,
|
|
685
|
+
...(state.snapshot === undefined ? {} : { snapshot: state.snapshot }),
|
|
686
|
+
...(state.clockMs === undefined ? {} : { clockMs: state.clockMs }),
|
|
687
|
+
};
|
|
688
|
+
writeRecord(h.jvm);
|
|
689
|
+
},
|
|
690
|
+
handOver: () => {
|
|
691
|
+
if (h.degraded !== null && !h.ended && !h.jvm.exited) {
|
|
692
|
+
(h.events?.log ?? poolLog)({
|
|
693
|
+
evt: 'jvm-handover-refused',
|
|
694
|
+
matchId: h.id,
|
|
695
|
+
hostId: h.jvm.hostId,
|
|
696
|
+
why: `this match cannot be kept across a restart: ${h.degraded}`,
|
|
697
|
+
});
|
|
698
|
+
return false;
|
|
699
|
+
}
|
|
700
|
+
if (h.ended || h.stopping || h.jvm.exited || h.jvm.released || h.events === null)
|
|
701
|
+
return false;
|
|
702
|
+
// A bot game is not kept: nobody can watch it, and the room plays it again (`play/botgames.ts`).
|
|
703
|
+
if (h.entry.spec.headless !== undefined)
|
|
704
|
+
return false;
|
|
705
|
+
h.handed = true;
|
|
706
|
+
return true;
|
|
707
|
+
},
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
/** Where a new match goes: the fullest warm JVM with room, else one booting, else none. */
|
|
711
|
+
/**
|
|
712
|
+
* A JVM with room for `matchId`. Never one still holding a match of that id: a restored match
|
|
713
|
+
* keeps its id (`MatchRegistry.restore`), and a JVM routes its lines by id, so the new game must
|
|
714
|
+
* not land beside the old one in the moment before the old one is gone.
|
|
715
|
+
*/
|
|
716
|
+
function placeFor(matchId) {
|
|
717
|
+
const open = liveJvms().filter((j) => hasRoom(j) && (matchId === undefined || !j.matches.has(matchId)));
|
|
718
|
+
const warm = open.filter((j) => j.warm).sort((a, b) => b.matches.size - a.matches.size);
|
|
719
|
+
return warm[0] ?? open[0] ?? null;
|
|
720
|
+
}
|
|
721
|
+
/**
|
|
722
|
+
* Throws when the match cannot be given a JVM (every JVM full at `maxJvms`, or a launch that
|
|
723
|
+
* throws), and leaves nothing behind. The caller fails the match (`Match.start`).
|
|
724
|
+
*/
|
|
725
|
+
const factory = ((matchId, spec, events) => {
|
|
726
|
+
let jvm = placeFor(matchId);
|
|
727
|
+
if (jvm === null) {
|
|
728
|
+
if (jvms.size >= maxJvms())
|
|
729
|
+
throw new Error(`every Forge JVM is full (${jvms.size} of ${maxJvms()}, ${perJvm} matches each)`);
|
|
730
|
+
jvm = launch();
|
|
731
|
+
}
|
|
732
|
+
if (jvm.idle !== null)
|
|
733
|
+
clearTimeout(jvm.idle);
|
|
734
|
+
jvm.idle = null;
|
|
735
|
+
const h = {
|
|
736
|
+
id: matchId,
|
|
737
|
+
jvm,
|
|
738
|
+
events,
|
|
739
|
+
entry: { matchId, startedAt: new Date().toISOString(), epoch: null, owners: [], checkpoint: null, spec },
|
|
740
|
+
onDisk: null,
|
|
741
|
+
stopping: false,
|
|
742
|
+
ended: false,
|
|
743
|
+
handed: false,
|
|
744
|
+
degraded: null,
|
|
745
|
+
fatal: null,
|
|
746
|
+
stopTimer: null,
|
|
747
|
+
stream: null,
|
|
748
|
+
};
|
|
749
|
+
jvm.matches.set(matchId, h);
|
|
750
|
+
writeRecord(jvm);
|
|
751
|
+
events.log({ evt: 'match-placed', matchId, hostId: jvm.hostId, warm: jvm.warm, matches: jvm.matches.size, aiFlatMc: config.aiFlatMc ?? 'off' });
|
|
752
|
+
jvm.transport.send(JSON.stringify({ t: 'start', config: bridgeConfig(matchId, spec, config.aiFlatMc ?? 'off'), match: jvm.hostId }));
|
|
753
|
+
notifyRoom();
|
|
754
|
+
replenish();
|
|
755
|
+
return hostFace(h);
|
|
756
|
+
});
|
|
757
|
+
factory.room = () => {
|
|
758
|
+
const at = placeFor();
|
|
759
|
+
return at === null ? 'none' : at.warm ? 'warm' : 'booting';
|
|
760
|
+
};
|
|
761
|
+
factory.booting = () => liveJvms().filter((j) => !j.warm).length;
|
|
762
|
+
factory.watchRoom = (listener) => {
|
|
763
|
+
roomWatchers.push(listener);
|
|
764
|
+
};
|
|
765
|
+
factory.jvms = () => [...jvms].map((j) => ({ hostId: j.hostId, warm: j.warm, draining: j.draining, adopted: j.adopted, matches: [...j.matches.keys()] }));
|
|
766
|
+
factory.drain = (hostId, why) => {
|
|
767
|
+
const jvm = [...jvms].find((j) => j.hostId === hostId);
|
|
768
|
+
if (jvm === undefined || jvm.draining || jvm.exited || jvm.released)
|
|
769
|
+
return;
|
|
770
|
+
jvm.draining = true;
|
|
771
|
+
poolLog({ evt: 'jvm-draining', id: hostId, why, matches: [...jvm.matches.keys()] });
|
|
772
|
+
idle(jvm);
|
|
773
|
+
notifyRoom();
|
|
774
|
+
replenish();
|
|
775
|
+
};
|
|
776
|
+
factory.abandon = (why) => {
|
|
777
|
+
closed = true;
|
|
778
|
+
if (retry !== null)
|
|
779
|
+
clearTimeout(retry);
|
|
780
|
+
for (const jvm of [...jvms]) {
|
|
781
|
+
poolLog({ evt: 'jvm-abandoned', id: jvm.hostId, why, matches: [...jvm.matches.keys()] });
|
|
782
|
+
jvm.unresponsive = why;
|
|
783
|
+
jvm.transport.kill();
|
|
784
|
+
}
|
|
785
|
+
};
|
|
786
|
+
factory.close = () => {
|
|
787
|
+
closed = true;
|
|
788
|
+
if (retry !== null)
|
|
789
|
+
clearTimeout(retry);
|
|
790
|
+
if (claimDeadline !== null)
|
|
791
|
+
clearTimeout(claimDeadline);
|
|
792
|
+
for (const jvm of [...jvms])
|
|
793
|
+
if (jvm.matches.size === 0)
|
|
794
|
+
retire(jvm, 'closing');
|
|
795
|
+
};
|
|
796
|
+
/* ── Re-attach ── */
|
|
797
|
+
/** Take back a JVM an earlier server left running, and every match its record names. */
|
|
798
|
+
function adopt(hostId) {
|
|
799
|
+
const dir = join(config.workDir, hostId);
|
|
800
|
+
const record = readJvmRecord(dir);
|
|
801
|
+
const transport = launcher.attach(hostId);
|
|
802
|
+
const jvm = track(hostId, dir, transport, { warm: true, startedAt: record?.startedAt ?? new Date().toISOString(), orphanedAt: null, adopted: true });
|
|
803
|
+
const age = record === null ? 0 : Date.now() - Date.parse(record.orphanedAt ?? record.startedAt);
|
|
804
|
+
if (record === null || !(age < ORPHAN_MAX_MS)) {
|
|
805
|
+
retire(jvm, record === null ? 'no readable record: its matches cannot be named' : `orphaned ${age} ms ago, past ORPHAN_MAX_MS`);
|
|
806
|
+
return;
|
|
807
|
+
}
|
|
808
|
+
jvm.orphanedAt = record.orphanedAt ?? null;
|
|
809
|
+
for (const entry of record.matches) {
|
|
810
|
+
const h = {
|
|
811
|
+
id: entry.matchId,
|
|
812
|
+
jvm,
|
|
813
|
+
events: null,
|
|
814
|
+
entry,
|
|
815
|
+
onDisk: JSON.stringify(entry),
|
|
816
|
+
stopping: false,
|
|
817
|
+
ended: false,
|
|
818
|
+
handed: false,
|
|
819
|
+
degraded: null,
|
|
820
|
+
fatal: null,
|
|
821
|
+
stopTimer: null,
|
|
822
|
+
stream: null,
|
|
823
|
+
};
|
|
824
|
+
jvm.matches.set(h.id, h);
|
|
825
|
+
if (existsSync(streamFile(dir, h.id))) {
|
|
826
|
+
pending.set(h.id, h);
|
|
827
|
+
}
|
|
828
|
+
else {
|
|
829
|
+
// Its stream broke under the server before: a replay would be wrong, so it ends here.
|
|
830
|
+
poolLog({ evt: 'match-not-resumable', matchId: h.id, hostId, why: 'its stream is not whole' });
|
|
831
|
+
stopMatch(h);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
poolLog({ evt: 'jvm-adopted', id: hostId, matches: [...jvm.matches.keys()], orphanedAt: jvm.orphanedAt });
|
|
835
|
+
if (jvm.matches.size === 0)
|
|
836
|
+
idle(jvm);
|
|
837
|
+
}
|
|
838
|
+
/** Nobody came back for these. See "A match outlives its server". */
|
|
839
|
+
function dropUnclaimed() {
|
|
840
|
+
for (const h of [...pending.values()]) {
|
|
841
|
+
poolLog({ evt: 'orphan-unclaimed', matchId: h.id, hostId: h.jvm.hostId, afterMs: ORPHAN_CLAIM_MS });
|
|
842
|
+
pending.delete(h.id);
|
|
843
|
+
stopMatch(h);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
// The warm JVM is launched once discovery has answered, not before: a JVM taken back with room
|
|
847
|
+
// IS the warm one, and a fresh boot beside it after every deploy only spent a boot's CPU while
|
|
848
|
+
// the matches were resuming (Forge run 40566453363: each e2e restart booted a second JVM, 30 s+
|
|
849
|
+
// on a loaded claw).
|
|
850
|
+
const found = launcher
|
|
851
|
+
.running()
|
|
852
|
+
.then((hostIds) => {
|
|
853
|
+
for (const hostId of hostIds) {
|
|
854
|
+
try {
|
|
855
|
+
adopt(hostId);
|
|
856
|
+
}
|
|
857
|
+
catch (err) {
|
|
858
|
+
poolLog({ evt: 'jvm-adopt-failed', id: hostId, error: err.message });
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
if (pending.size > 0) {
|
|
862
|
+
claimDeadline = setTimeout(dropUnclaimed, ORPHAN_CLAIM_MS);
|
|
863
|
+
claimDeadline.unref();
|
|
864
|
+
}
|
|
865
|
+
replenish();
|
|
866
|
+
})
|
|
867
|
+
.catch((err) => {
|
|
868
|
+
poolLog({ evt: 'jvm-running-failed', error: err.message });
|
|
869
|
+
replenish();
|
|
870
|
+
});
|
|
871
|
+
factory.orphans = async () => {
|
|
872
|
+
await found;
|
|
873
|
+
return [...pending.values()]
|
|
874
|
+
.map((h) => ({ ...h.entry, hostId: h.jvm.hostId, ...(h.jvm.orphanedAt === null ? {} : { orphanedAt: h.jvm.orphanedAt }) }))
|
|
875
|
+
.sort((a, b) => Date.parse(a.startedAt) - Date.parse(b.startedAt));
|
|
876
|
+
};
|
|
877
|
+
factory.attach = (record, events) => {
|
|
878
|
+
const h = pending.get(record.matchId);
|
|
879
|
+
if (h === undefined)
|
|
880
|
+
throw new Error(`there is no orphan match ${record.matchId} to attach to`);
|
|
881
|
+
pending.delete(record.matchId);
|
|
882
|
+
// Replay, THEN deliver live lines. A live delta applied before the `setGameView` it follows
|
|
883
|
+
// would give the seat a board Forge does not have; the order holds because the replay is
|
|
884
|
+
// synchronous and Node is single-threaded.
|
|
885
|
+
const replayed = replay(h, events);
|
|
886
|
+
h.events = events;
|
|
887
|
+
events.log({ evt: 'match-resumed', matchId: h.id, hostId: h.jvm.hostId, replayed });
|
|
888
|
+
return hostFace(h);
|
|
889
|
+
};
|
|
890
|
+
/* ── Going down ── */
|
|
891
|
+
/** Let go of a JVM with a handed-over match: stamped, released, its matches left to the next server. */
|
|
892
|
+
function release(jvm) {
|
|
893
|
+
jvm.orphanedAt = new Date().toISOString();
|
|
894
|
+
writeRecord(jvm);
|
|
895
|
+
jvm.released = true;
|
|
896
|
+
if (jvm.liveness !== null)
|
|
897
|
+
clearInterval(jvm.liveness);
|
|
898
|
+
if (jvm.idle !== null)
|
|
899
|
+
clearTimeout(jvm.idle);
|
|
900
|
+
for (const h of jvm.matches.values()) {
|
|
901
|
+
if (h.stopTimer !== null)
|
|
902
|
+
clearTimeout(h.stopTimer);
|
|
903
|
+
closeStream(h);
|
|
904
|
+
}
|
|
905
|
+
jvms.delete(jvm);
|
|
906
|
+
jvm.transport.release();
|
|
907
|
+
}
|
|
908
|
+
/** SIGTERM: every live match is left running for the next process. Returns the matches handed over. */
|
|
909
|
+
factory.handOverAll = () => {
|
|
910
|
+
const handed = [];
|
|
911
|
+
for (const jvm of [...jvms]) {
|
|
912
|
+
for (const h of jvm.matches.values())
|
|
913
|
+
if (hostFace(h).handOver())
|
|
914
|
+
handed.push(h.id);
|
|
915
|
+
}
|
|
916
|
+
factory.close();
|
|
917
|
+
for (const jvm of [...jvms])
|
|
918
|
+
if ([...jvm.matches.values()].some((h) => h.handed))
|
|
919
|
+
release(jvm);
|
|
920
|
+
return handed;
|
|
921
|
+
};
|
|
922
|
+
/**
|
|
923
|
+
* The last word before `process.exit`. It asks the pool, not the match statuses: the only
|
|
924
|
+
* question is whether a JVM holds a match that was handed over. Orphans from an EARLIER server
|
|
925
|
+
* that nobody claimed are among the JVMs and are let go again if a match of theirs was handed
|
|
926
|
+
* over, else ended — a match nobody claimed was stopped.
|
|
927
|
+
*/
|
|
928
|
+
factory.reapOnExit = () => {
|
|
929
|
+
closed = true;
|
|
930
|
+
reaping = true;
|
|
931
|
+
if (retry !== null)
|
|
932
|
+
clearTimeout(retry);
|
|
933
|
+
if (claimDeadline !== null)
|
|
934
|
+
clearTimeout(claimDeadline);
|
|
935
|
+
const result = { killed: [], kept: [], left: [] };
|
|
936
|
+
for (const jvm of [...jvms]) {
|
|
937
|
+
if ([...jvm.matches.values()].some((h) => h.handed || (pending.has(h.id) && h.degraded === null))) {
|
|
938
|
+
release(jvm);
|
|
939
|
+
result.kept.push(jvm.hostId);
|
|
940
|
+
continue;
|
|
941
|
+
}
|
|
942
|
+
jvm.transport.send(JSON.stringify({ t: 'stop', match: jvm.hostId }));
|
|
943
|
+
jvm.transport.kill();
|
|
944
|
+
(jvm.exited ? result.killed : result.left).push(jvm.hostId);
|
|
945
|
+
}
|
|
946
|
+
poolLog({ evt: 'jvms-reaped', ...result });
|
|
947
|
+
return result;
|
|
948
|
+
};
|
|
949
|
+
return factory;
|
|
950
|
+
}
|