hoi4-agent-tools 4.0.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +29 -53
- package/dist/hoi4_agent_tools/core/condition-evaluator.js +18 -0
- package/dist/hoi4_agent_tools/core/job-worker-host.js +115 -27
- package/dist/hoi4_agent_tools/core/job-worker.js +7 -2
- package/dist/hoi4_agent_tools/core/temporary-directories.js +293 -0
- package/dist/hoi4_agent_tools/core/worker-exit.js +80 -0
- package/dist/hoi4_agent_tools/gui/assets.js +3 -0
- package/dist/hoi4_agent_tools/gui/dds.js +8 -4
- package/dist/hoi4_agent_tools/gui/layout.js +45 -5
- package/dist/hoi4_agent_tools/gui/native-state.js +244 -0
- package/dist/hoi4_agent_tools/gui/renderer.js +12 -7
- package/dist/hoi4_agent_tools/gui/source-graph.js +17 -4
- package/dist/hoi4_agent_tools/gui/studio.js +33 -21
- package/dist/hoi4_agent_tools/mcp/prompts/probability.js +2 -2
- package/dist/hoi4_agent_tools/mcp/server/instructions.js +1 -1
- package/dist/hoi4_agent_tools/mcp/tools/analysis.js +8 -8
- package/dist/hoi4_agent_tools/mcp/tools/event.js +3 -3
- package/dist/hoi4_agent_tools/mcp/tools/focus.js +5 -5
- package/dist/hoi4_agent_tools/mcp/tools/gui.js +6 -6
- package/dist/hoi4_agent_tools/mcp/tools/job.js +4 -4
- package/dist/hoi4_agent_tools/mcp/tools/map.js +4 -4
- package/dist/hoi4_agent_tools/mcp/tools/probability.js +4 -4
- package/dist/hoi4_agent_tools/mcp/tools/reference.js +12 -12
- package/dist/hoi4_agent_tools/mcp/tools/technology.js +3 -3
- package/dist/hoi4_agent_tools/runtime.js +14 -1
- package/dist/hoi4_agent_tools/schemas/map-requests.js +1 -1
- package/dist/hoi4_agent_tools/version.js +1 -1
- package/docs/analysis.md +2 -2
- package/docs/development.md +2 -0
- package/docs/events.md +2 -3
- package/docs/examples.md +8 -23
- package/docs/focus.md +3 -2
- package/docs/gui.md +7 -4
- package/docs/helper-expansion.md +1 -1
- package/docs/jobs.md +5 -2
- package/docs/map.md +5 -4
- package/docs/mechanics.md +2 -2
- package/docs/probability.md +5 -2
- package/docs/reference.md +2 -1
- package/docs/setup.md +2 -4
- package/docs/technology.md +2 -3
- package/docs/tools.md +12 -13
- package/docs/troubleshooting.md +1 -1
- package/npm-shrinkwrap.json +2 -2
- package/package.json +2 -2
- package/schemas/configuration.schema.json +1 -1
- package/schemas/continuous-focus-palette.schema.json +1 -1
- package/schemas/custom-weighted-pool.schema.json +1 -1
- package/schemas/decision-inspect-input.schema.json +1 -1
- package/schemas/error-log-input.schema.json +1 -1
- package/schemas/event-feature-manifest.schema.json +1 -1
- package/schemas/focus-plan.schema.json +1 -1
- package/schemas/focus-planning-sidecar.schema.json +1 -1
- package/schemas/gui-animation-source.schema.json +1 -1
- package/schemas/gui-helper.schema.json +1 -1
- package/schemas/gui-scenario.schema.json +1 -1
- package/schemas/helper-expansion-request.schema.json +1 -1
- package/schemas/helper-expansion-summary.schema.json +1 -1
- package/schemas/impact-inspect-input.schema.json +1 -1
- package/schemas/map-operation.schema.json +1 -1
- package/schemas/mechanic-test-input.schema.json +1 -1
- package/schemas/mod-index-input.schema.json +1 -1
- package/schemas/operation-result.schema.json +1 -1
- package/schemas/package-check-input.schema.json +1 -1
- package/schemas/package-manifest.schema.json +1 -1
- package/schemas/probability-analysis-result.schema.json +1 -1
- package/schemas/probability-compare-input.schema.json +1 -1
- package/schemas/probability-evaluate-input.schema.json +1 -1
- package/schemas/probability-inspect-input.schema.json +1 -1
- package/schemas/probability-render-input.schema.json +1 -1
- package/schemas/probability-scenario-set.schema.json +1 -1
- package/schemas/probability-sequence-input.schema.json +1 -1
- package/schemas/probability-simulate-input.schema.json +1 -1
- package/schemas/probability-sweep-input.schema.json +1 -1
- package/schemas/reference-context-input.schema.json +1 -1
- package/schemas/reference-read-input.schema.json +1 -1
- package/schemas/reference-search-input.schema.json +1 -1
- package/schemas/scenario-suite.schema.json +1 -1
- package/schemas/scenario-test-input.schema.json +1 -1
- package/schemas/script-validate-input.schema.json +1 -1
- package/schemas/shared-scenario.schema.json +1 -1
- package/schemas/source-lookup-input.schema.json +1 -1
- package/server.json +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,28 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 4.0.1 - 2026-10-12
|
|
6
|
+
|
|
7
|
+
- Clean temporary test fixtures after each test, suite teardown and normal process exit. Reject fixture truncation above a 256 MiB aggregate disk budget before Windows allocates the file.
|
|
8
|
+
- Recover stale reference-source and random-name event-evaluation scratch on startup after three hours, with local process leases, lock and canonical-path checks. Evict dead owners oldest first under a 256 MiB scratch cap; preserve live owners and dated benchmark evidence.
|
|
9
|
+
- Document the leak's test-fixture origin. MCP reference lookup already reads configured sources directly and uses the shared persistent analysis cache; it does not copy a source tree per call or process.
|
|
10
|
+
- Describe modding tasks in plain language across the README, guides, package metadata and MCP tool descriptions.
|
|
11
|
+
|
|
12
|
+
### GUI render
|
|
13
|
+
|
|
14
|
+
- Decode uncompressed 16-bit RGB DDS portraits, including the 5:5:5 format used by vanilla leaders; truncated pixel data remains unsupported.
|
|
15
|
+
- Preserve declared font colours when a same-name bitmap atlas is loaded, including the dark super-event description font. Atlas metadata no longer outranks a named font declaration, and source links retain that declaration.
|
|
16
|
+
- `hoi4.gui_render` and `hoi4.gui_inspect` model the engine-controlled branches of `countrypoliticsview` instead of drawing all of them: no faction, faction leader, faction member and balance of power are chosen from the scenario country (`tag`, `ideology`, `faction`, `balanceOfPower`, or its `history/countries` facts at the scenario date), and `pol_faction_icon` follows the ruling ideology instead of its static frame. Explicit scenario visibility, values and selected frames still win.
|
|
17
|
+
- GUI text keys are matched without regard to case, so a window that spells `buttontext` in lower case (the vanilla trade window) shows its localised label instead of the key.
|
|
18
|
+
- Element visibility honours `show` as well as `visible`, `hide` and `hidden`, and scenario `name.visible` values may be yes/no words or 1/0 as well as booleans.
|
|
19
|
+
|
|
20
|
+
### Job workers
|
|
21
|
+
|
|
22
|
+
- Uncaught worker failures log only a bounded diagnostic code; exception messages, stacks and thrown values are excluded to protect source text and credentials.
|
|
23
|
+
- A worker that stops without a result is now reported by its real cause. The host keeps each worker's exit code, signal and last standard error and fails the job with `JOB_WORKER_EXIT` naming a V8 heap limit (with its size), a native out-of-memory, a native crash, an external termination, a stop by the server's own deadline, or an uncaught job error with its error code. Earlier releases discarded this and always suggested that memory ran out. Logs of workers that ended unexpectedly are kept, newest ten, in `job-workers/logs` under the server state root.
|
|
24
|
+
- Workers start with a V8 heap ceiling of a quarter of physical memory, between 2,048 and 8,192 MiB, when `jobWorkerMaxHeapMiB` is not set. Node's own default tops out near 4.5 GB, which a large GUI render on a big mod can approach.
|
|
25
|
+
- `hoi4.gui_render` rasterizes its derived variants, the comparison renders and the galleries one at a time instead of together, which lowers the peak native memory of a render.
|
|
26
|
+
|
|
5
27
|
## 4.0.0 - 2026-10-10
|
|
6
28
|
|
|
7
29
|
### Known mistakes
|
package/README.md
CHANGED
|
@@ -1,32 +1,26 @@
|
|
|
1
1
|
# HOI4 Agent Tools
|
|
2
2
|
|
|
3
|
+
<!-- TODO-VIDEO-LINK: replace with the promo video URL -->
|
|
4
|
+
|
|
5
|
+
[](https://www.youtube.com/@ChaosRedux)
|
|
6
|
+
|
|
3
7
|
[](https://www.npmjs.com/package/hoi4-agent-tools)
|
|
4
8
|
[](https://nodejs.org/)
|
|
5
9
|
[](LICENSE)
|
|
10
|
+
[](https://discord.gg/rAXesGcT2t)
|
|
11
|
+
[](https://www.youtube.com/@ChaosRedux)
|
|
12
|
+
[](https://ko-fi.com/klimpaskov)
|
|
13
|
+
[](https://github.com/klimPaskov/hoi4-agent-tools/releases/latest)
|
|
14
|
+
[](https://github.com/klimPaskov/hoi4-agent-tools/issues)
|
|
6
15
|
|
|
7
|
-
An [MCP](https://modelcontextprotocol.io/) server that
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
[](docs/images/comparisons/holy-realm-mcp.png)
|
|
16
|
+
An [MCP](https://modelcontextprotocol.io/) server that helps coding agents build and edit Hearts of Iron IV mods.
|
|
17
|
+
Make focus trees, events and decisions. Edit maps and generate a new world. Preview windows, trees and maps, find the right rule in the docs, fix broken script and balance AI choices.
|
|
11
18
|
|
|
12
19
|
## What it does
|
|
13
20
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
| Check before launch | "Check `events/my_events.txt` for scope errors, then group the last `error.log` by category." |
|
|
18
|
-
| See the tree | "Render this mod's largest focus tree and list focuses with missing icons." |
|
|
19
|
-
| Preview the window | "Preview this scripted GUI at 100% and 125% UI scale and check for clipped text." |
|
|
20
|
-
| Read the map | "Find Brandenburg, list its provinces and neighbours, and show the files a change would touch." |
|
|
21
|
-
| Reshape the map | "Split Brandenburg in two and update every event and focus that names it." |
|
|
22
|
-
| Build a new world | "Create a 2048x1024 world with six countries, then list what still needs hand-made content." |
|
|
23
|
-
| Follow the events | "Trace this event's options and follow-up events and point out unresolved calls." |
|
|
24
|
-
| Weigh the AI | "Show the AI weights of event `my_mod.12`'s options, then compare them before and after my change." |
|
|
25
|
-
| Plan new content | "Give me the next free event ID in my namespace and summarize what the mod defines." |
|
|
26
|
-
| Edit with a safety net | "Add a new branch to this focus tree" — the edit is validated before anything is written. |
|
|
27
|
-
|
|
28
|
-
The server never launches the game.
|
|
29
|
-
Previews and tests are built from source files, so they show what the source says, not what a running campaign did.
|
|
21
|
+
Your agent can build and edit focus trees, events, decisions and maps, and generate a new world.
|
|
22
|
+
It previews focus trees, technology folders, windows and maps, finds the right rule in the game's documentation, fixes broken script and tunes AI choices.
|
|
23
|
+
You describe what you want; the agent picks the right tools.
|
|
30
24
|
|
|
31
25
|
## Quick start
|
|
32
26
|
|
|
@@ -45,14 +39,17 @@ Previews and tests are built from source files, so they show what the source say
|
|
|
45
39
|
|
|
46
40
|
Paste the `globalInstall` block into a JSON client such as Claude Code, Claude Desktop or Cursor, or the `codexTomlGlobal` block into Codex's `config.toml`.
|
|
47
41
|
|
|
48
|
-
4. Open your mod folder as the agent's workspace and
|
|
42
|
+
4. Open your mod folder as the agent's workspace and tell it what you want to build or fix.
|
|
49
43
|
|
|
50
44
|
The [setup guide](docs/setup.md) covers several mods, explicit game paths, local wiki snapshots and troubleshooting with `--discover` and `--diagnose`.
|
|
51
45
|
|
|
52
|
-
[Watch the Agent Tools demo](https://github.com/klimPaskov/Agentic-HOI4-Modding/releases/latest/download/hoi4_agent_tools_promo.mp4) or use [HOI4 Mod Setup](https://github.com/klimPaskov/HOI4-Mod-Setup/releases/latest) to install and
|
|
46
|
+
[Watch the Agent Tools demo](https://github.com/klimPaskov/Agentic-HOI4-Modding/releases/latest/download/hoi4_agent_tools_promo.mp4) or use [HOI4 Mod Setup](https://github.com/klimPaskov/HOI4-Mod-Setup/releases/latest) to install and connect the server for your mod.
|
|
47
|
+
Part of the HOI4 modding toolkit with [Agentic HOI4 Modding](https://github.com/klimPaskov/Agentic-HOI4-Modding) (skills and instructions for your agent) and [HOI4 Live Tools](https://github.com/klimPaskov/HOI4-Live-Tools) (testing in the running game).
|
|
53
48
|
|
|
54
49
|
## Examples
|
|
55
50
|
|
|
51
|
+
[](docs/images/comparisons/holy-realm-mcp.png)
|
|
52
|
+
|
|
56
53
|
Each pair shows a capture from the game next to the server's preview of the same source.
|
|
57
54
|
|
|
58
55
|
| In game | Server preview |
|
|
@@ -61,40 +58,19 @@ Each pair shows a capture from the game next to the server's preview of the same
|
|
|
61
58
|
| [](docs/images/comparisons/infantry-ingame.png) | [](docs/images/comparisons/infantry-mcp.png) |
|
|
62
59
|
| [](docs/images/comparisons/gui/options-video-ingame.png) | [](docs/images/comparisons/gui/options-video-mcp.png) |
|
|
63
60
|
|
|
64
|
-
|
|
65
|
-
The focus screen in the game has its own framing, labels and icons, and the video options preview has no observed gamma value.
|
|
66
|
-
More comparisons, with their source revisions and fidelity notes, are in the [focus and technology gallery](https://github.com/klimPaskov/hoi4-agent-tools/blob/main/docs/visual-comparisons.md), the [interface gallery](https://github.com/klimPaskov/hoi4-agent-tools/blob/main/docs/gui-comparisons.md) and the [examples guide](docs/examples.md), which also shows map and event renders.
|
|
61
|
+
See more in the [examples gallery](docs/examples.md).
|
|
67
62
|
|
|
68
63
|
## Tools
|
|
69
64
|
|
|
70
|
-
The
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
| Events | `event_inspect`, `event_render` | — |
|
|
80
|
-
| Technology and doctrines | `tech_inspect`, `tech_render` | — |
|
|
81
|
-
| AI weights and timing | `probability_inspect`, `probability_analyze` | — |
|
|
82
|
-
| Decisions, impact and tests | `impact_inspect`, `decision_inspect`, `mechanic_test`, `package_check`, `scenario_test` | — |
|
|
83
|
-
| Background jobs | `job_inspect`, `job_cancel` | job state |
|
|
84
|
-
|
|
85
|
-
Every name starts with `hoi4.`.
|
|
86
|
-
"Review files" are images and reports in the server's own storage; only the three `*_rewrite` tools change mod files.
|
|
87
|
-
The [tool reference](docs/tools.md) gives each tool's purpose, prompts, resources and long-running behaviour.
|
|
88
|
-
|
|
89
|
-
## Safety and limits
|
|
90
|
-
|
|
91
|
-
- The server reads the mod, its dependencies, the installed game, local reference folders and the game's error log.
|
|
92
|
-
- Inspection and rendering never change mod files.
|
|
93
|
-
The focus, GUI and map rewrite tools validate a proposed edit before writing it and write it as a journaled transaction, so an interrupted write is restored rather than left half-applied.
|
|
94
|
-
- Large renders and reports are stored as linked files, and documentation is returned as cited sections rather than whole manuals.
|
|
95
|
-
- The server does not launch or control the game.
|
|
96
|
-
Previews and source tests cannot prove live clicks, AI choices or engine timing; a source score alone does not establish an AI click probability.
|
|
97
|
-
- Your repository instructions and review process still apply.
|
|
65
|
+
The server gives your agent previews, documentation, script analysis and editing tools.
|
|
66
|
+
The agent combines them with its file editor to build events and decisions, repair scripts and adjust AI weights.
|
|
67
|
+
The [tool reference](docs/tools.md) lists exact names, arguments and results for integrations.
|
|
68
|
+
|
|
69
|
+
## FAQ
|
|
70
|
+
|
|
71
|
+
**Where do edits go?** Changes are written to your mod folder; use Git to review or undo them.
|
|
72
|
+
|
|
73
|
+
**Does it run the game?** No. Previews come from mod source; test gameplay in HOI4 to see live clicks, AI choices and timing.
|
|
98
74
|
|
|
99
75
|
## Configuration
|
|
100
76
|
|
|
@@ -176,6 +176,24 @@ function evaluateAssignmentCore(assignment, scenario, candidate, definitions, he
|
|
|
176
176
|
const malformed = malformedBlock(assignment.value, candidate);
|
|
177
177
|
if (malformed !== undefined)
|
|
178
178
|
return malformed;
|
|
179
|
+
if (key === 'has_country_flag' || key === 'has_global_flag' || key === 'has_state_flag') {
|
|
180
|
+
// The block form tests a flag's stored value. A scenario declares that value as a number
|
|
181
|
+
// under `flag.<name>` (or `global_flag.<name>` for global flags).
|
|
182
|
+
const flag = firstScalar(assignment.value, 'flag')?.value;
|
|
183
|
+
const expected = firstScalar(assignment.value, 'value')?.value;
|
|
184
|
+
if (flag !== undefined) {
|
|
185
|
+
const names = key === 'has_global_flag' ? [`global_flag.${flag}`, `flag.${flag}`] : [`flag.${flag}`];
|
|
186
|
+
for (const name of names) {
|
|
187
|
+
const declared = scopeStateValue(scenario, name, scopeContext);
|
|
188
|
+
if (typeof declared === 'number' && expected !== undefined)
|
|
189
|
+
return { state: declared === Number(expected) ? 'true' : 'false', unresolved: [] };
|
|
190
|
+
if (typeof declared === 'boolean' && expected === undefined)
|
|
191
|
+
return { state: declared ? 'true' : 'false', unresolved: [] };
|
|
192
|
+
}
|
|
193
|
+
if (expected !== undefined)
|
|
194
|
+
return unresolved(assignment, candidate, `Scenario does not declare the value of flag ${flag}`, `flag.${flag}`);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
179
197
|
if (key === 'AND' ||
|
|
180
198
|
key === 'hidden_trigger' ||
|
|
181
199
|
key === 'custom_trigger_tooltip' ||
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
|
+
import { closeSync, fstatSync, openSync, readSync, readdirSync, rmSync, statSync } from 'node:fs';
|
|
2
3
|
import { mkdir } from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
3
5
|
import { fileURLToPath } from 'node:url';
|
|
4
6
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
5
7
|
import { JobService } from './job-service.js';
|
|
@@ -9,11 +11,15 @@ import { SharedRequestCapacity } from './shared-request-capacity.js';
|
|
|
9
11
|
import { containedGeneratedPath } from './workspace.js';
|
|
10
12
|
import { ServiceError } from './result.js';
|
|
11
13
|
import { isDomainToolName } from './domain-tools.js';
|
|
14
|
+
import { classifyWorkerExit, defaultWorkerHeapMiB, } from './worker-exit.js';
|
|
12
15
|
const maximumCheckpointRecoveries = 2;
|
|
13
16
|
const maximumPreDispatchRecoveries = 2;
|
|
14
|
-
/**
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
+
/** Used only when the exit status of the worker is unknown to this process. */
|
|
18
|
+
const WORKER_EXIT_MESSAGE = 'The worker stopped without publishing a terminal job outcome; its exit status is not available to this server, so retry the call';
|
|
19
|
+
/** Bytes of a worker's standard error read back when it exits unexpectedly. */
|
|
20
|
+
const STDERR_TAIL_BYTES = 16_384;
|
|
21
|
+
/** Standard-error logs of workers that exited unexpectedly kept for diagnosis. */
|
|
22
|
+
const RETAINED_STDERR_LOGS = 10;
|
|
17
23
|
// Native and IPC codes are fixed vocabulary. Error messages can contain paths or source
|
|
18
24
|
// text, so only an allowlisted code and the fixed coordination stage are reported.
|
|
19
25
|
const nativeFailureCodes = new Set([
|
|
@@ -62,13 +68,68 @@ export function workerFailure(error, stage) {
|
|
|
62
68
|
};
|
|
63
69
|
}
|
|
64
70
|
/** Stop a supervised worker and wait, bounded, until the operating system reports its exit. */
|
|
65
|
-
async function stopChild(child) {
|
|
71
|
+
async function stopChild(child, stopped) {
|
|
66
72
|
if (child.exitCode !== null || child.signalCode !== null)
|
|
67
73
|
return;
|
|
74
|
+
stopped?.add(child);
|
|
68
75
|
const exited = new Promise((resolve) => child.once('exit', () => resolve()));
|
|
69
76
|
child.kill('SIGKILL');
|
|
70
77
|
await Promise.race([exited, delay(10_000)]);
|
|
71
78
|
}
|
|
79
|
+
/** Read the last bytes of a worker's standard-error log; empty when it cannot be read. */
|
|
80
|
+
function readTail(file) {
|
|
81
|
+
try {
|
|
82
|
+
const fd = openSync(file, 'r');
|
|
83
|
+
try {
|
|
84
|
+
const size = fstatSync(fd).size;
|
|
85
|
+
const length = Math.min(size, STDERR_TAIL_BYTES);
|
|
86
|
+
const buffer = Buffer.alloc(length);
|
|
87
|
+
readSync(fd, buffer, 0, length, size - length);
|
|
88
|
+
return buffer.toString('utf8');
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
closeSync(fd);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
return '';
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** Keep only the newest few worker logs; the rest are older diagnostics. */
|
|
99
|
+
function pruneWorkerLogs(logRoot) {
|
|
100
|
+
try {
|
|
101
|
+
const logs = readdirSync(logRoot)
|
|
102
|
+
.filter((name) => name.startsWith('worker-') && name.endsWith('.log'))
|
|
103
|
+
.map((name) => ({ name, modified: statSync(path.join(logRoot, name)).mtimeMs }))
|
|
104
|
+
.sort((left, right) => right.modified - left.modified);
|
|
105
|
+
// A recent log may belong to a worker that is still running, so only older ones go.
|
|
106
|
+
const cutoff = Date.now() - 10 * 60_000;
|
|
107
|
+
for (const { name, modified } of logs.slice(RETAINED_STDERR_LOGS))
|
|
108
|
+
if (modified < cutoff)
|
|
109
|
+
rmSync(path.join(logRoot, name), { force: true });
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
// Diagnostics are best effort and must never change a job outcome.
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Remove the log of a worker that ended normally or because the host stopped it. Keep the
|
|
117
|
+
* newest few logs of workers that ended unexpectedly, because their stack is the evidence.
|
|
118
|
+
*/
|
|
119
|
+
function retainOrRemoveLog(file, evidence, logRoot) {
|
|
120
|
+
try {
|
|
121
|
+
const unexpected = !evidence.stoppedByHost &&
|
|
122
|
+
(evidence.signal !== null || (evidence.code !== null && evidence.code !== 0));
|
|
123
|
+
if (!unexpected) {
|
|
124
|
+
rmSync(file, { force: true });
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
pruneWorkerLogs(logRoot);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
// Diagnostics are best effort and must never change a job outcome.
|
|
131
|
+
}
|
|
132
|
+
}
|
|
72
133
|
/**
|
|
73
134
|
* Bounded, fixed-entry child execution. Client disconnection never kills a worker; an
|
|
74
135
|
* overdue or unresponsive cancelled read-only worker is stopped by its supervising host.
|
|
@@ -78,12 +139,14 @@ export class JobWorkerHost {
|
|
|
78
139
|
jobs;
|
|
79
140
|
capacity;
|
|
80
141
|
scheduler;
|
|
142
|
+
logRoot;
|
|
81
143
|
owner = {};
|
|
82
|
-
constructor(engine, jobs, capacity, scheduler) {
|
|
144
|
+
constructor(engine, jobs, capacity, scheduler, logRoot) {
|
|
83
145
|
this.engine = engine;
|
|
84
146
|
this.jobs = jobs;
|
|
85
147
|
this.capacity = capacity;
|
|
86
148
|
this.scheduler = scheduler;
|
|
149
|
+
this.logRoot = logRoot;
|
|
87
150
|
}
|
|
88
151
|
static async create(engine, jobs) {
|
|
89
152
|
const state = engine.resolver.serverState();
|
|
@@ -91,8 +154,11 @@ export class JobWorkerHost {
|
|
|
91
154
|
throw new ServiceError('JOB_STORAGE_UNAVAILABLE', 'Workers require persistent server state');
|
|
92
155
|
const root = await containedGeneratedPath(state.root, 'job-workers');
|
|
93
156
|
await mkdir(root, { recursive: true, mode: 0o700 });
|
|
157
|
+
const logRoot = path.join(root, 'logs');
|
|
158
|
+
await mkdir(logRoot, { recursive: true, mode: 0o700 });
|
|
159
|
+
pruneWorkerLogs(logRoot);
|
|
94
160
|
const config = engine.resolver.config();
|
|
95
|
-
return new JobWorkerHost(engine, jobs ?? (await JobService.create(engine.resolver)), new SharedRequestCapacity(root, config.maxSharedTools), new RequestScheduler(config.maxConcurrentTools));
|
|
161
|
+
return new JobWorkerHost(engine, jobs ?? (await JobService.create(engine.resolver)), new SharedRequestCapacity(root, config.maxSharedTools), new RequestScheduler(config.maxConcurrentTools), logRoot);
|
|
96
162
|
}
|
|
97
163
|
async run(workspaceId, id, principal) {
|
|
98
164
|
const initial = await this.jobs.get(workspaceId, id, principal);
|
|
@@ -153,7 +219,7 @@ export class JobWorkerHost {
|
|
|
153
219
|
if (liveness !== 'alive')
|
|
154
220
|
throw new ServiceError(liveness === 'unknown' ? 'JOB_OWNER_UNRESOLVED' : 'JOB_WORKER_EXIT', liveness === 'unknown'
|
|
155
221
|
? 'The replacement execution owner cannot be verified on this host'
|
|
156
|
-
: `The replacement worker stopped without publishing a terminal job outcome
|
|
222
|
+
: `The replacement worker stopped without publishing a terminal job outcome; its exit status belongs to the server that started it, so retry the call`);
|
|
157
223
|
await delay(100);
|
|
158
224
|
}
|
|
159
225
|
}
|
|
@@ -206,7 +272,7 @@ export class JobWorkerHost {
|
|
|
206
272
|
if (reused !== undefined && error instanceof ServiceError)
|
|
207
273
|
throw new ServiceError('REQUEST_LEASE_HANDOFF_LOST', 'The warm worker stopped before it received the job');
|
|
208
274
|
if (reused === undefined)
|
|
209
|
-
await stopChild(child);
|
|
275
|
+
await stopChild(child, this.stoppedByHost);
|
|
210
276
|
throw error;
|
|
211
277
|
}
|
|
212
278
|
progress.stage = 'dispatch';
|
|
@@ -240,7 +306,7 @@ export class JobWorkerHost {
|
|
|
240
306
|
const unresponsive = cancellationSeenAt !== undefined &&
|
|
241
307
|
Date.now() - cancellationSeenAt > jobCancelGraceSeconds * 1000;
|
|
242
308
|
if (overdue || unresponsive) {
|
|
243
|
-
await stopChild(child);
|
|
309
|
+
await stopChild(child, this.stoppedByHost);
|
|
244
310
|
await this.jobs.failInterrupted(workspaceId, id, {
|
|
245
311
|
code: 'JOB_DEADLINE_EXCEEDED',
|
|
246
312
|
message: `The job ran longer than ${jobDeadlineSeconds} seconds and was stopped; narrow its selector or limits, or raise jobDeadlineSeconds`,
|
|
@@ -248,18 +314,17 @@ export class JobWorkerHost {
|
|
|
248
314
|
return;
|
|
249
315
|
}
|
|
250
316
|
}
|
|
251
|
-
if (child.
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
}
|
|
255
|
-
catch (error) {
|
|
256
|
-
if (error.code === 'ESRCH')
|
|
257
|
-
throw new ServiceError('JOB_WORKER_EXIT', WORKER_EXIT_MESSAGE);
|
|
258
|
-
}
|
|
317
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
318
|
+
const evidence = await this.exits.get(child);
|
|
319
|
+
throw new ServiceError('JOB_WORKER_EXIT', evidence === undefined ? WORKER_EXIT_MESSAGE : classifyWorkerExit(evidence).message);
|
|
259
320
|
}
|
|
260
321
|
await delay(100);
|
|
261
322
|
}
|
|
262
323
|
}
|
|
324
|
+
/** Workers this host stopped itself, so their exit is not reported as a fault. */
|
|
325
|
+
stoppedByHost = new WeakSet();
|
|
326
|
+
/** Settles with the exit status and last standard error of each worker this host started. */
|
|
327
|
+
exits = new WeakMap();
|
|
263
328
|
/** Idle warm workers of this host, most recently used last. */
|
|
264
329
|
idleWorkers = [];
|
|
265
330
|
/**
|
|
@@ -268,6 +333,7 @@ export class JobWorkerHost {
|
|
|
268
333
|
*/
|
|
269
334
|
reportedIdle = new WeakSet();
|
|
270
335
|
watched = new WeakSet();
|
|
336
|
+
workerSequence = 0;
|
|
271
337
|
watchIdle(child) {
|
|
272
338
|
if (this.watched.has(child))
|
|
273
339
|
return;
|
|
@@ -302,16 +368,38 @@ export class JobWorkerHost {
|
|
|
302
368
|
async startWorker(maxHeapMiB) {
|
|
303
369
|
const sourceMode = import.meta.url.endsWith('.ts');
|
|
304
370
|
const entry = fileURLToPath(new URL(sourceMode ? './job-worker.ts' : './job-worker.js', import.meta.url));
|
|
305
|
-
const
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
371
|
+
const heapLimitMiB = maxHeapMiB ?? defaultWorkerHeapMiB();
|
|
372
|
+
const stderrLog = path.join(this.logRoot, `worker-${process.pid}-${++this.workerSequence}.log`);
|
|
373
|
+
const stderrFd = openSync(stderrLog, 'w', 0o600);
|
|
374
|
+
let child;
|
|
375
|
+
try {
|
|
376
|
+
child = spawn(process.execPath, [
|
|
377
|
+
`--max-old-space-size=${heapLimitMiB}`,
|
|
378
|
+
...(sourceMode ? ['--import', import.meta.resolve('tsx')] : []),
|
|
379
|
+
entry,
|
|
380
|
+
], {
|
|
381
|
+
detached: true,
|
|
382
|
+
cwd: process.cwd(),
|
|
383
|
+
stdio: ['ignore', 'ignore', stderrFd, 'ipc'],
|
|
384
|
+
windowsHide: true,
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
finally {
|
|
388
|
+
closeSync(stderrFd);
|
|
389
|
+
}
|
|
390
|
+
this.exits.set(child, new Promise((resolve) => {
|
|
391
|
+
child.once('exit', (code, signal) => {
|
|
392
|
+
const evidence = {
|
|
393
|
+
code,
|
|
394
|
+
signal,
|
|
395
|
+
stderrTail: readTail(stderrLog),
|
|
396
|
+
stoppedByHost: this.stoppedByHost.has(child),
|
|
397
|
+
heapLimitMiB,
|
|
398
|
+
};
|
|
399
|
+
retainOrRemoveLog(stderrLog, evidence, this.logRoot);
|
|
400
|
+
resolve(evidence);
|
|
401
|
+
});
|
|
402
|
+
}));
|
|
315
403
|
try {
|
|
316
404
|
await new Promise((resolve, reject) => {
|
|
317
405
|
const onExit = (code, signal) => reject(new ServiceError('JOB_WORKER_STARTUP_EXIT', `The worker exited before completing its startup handshake (phase waiting_for_ready, exit ${code ?? 'none'}, signal ${signal ?? 'none'})`));
|
|
@@ -8,6 +8,7 @@ import { JobExecutor } from './job-executor.js';
|
|
|
8
8
|
import { JobService } from './job-service.js';
|
|
9
9
|
import { WorkspaceResolver } from './workspace.js';
|
|
10
10
|
import { registerWorkerOperations, workerDomain } from './job-operation-registration.js';
|
|
11
|
+
import { workerErrorMarker } from './worker-exit.js';
|
|
11
12
|
// Fixed internal process entry point. It accepts only a trusted host's structured IPC
|
|
12
13
|
// message, never a script path, module name, callback, command, or transport request.
|
|
13
14
|
//
|
|
@@ -32,9 +33,11 @@ let started = false;
|
|
|
32
33
|
let idleTimer;
|
|
33
34
|
/** Share of the heap limit a worker may still hold after a job and a full collection. */
|
|
34
35
|
const WARM_HEAP_SHARE = 0.4;
|
|
36
|
+
/** Absolute ceiling on that retained heap, so a large heap limit does not keep gigabytes idle. */
|
|
37
|
+
const WARM_HEAP_MAXIMUM_BYTES = 1_536 * 1024 * 1024;
|
|
35
38
|
/** Whether the heap, after a full collection, is small enough to serve another job. */
|
|
36
39
|
function heapAllowsReuse() {
|
|
37
|
-
const limit = getHeapStatistics().heap_size_limit * WARM_HEAP_SHARE;
|
|
40
|
+
const limit = Math.min(getHeapStatistics().heap_size_limit * WARM_HEAP_SHARE, WARM_HEAP_MAXIMUM_BYTES);
|
|
38
41
|
if (getHeapStatistics().used_heap_size <= limit)
|
|
39
42
|
return true;
|
|
40
43
|
try {
|
|
@@ -105,7 +108,9 @@ process.on('message', (value) => {
|
|
|
105
108
|
if (idleTimer !== undefined)
|
|
106
109
|
clearTimeout(idleTimer);
|
|
107
110
|
void runJob(value)
|
|
108
|
-
.catch(() => {
|
|
111
|
+
.catch((error) => {
|
|
112
|
+
// The host reads this line back to tell an uncaught job error from a memory failure.
|
|
113
|
+
process.stderr.write(workerErrorMarker(error));
|
|
109
114
|
process.exitCode = 1;
|
|
110
115
|
})
|
|
111
116
|
.finally(() => {
|