hoi4-agent-tools 4.0.0 → 4.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/README.md +29 -53
  3. package/dist/hoi4_agent_tools/core/condition-evaluator.js +18 -0
  4. package/dist/hoi4_agent_tools/core/job-worker-host.js +115 -27
  5. package/dist/hoi4_agent_tools/core/job-worker.js +7 -2
  6. package/dist/hoi4_agent_tools/core/temporary-directories.js +293 -0
  7. package/dist/hoi4_agent_tools/core/worker-exit.js +80 -0
  8. package/dist/hoi4_agent_tools/gui/assets.js +3 -0
  9. package/dist/hoi4_agent_tools/gui/dds.js +8 -4
  10. package/dist/hoi4_agent_tools/gui/layout.js +45 -5
  11. package/dist/hoi4_agent_tools/gui/native-state.js +244 -0
  12. package/dist/hoi4_agent_tools/gui/renderer.js +12 -7
  13. package/dist/hoi4_agent_tools/gui/source-graph.js +17 -4
  14. package/dist/hoi4_agent_tools/gui/studio.js +33 -21
  15. package/dist/hoi4_agent_tools/mcp/prompts/probability.js +2 -2
  16. package/dist/hoi4_agent_tools/mcp/server/instructions.js +1 -1
  17. package/dist/hoi4_agent_tools/mcp/tools/analysis.js +8 -8
  18. package/dist/hoi4_agent_tools/mcp/tools/event.js +3 -3
  19. package/dist/hoi4_agent_tools/mcp/tools/focus.js +5 -5
  20. package/dist/hoi4_agent_tools/mcp/tools/gui.js +6 -6
  21. package/dist/hoi4_agent_tools/mcp/tools/job.js +4 -4
  22. package/dist/hoi4_agent_tools/mcp/tools/map.js +4 -4
  23. package/dist/hoi4_agent_tools/mcp/tools/probability.js +4 -4
  24. package/dist/hoi4_agent_tools/mcp/tools/reference.js +12 -12
  25. package/dist/hoi4_agent_tools/mcp/tools/technology.js +3 -3
  26. package/dist/hoi4_agent_tools/runtime.js +14 -1
  27. package/dist/hoi4_agent_tools/schemas/map-requests.js +1 -1
  28. package/dist/hoi4_agent_tools/version.js +1 -1
  29. package/docs/analysis.md +2 -2
  30. package/docs/development.md +2 -0
  31. package/docs/events.md +2 -3
  32. package/docs/examples.md +8 -23
  33. package/docs/focus.md +3 -2
  34. package/docs/gui.md +7 -4
  35. package/docs/helper-expansion.md +1 -1
  36. package/docs/jobs.md +5 -2
  37. package/docs/map.md +5 -4
  38. package/docs/mechanics.md +2 -2
  39. package/docs/probability.md +5 -2
  40. package/docs/reference.md +2 -1
  41. package/docs/setup.md +2 -4
  42. package/docs/technology.md +2 -3
  43. package/docs/tools.md +12 -13
  44. package/docs/troubleshooting.md +1 -1
  45. package/npm-shrinkwrap.json +2 -2
  46. package/package.json +2 -2
  47. package/schemas/configuration.schema.json +1 -1
  48. package/schemas/continuous-focus-palette.schema.json +1 -1
  49. package/schemas/custom-weighted-pool.schema.json +1 -1
  50. package/schemas/decision-inspect-input.schema.json +1 -1
  51. package/schemas/error-log-input.schema.json +1 -1
  52. package/schemas/event-feature-manifest.schema.json +1 -1
  53. package/schemas/focus-plan.schema.json +1 -1
  54. package/schemas/focus-planning-sidecar.schema.json +1 -1
  55. package/schemas/gui-animation-source.schema.json +1 -1
  56. package/schemas/gui-helper.schema.json +1 -1
  57. package/schemas/gui-scenario.schema.json +1 -1
  58. package/schemas/helper-expansion-request.schema.json +1 -1
  59. package/schemas/helper-expansion-summary.schema.json +1 -1
  60. package/schemas/impact-inspect-input.schema.json +1 -1
  61. package/schemas/map-operation.schema.json +1 -1
  62. package/schemas/mechanic-test-input.schema.json +1 -1
  63. package/schemas/mod-index-input.schema.json +1 -1
  64. package/schemas/operation-result.schema.json +1 -1
  65. package/schemas/package-check-input.schema.json +1 -1
  66. package/schemas/package-manifest.schema.json +1 -1
  67. package/schemas/probability-analysis-result.schema.json +1 -1
  68. package/schemas/probability-compare-input.schema.json +1 -1
  69. package/schemas/probability-evaluate-input.schema.json +1 -1
  70. package/schemas/probability-inspect-input.schema.json +1 -1
  71. package/schemas/probability-render-input.schema.json +1 -1
  72. package/schemas/probability-scenario-set.schema.json +1 -1
  73. package/schemas/probability-sequence-input.schema.json +1 -1
  74. package/schemas/probability-simulate-input.schema.json +1 -1
  75. package/schemas/probability-sweep-input.schema.json +1 -1
  76. package/schemas/reference-context-input.schema.json +1 -1
  77. package/schemas/reference-read-input.schema.json +1 -1
  78. package/schemas/reference-search-input.schema.json +1 -1
  79. package/schemas/scenario-suite.schema.json +1 -1
  80. package/schemas/scenario-test-input.schema.json +1 -1
  81. package/schemas/script-validate-input.schema.json +1 -1
  82. package/schemas/shared-scenario.schema.json +1 -1
  83. package/schemas/source-lookup-input.schema.json +1 -1
  84. package/server.json +3 -3
package/CHANGELOG.md CHANGED
@@ -2,6 +2,28 @@
2
2
 
3
3
  ## Unreleased
4
4
 
5
+ ## 4.0.1 - 2026-10-12
6
+
7
+ - Clean temporary test fixtures after each test, suite teardown and normal process exit. Reject fixture truncation above a 256 MiB aggregate disk budget before Windows allocates the file.
8
+ - Recover stale reference-source and random-name event-evaluation scratch on startup after three hours, with local process leases, lock and canonical-path checks. Evict dead owners oldest first under a 256 MiB scratch cap; preserve live owners and dated benchmark evidence.
9
+ - Document the leak's test-fixture origin. MCP reference lookup already reads configured sources directly and uses the shared persistent analysis cache; it does not copy a source tree per call or process.
10
+ - Describe modding tasks in plain language across the README, guides, package metadata and MCP tool descriptions.
11
+
12
+ ### GUI render
13
+
14
+ - Decode uncompressed 16-bit RGB DDS portraits, including the 5:5:5 format used by vanilla leaders; truncated pixel data remains unsupported.
15
+ - Preserve declared font colours when a same-name bitmap atlas is loaded, including the dark super-event description font. Atlas metadata no longer outranks a named font declaration, and source links retain that declaration.
16
+ - `hoi4.gui_render` and `hoi4.gui_inspect` model the engine-controlled branches of `countrypoliticsview` instead of drawing all of them: no faction, faction leader, faction member and balance of power are chosen from the scenario country (`tag`, `ideology`, `faction`, `balanceOfPower`, or its `history/countries` facts at the scenario date), and `pol_faction_icon` follows the ruling ideology instead of its static frame. Explicit scenario visibility, values and selected frames still win.
17
+ - GUI text keys are matched without regard to case, so a window that spells `buttontext` in lower case (the vanilla trade window) shows its localised label instead of the key.
18
+ - Element visibility honours `show` as well as `visible`, `hide` and `hidden`, and scenario `name.visible` values may be yes/no words or 1/0 as well as booleans.
19
+
20
+ ### Job workers
21
+
22
+ - Uncaught worker failures log only a bounded diagnostic code; exception messages, stacks and thrown values are excluded to protect source text and credentials.
23
+ - A worker that stops without a result is now reported by its real cause. The host keeps each worker's exit code, signal and last standard error and fails the job with `JOB_WORKER_EXIT` naming a V8 heap limit (with its size), a native out-of-memory, a native crash, an external termination, a stop by the server's own deadline, or an uncaught job error with its error code. Earlier releases discarded this and always suggested that memory ran out. Logs of workers that ended unexpectedly are kept, newest ten, in `job-workers/logs` under the server state root.
24
+ - Workers start with a V8 heap ceiling of a quarter of physical memory, between 2,048 and 8,192 MiB, when `jobWorkerMaxHeapMiB` is not set. Node's own default tops out near 4.5 GB, which a large GUI render on a big mod can approach.
25
+ - `hoi4.gui_render` rasterizes its derived variants, the comparison renders and the galleries one at a time instead of together, which lowers the peak native memory of a render.
26
+
5
27
  ## 4.0.0 - 2026-10-10
6
28
 
7
29
  ### Known mistakes
package/README.md CHANGED
@@ -1,32 +1,26 @@
1
1
  # HOI4 Agent Tools
2
2
 
3
+ <!-- TODO-VIDEO-LINK: replace with the promo video URL -->
4
+
5
+ [![Watch the HOI4 Agent Tools demo](docs/images/agent-tools-demo-thumbnail.png)](https://www.youtube.com/@ChaosRedux)
6
+
3
7
  [![npm](https://img.shields.io/npm/v/hoi4-agent-tools)](https://www.npmjs.com/package/hoi4-agent-tools)
4
8
  [![Node.js](https://img.shields.io/node/v/hoi4-agent-tools)](https://nodejs.org/)
5
9
  [![License](https://img.shields.io/npm/l/hoi4-agent-tools)](LICENSE)
10
+ [![Discord](https://img.shields.io/badge/Discord-Join%20Community-7289da?logo=discord&logoColor=white)](https://discord.gg/rAXesGcT2t)
11
+ [![YouTube](https://img.shields.io/badge/YouTube-Chaos%20Redux-ff0000?logo=youtube&logoColor=white)](https://www.youtube.com/@ChaosRedux)
12
+ [![Ko-fi](https://img.shields.io/badge/Ko--fi-Support%20the%20Project-ff5e5b?logo=kofi&logoColor=white)](https://ko-fi.com/klimpaskov)
13
+ [![Download](https://img.shields.io/github/v/release/klimPaskov/hoi4-agent-tools?label=Release&logo=github&color=2ea44f)](https://github.com/klimPaskov/hoi4-agent-tools/releases/latest)
14
+ [![Issues](https://img.shields.io/badge/Bugs%20%26%20Ideas-Issues-24292f?logo=github&logoColor=white)](https://github.com/klimPaskov/hoi4-agent-tools/issues)
6
15
 
7
- An [MCP](https://modelcontextprotocol.io/) server that lets a coding agent such as Claude Code, Codex or Cursor read, preview, analyse and safely edit a Hearts of Iron IV mod.
8
- Your agent gets cited answers from the game's own documentation, pictures of focus trees, technology folders, interfaces and maps, and checks that catch load errors before you launch the game.
9
-
10
- [![A large focus tree rendered from mod source](docs/images/comparisons/holy-realm-mcp.png)](docs/images/comparisons/holy-realm-mcp.png)
16
+ An [MCP](https://modelcontextprotocol.io/) server that helps coding agents build and edit Hearts of Iron IV mods.
17
+ Make focus trees, events and decisions. Edit maps and generate a new world. Preview windows, trees and maps, find the right rule in the docs, fix broken script and balance AI choices.
11
18
 
12
19
  ## What it does
13
20
 
14
- | Area | Ask your agent, for example |
15
- | ---------------------- | ---------------------------------------------------------------------------------------------------------- |
16
- | Find the rule | "Find the installed documentation for `save_event_target_as` and show a vanilla usage with file and line." |
17
- | Check before launch | "Check `events/my_events.txt` for scope errors, then group the last `error.log` by category." |
18
- | See the tree | "Render this mod's largest focus tree and list focuses with missing icons." |
19
- | Preview the window | "Preview this scripted GUI at 100% and 125% UI scale and check for clipped text." |
20
- | Read the map | "Find Brandenburg, list its provinces and neighbours, and show the files a change would touch." |
21
- | Reshape the map | "Split Brandenburg in two and update every event and focus that names it." |
22
- | Build a new world | "Create a 2048x1024 world with six countries, then list what still needs hand-made content." |
23
- | Follow the events | "Trace this event's options and follow-up events and point out unresolved calls." |
24
- | Weigh the AI | "Show the AI weights of event `my_mod.12`'s options, then compare them before and after my change." |
25
- | Plan new content | "Give me the next free event ID in my namespace and summarize what the mod defines." |
26
- | Edit with a safety net | "Add a new branch to this focus tree" — the edit is validated before anything is written. |
27
-
28
- The server never launches the game.
29
- Previews and tests are built from source files, so they show what the source says, not what a running campaign did.
21
+ Your agent can build and edit focus trees, events, decisions and maps, and generate a new world.
22
+ It previews focus trees, technology folders, windows and maps, finds the right rule in the game's documentation, fixes broken script and tunes AI choices.
23
+ You describe what you want; the agent picks the right tools.
30
24
 
31
25
  ## Quick start
32
26
 
@@ -45,14 +39,17 @@ Previews and tests are built from source files, so they show what the source say
45
39
 
46
40
  Paste the `globalInstall` block into a JSON client such as Claude Code, Claude Desktop or Cursor, or the `codexTomlGlobal` block into Codex's `config.toml`.
47
41
 
48
- 4. Open your mod folder as the agent's workspace and try: "Render this mod's largest focus tree."
42
+ 4. Open your mod folder as the agent's workspace and tell it what you want to build or fix.
49
43
 
50
44
  The [setup guide](docs/setup.md) covers several mods, explicit game paths, local wiki snapshots and troubleshooting with `--discover` and `--diagnose`.
51
45
 
52
- [Watch the Agent Tools demo](https://github.com/klimPaskov/Agentic-HOI4-Modding/releases/latest/download/hoi4_agent_tools_promo.mp4) or use [HOI4 Mod Setup](https://github.com/klimPaskov/HOI4-Mod-Setup/releases/latest) to install and register the verified server for your mod.
46
+ [Watch the Agent Tools demo](https://github.com/klimPaskov/Agentic-HOI4-Modding/releases/latest/download/hoi4_agent_tools_promo.mp4) or use [HOI4 Mod Setup](https://github.com/klimPaskov/HOI4-Mod-Setup/releases/latest) to install and connect the server for your mod.
47
+ Part of the HOI4 modding toolkit with [Agentic HOI4 Modding](https://github.com/klimPaskov/Agentic-HOI4-Modding) (skills and instructions for your agent) and [HOI4 Live Tools](https://github.com/klimPaskov/HOI4-Live-Tools) (testing in the running game).
53
48
 
54
49
  ## Examples
55
50
 
51
+ [![A large focus tree rendered from mod source](docs/images/comparisons/holy-realm-mcp.png)](docs/images/comparisons/holy-realm-mcp.png)
52
+
56
53
  Each pair shows a capture from the game next to the server's preview of the same source.
57
54
 
58
55
  | In game | Server preview |
@@ -61,40 +58,19 @@ Each pair shows a capture from the game next to the server's preview of the same
61
58
  | [![Infantry technology folder in game](docs/images/comparisons/infantry-ingame.png)](docs/images/comparisons/infantry-ingame.png) | [![Infantry technology folder preview](docs/images/comparisons/infantry-mcp.png)](docs/images/comparisons/infantry-mcp.png) |
62
59
  | [![Video options in game](docs/images/comparisons/gui/options-video-ingame.png)](docs/images/comparisons/gui/options-video-ingame.png) | [![Video options preview](docs/images/comparisons/gui/options-video-mcp.png)](docs/images/comparisons/gui/options-video-mcp.png) |
63
60
 
64
- In focus previews, dotted lines mark alternative prerequisites: any one of them unlocks the focus.
65
- The focus screen in the game has its own framing, labels and icons, and the video options preview has no observed gamma value.
66
- More comparisons, with their source revisions and fidelity notes, are in the [focus and technology gallery](https://github.com/klimPaskov/hoi4-agent-tools/blob/main/docs/visual-comparisons.md), the [interface gallery](https://github.com/klimPaskov/hoi4-agent-tools/blob/main/docs/gui-comparisons.md) and the [examples guide](docs/examples.md), which also shows map and event renders.
61
+ See more in the [examples gallery](docs/examples.md).
67
62
 
68
63
  ## Tools
69
64
 
70
- The default server has 28 tools.
71
- Agents pick them from ordinary language, so you rarely need their names.
72
-
73
- | Area | Tools | Writes |
74
- | --------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------- |
75
- | References and source | `reference_context`, `reference_search`, `reference_read`, `source_lookup`, `script_validate`, `error_log`, `mod_index` | — |
76
- | Focus trees | `focus_inspect`, `focus_render`, `focus_rewrite` | review files; mod files |
77
- | Scripted GUIs | `gui_inspect`, `gui_render`, `gui_rewrite` | review files; mod files |
78
- | Maps | `map_inspect`, `map_rewrite` | review files; mod files |
79
- | Events | `event_inspect`, `event_render` | — |
80
- | Technology and doctrines | `tech_inspect`, `tech_render` | — |
81
- | AI weights and timing | `probability_inspect`, `probability_analyze` | — |
82
- | Decisions, impact and tests | `impact_inspect`, `decision_inspect`, `mechanic_test`, `package_check`, `scenario_test` | — |
83
- | Background jobs | `job_inspect`, `job_cancel` | job state |
84
-
85
- Every name starts with `hoi4.`.
86
- "Review files" are images and reports in the server's own storage; only the three `*_rewrite` tools change mod files.
87
- The [tool reference](docs/tools.md) gives each tool's purpose, prompts, resources and long-running behaviour.
88
-
89
- ## Safety and limits
90
-
91
- - The server reads the mod, its dependencies, the installed game, local reference folders and the game's error log.
92
- - Inspection and rendering never change mod files.
93
- The focus, GUI and map rewrite tools validate a proposed edit before writing it and write it as a journaled transaction, so an interrupted write is restored rather than left half-applied.
94
- - Large renders and reports are stored as linked files, and documentation is returned as cited sections rather than whole manuals.
95
- - The server does not launch or control the game.
96
- Previews and source tests cannot prove live clicks, AI choices or engine timing; a source score alone does not establish an AI click probability.
97
- - Your repository instructions and review process still apply.
65
+ The server gives your agent previews, documentation, script analysis and editing tools.
66
+ The agent combines them with its file editor to build events and decisions, repair scripts and adjust AI weights.
67
+ The [tool reference](docs/tools.md) lists exact names, arguments and results for integrations.
68
+
69
+ ## FAQ
70
+
71
+ **Where do edits go?** Changes are written to your mod folder; use Git to review or undo them.
72
+
73
+ **Does it run the game?** No. Previews come from mod source; test gameplay in HOI4 to see live clicks, AI choices and timing.
98
74
 
99
75
  ## Configuration
100
76
 
@@ -176,6 +176,24 @@ function evaluateAssignmentCore(assignment, scenario, candidate, definitions, he
176
176
  const malformed = malformedBlock(assignment.value, candidate);
177
177
  if (malformed !== undefined)
178
178
  return malformed;
179
+ if (key === 'has_country_flag' || key === 'has_global_flag' || key === 'has_state_flag') {
180
+ // The block form tests a flag's stored value. A scenario declares that value as a number
181
+ // under `flag.<name>` (or `global_flag.<name>` for global flags).
182
+ const flag = firstScalar(assignment.value, 'flag')?.value;
183
+ const expected = firstScalar(assignment.value, 'value')?.value;
184
+ if (flag !== undefined) {
185
+ const names = key === 'has_global_flag' ? [`global_flag.${flag}`, `flag.${flag}`] : [`flag.${flag}`];
186
+ for (const name of names) {
187
+ const declared = scopeStateValue(scenario, name, scopeContext);
188
+ if (typeof declared === 'number' && expected !== undefined)
189
+ return { state: declared === Number(expected) ? 'true' : 'false', unresolved: [] };
190
+ if (typeof declared === 'boolean' && expected === undefined)
191
+ return { state: declared ? 'true' : 'false', unresolved: [] };
192
+ }
193
+ if (expected !== undefined)
194
+ return unresolved(assignment, candidate, `Scenario does not declare the value of flag ${flag}`, `flag.${flag}`);
195
+ }
196
+ }
179
197
  if (key === 'AND' ||
180
198
  key === 'hidden_trigger' ||
181
199
  key === 'custom_trigger_tooltip' ||
@@ -1,5 +1,7 @@
1
1
  import { spawn } from 'node:child_process';
2
+ import { closeSync, fstatSync, openSync, readSync, readdirSync, rmSync, statSync } from 'node:fs';
2
3
  import { mkdir } from 'node:fs/promises';
4
+ import path from 'node:path';
3
5
  import { fileURLToPath } from 'node:url';
4
6
  import { setTimeout as delay } from 'node:timers/promises';
5
7
  import { JobService } from './job-service.js';
@@ -9,11 +11,15 @@ import { SharedRequestCapacity } from './shared-request-capacity.js';
9
11
  import { containedGeneratedPath } from './workspace.js';
10
12
  import { ServiceError } from './result.js';
11
13
  import { isDomainToolName } from './domain-tools.js';
14
+ import { classifyWorkerExit, defaultWorkerHeapMiB, } from './worker-exit.js';
12
15
  const maximumCheckpointRecoveries = 2;
13
16
  const maximumPreDispatchRecoveries = 2;
14
- /** An exit with no outcome is most often the worker running out of memory on a large mod. */
15
- const WORKER_EXIT_HINT = '; if the mod is large the worker may have run out of memory, so retry the call or raise jobWorkerMaxHeapMiB';
16
- const WORKER_EXIT_MESSAGE = `The worker stopped without publishing a terminal job outcome${WORKER_EXIT_HINT}`;
17
+ /** Used only when the exit status of the worker is unknown to this process. */
18
+ const WORKER_EXIT_MESSAGE = 'The worker stopped without publishing a terminal job outcome; its exit status is not available to this server, so retry the call';
19
+ /** Bytes of a worker's standard error read back when it exits unexpectedly. */
20
+ const STDERR_TAIL_BYTES = 16_384;
21
+ /** Standard-error logs of workers that exited unexpectedly kept for diagnosis. */
22
+ const RETAINED_STDERR_LOGS = 10;
17
23
  // Native and IPC codes are fixed vocabulary. Error messages can contain paths or source
18
24
  // text, so only an allowlisted code and the fixed coordination stage are reported.
19
25
  const nativeFailureCodes = new Set([
@@ -62,13 +68,68 @@ export function workerFailure(error, stage) {
62
68
  };
63
69
  }
64
70
  /** Stop a supervised worker and wait, bounded, until the operating system reports its exit. */
65
- async function stopChild(child) {
71
+ async function stopChild(child, stopped) {
66
72
  if (child.exitCode !== null || child.signalCode !== null)
67
73
  return;
74
+ stopped?.add(child);
68
75
  const exited = new Promise((resolve) => child.once('exit', () => resolve()));
69
76
  child.kill('SIGKILL');
70
77
  await Promise.race([exited, delay(10_000)]);
71
78
  }
79
+ /** Read the last bytes of a worker's standard-error log; empty when it cannot be read. */
80
+ function readTail(file) {
81
+ try {
82
+ const fd = openSync(file, 'r');
83
+ try {
84
+ const size = fstatSync(fd).size;
85
+ const length = Math.min(size, STDERR_TAIL_BYTES);
86
+ const buffer = Buffer.alloc(length);
87
+ readSync(fd, buffer, 0, length, size - length);
88
+ return buffer.toString('utf8');
89
+ }
90
+ finally {
91
+ closeSync(fd);
92
+ }
93
+ }
94
+ catch {
95
+ return '';
96
+ }
97
+ }
98
+ /** Keep only the newest few worker logs; the rest are older diagnostics. */
99
+ function pruneWorkerLogs(logRoot) {
100
+ try {
101
+ const logs = readdirSync(logRoot)
102
+ .filter((name) => name.startsWith('worker-') && name.endsWith('.log'))
103
+ .map((name) => ({ name, modified: statSync(path.join(logRoot, name)).mtimeMs }))
104
+ .sort((left, right) => right.modified - left.modified);
105
+ // A recent log may belong to a worker that is still running, so only older ones go.
106
+ const cutoff = Date.now() - 10 * 60_000;
107
+ for (const { name, modified } of logs.slice(RETAINED_STDERR_LOGS))
108
+ if (modified < cutoff)
109
+ rmSync(path.join(logRoot, name), { force: true });
110
+ }
111
+ catch {
112
+ // Diagnostics are best effort and must never change a job outcome.
113
+ }
114
+ }
115
+ /**
116
+ * Remove the log of a worker that ended normally or because the host stopped it. Keep the
117
+ * newest few logs of workers that ended unexpectedly, because their stack is the evidence.
118
+ */
119
+ function retainOrRemoveLog(file, evidence, logRoot) {
120
+ try {
121
+ const unexpected = !evidence.stoppedByHost &&
122
+ (evidence.signal !== null || (evidence.code !== null && evidence.code !== 0));
123
+ if (!unexpected) {
124
+ rmSync(file, { force: true });
125
+ return;
126
+ }
127
+ pruneWorkerLogs(logRoot);
128
+ }
129
+ catch {
130
+ // Diagnostics are best effort and must never change a job outcome.
131
+ }
132
+ }
72
133
  /**
73
134
  * Bounded, fixed-entry child execution. Client disconnection never kills a worker; an
74
135
  * overdue or unresponsive cancelled read-only worker is stopped by its supervising host.
@@ -78,12 +139,14 @@ export class JobWorkerHost {
78
139
  jobs;
79
140
  capacity;
80
141
  scheduler;
142
+ logRoot;
81
143
  owner = {};
82
- constructor(engine, jobs, capacity, scheduler) {
144
+ constructor(engine, jobs, capacity, scheduler, logRoot) {
83
145
  this.engine = engine;
84
146
  this.jobs = jobs;
85
147
  this.capacity = capacity;
86
148
  this.scheduler = scheduler;
149
+ this.logRoot = logRoot;
87
150
  }
88
151
  static async create(engine, jobs) {
89
152
  const state = engine.resolver.serverState();
@@ -91,8 +154,11 @@ export class JobWorkerHost {
91
154
  throw new ServiceError('JOB_STORAGE_UNAVAILABLE', 'Workers require persistent server state');
92
155
  const root = await containedGeneratedPath(state.root, 'job-workers');
93
156
  await mkdir(root, { recursive: true, mode: 0o700 });
157
+ const logRoot = path.join(root, 'logs');
158
+ await mkdir(logRoot, { recursive: true, mode: 0o700 });
159
+ pruneWorkerLogs(logRoot);
94
160
  const config = engine.resolver.config();
95
- return new JobWorkerHost(engine, jobs ?? (await JobService.create(engine.resolver)), new SharedRequestCapacity(root, config.maxSharedTools), new RequestScheduler(config.maxConcurrentTools));
161
+ return new JobWorkerHost(engine, jobs ?? (await JobService.create(engine.resolver)), new SharedRequestCapacity(root, config.maxSharedTools), new RequestScheduler(config.maxConcurrentTools), logRoot);
96
162
  }
97
163
  async run(workspaceId, id, principal) {
98
164
  const initial = await this.jobs.get(workspaceId, id, principal);
@@ -153,7 +219,7 @@ export class JobWorkerHost {
153
219
  if (liveness !== 'alive')
154
220
  throw new ServiceError(liveness === 'unknown' ? 'JOB_OWNER_UNRESOLVED' : 'JOB_WORKER_EXIT', liveness === 'unknown'
155
221
  ? 'The replacement execution owner cannot be verified on this host'
156
- : `The replacement worker stopped without publishing a terminal job outcome${WORKER_EXIT_HINT}`);
222
+ : `The replacement worker stopped without publishing a terminal job outcome; its exit status belongs to the server that started it, so retry the call`);
157
223
  await delay(100);
158
224
  }
159
225
  }
@@ -206,7 +272,7 @@ export class JobWorkerHost {
206
272
  if (reused !== undefined && error instanceof ServiceError)
207
273
  throw new ServiceError('REQUEST_LEASE_HANDOFF_LOST', 'The warm worker stopped before it received the job');
208
274
  if (reused === undefined)
209
- await stopChild(child);
275
+ await stopChild(child, this.stoppedByHost);
210
276
  throw error;
211
277
  }
212
278
  progress.stage = 'dispatch';
@@ -240,7 +306,7 @@ export class JobWorkerHost {
240
306
  const unresponsive = cancellationSeenAt !== undefined &&
241
307
  Date.now() - cancellationSeenAt > jobCancelGraceSeconds * 1000;
242
308
  if (overdue || unresponsive) {
243
- await stopChild(child);
309
+ await stopChild(child, this.stoppedByHost);
244
310
  await this.jobs.failInterrupted(workspaceId, id, {
245
311
  code: 'JOB_DEADLINE_EXCEEDED',
246
312
  message: `The job ran longer than ${jobDeadlineSeconds} seconds and was stopped; narrow its selector or limits, or raise jobDeadlineSeconds`,
@@ -248,18 +314,17 @@ export class JobWorkerHost {
248
314
  return;
249
315
  }
250
316
  }
251
- if (child.pid !== undefined) {
252
- try {
253
- process.kill(child.pid, 0);
254
- }
255
- catch (error) {
256
- if (error.code === 'ESRCH')
257
- throw new ServiceError('JOB_WORKER_EXIT', WORKER_EXIT_MESSAGE);
258
- }
317
+ if (child.exitCode !== null || child.signalCode !== null) {
318
+ const evidence = await this.exits.get(child);
319
+ throw new ServiceError('JOB_WORKER_EXIT', evidence === undefined ? WORKER_EXIT_MESSAGE : classifyWorkerExit(evidence).message);
259
320
  }
260
321
  await delay(100);
261
322
  }
262
323
  }
324
+ /** Workers this host stopped itself, so their exit is not reported as a fault. */
325
+ stoppedByHost = new WeakSet();
326
+ /** Settles with the exit status and last standard error of each worker this host started. */
327
+ exits = new WeakMap();
263
328
  /** Idle warm workers of this host, most recently used last. */
264
329
  idleWorkers = [];
265
330
  /**
@@ -268,6 +333,7 @@ export class JobWorkerHost {
268
333
  */
269
334
  reportedIdle = new WeakSet();
270
335
  watched = new WeakSet();
336
+ workerSequence = 0;
271
337
  watchIdle(child) {
272
338
  if (this.watched.has(child))
273
339
  return;
@@ -302,16 +368,38 @@ export class JobWorkerHost {
302
368
  async startWorker(maxHeapMiB) {
303
369
  const sourceMode = import.meta.url.endsWith('.ts');
304
370
  const entry = fileURLToPath(new URL(sourceMode ? './job-worker.ts' : './job-worker.js', import.meta.url));
305
- const child = spawn(process.execPath, [
306
- ...(maxHeapMiB === undefined ? [] : [`--max-old-space-size=${maxHeapMiB}`]),
307
- ...(sourceMode ? ['--import', import.meta.resolve('tsx')] : []),
308
- entry,
309
- ], {
310
- detached: true,
311
- cwd: process.cwd(),
312
- stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
313
- windowsHide: true,
314
- });
371
+ const heapLimitMiB = maxHeapMiB ?? defaultWorkerHeapMiB();
372
+ const stderrLog = path.join(this.logRoot, `worker-${process.pid}-${++this.workerSequence}.log`);
373
+ const stderrFd = openSync(stderrLog, 'w', 0o600);
374
+ let child;
375
+ try {
376
+ child = spawn(process.execPath, [
377
+ `--max-old-space-size=${heapLimitMiB}`,
378
+ ...(sourceMode ? ['--import', import.meta.resolve('tsx')] : []),
379
+ entry,
380
+ ], {
381
+ detached: true,
382
+ cwd: process.cwd(),
383
+ stdio: ['ignore', 'ignore', stderrFd, 'ipc'],
384
+ windowsHide: true,
385
+ });
386
+ }
387
+ finally {
388
+ closeSync(stderrFd);
389
+ }
390
+ this.exits.set(child, new Promise((resolve) => {
391
+ child.once('exit', (code, signal) => {
392
+ const evidence = {
393
+ code,
394
+ signal,
395
+ stderrTail: readTail(stderrLog),
396
+ stoppedByHost: this.stoppedByHost.has(child),
397
+ heapLimitMiB,
398
+ };
399
+ retainOrRemoveLog(stderrLog, evidence, this.logRoot);
400
+ resolve(evidence);
401
+ });
402
+ }));
315
403
  try {
316
404
  await new Promise((resolve, reject) => {
317
405
  const onExit = (code, signal) => reject(new ServiceError('JOB_WORKER_STARTUP_EXIT', `The worker exited before completing its startup handshake (phase waiting_for_ready, exit ${code ?? 'none'}, signal ${signal ?? 'none'})`));
@@ -8,6 +8,7 @@ import { JobExecutor } from './job-executor.js';
8
8
  import { JobService } from './job-service.js';
9
9
  import { WorkspaceResolver } from './workspace.js';
10
10
  import { registerWorkerOperations, workerDomain } from './job-operation-registration.js';
11
+ import { workerErrorMarker } from './worker-exit.js';
11
12
  // Fixed internal process entry point. It accepts only a trusted host's structured IPC
12
13
  // message, never a script path, module name, callback, command, or transport request.
13
14
  //
@@ -32,9 +33,11 @@ let started = false;
32
33
  let idleTimer;
33
34
  /** Share of the heap limit a worker may still hold after a job and a full collection. */
34
35
  const WARM_HEAP_SHARE = 0.4;
36
+ /** Absolute ceiling on that retained heap, so a large heap limit does not keep gigabytes idle. */
37
+ const WARM_HEAP_MAXIMUM_BYTES = 1_536 * 1024 * 1024;
35
38
  /** Whether the heap, after a full collection, is small enough to serve another job. */
36
39
  function heapAllowsReuse() {
37
- const limit = getHeapStatistics().heap_size_limit * WARM_HEAP_SHARE;
40
+ const limit = Math.min(getHeapStatistics().heap_size_limit * WARM_HEAP_SHARE, WARM_HEAP_MAXIMUM_BYTES);
38
41
  if (getHeapStatistics().used_heap_size <= limit)
39
42
  return true;
40
43
  try {
@@ -105,7 +108,9 @@ process.on('message', (value) => {
105
108
  if (idleTimer !== undefined)
106
109
  clearTimeout(idleTimer);
107
110
  void runJob(value)
108
- .catch(() => {
111
+ .catch((error) => {
112
+ // The host reads this line back to tell an uncaught job error from a memory failure.
113
+ process.stderr.write(workerErrorMarker(error));
109
114
  process.exitCode = 1;
110
115
  })
111
116
  .finally(() => {