hoi4-agent-tools 3.10.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +131 -0
- package/README.md +91 -147
- package/dist/bin/http.js +10 -0
- package/dist/bin/setup.js +3 -2
- package/dist/bin/stdio.js +10 -0
- package/dist/hoi4_agent_tools/core/artifacts.js +40 -5
- package/dist/hoi4_agent_tools/core/canonical.js +18 -4
- package/dist/hoi4_agent_tools/core/condition-evaluator.js +18 -0
- package/dist/hoi4_agent_tools/core/configuration.js +7 -0
- package/dist/hoi4_agent_tools/core/domain-scan-patterns.js +12 -0
- package/dist/hoi4_agent_tools/core/domain-tools.js +65 -0
- package/dist/hoi4_agent_tools/core/domain-validation.js +5 -2
- package/dist/hoi4_agent_tools/core/engine.js +29 -2
- package/dist/hoi4_agent_tools/core/index.js +2 -2
- package/dist/hoi4_agent_tools/core/job-executor.js +14 -1
- package/dist/hoi4_agent_tools/core/job-operation-registration.js +15 -7
- package/dist/hoi4_agent_tools/core/job-store.js +78 -5
- package/dist/hoi4_agent_tools/core/job-transactions.js +31 -2
- package/dist/hoi4_agent_tools/core/job-worker-host.js +314 -127
- package/dist/hoi4_agent_tools/core/job-worker.js +96 -37
- package/dist/hoi4_agent_tools/core/operation-result.js +123 -10
- package/dist/hoi4_agent_tools/core/operation-tasks.js +49 -89
- package/dist/hoi4_agent_tools/core/persistent-analysis-cache.js +25 -21
- package/dist/hoi4_agent_tools/core/server-registry.js +22 -5
- package/dist/hoi4_agent_tools/core/shared-request-capacity.js +26 -1
- package/dist/hoi4_agent_tools/core/source/cache.js +20 -10
- package/dist/hoi4_agent_tools/core/source/lexer.js +11 -3
- package/dist/hoi4_agent_tools/core/temporary-directories.js +293 -0
- package/dist/hoi4_agent_tools/core/transactions.js +114 -23
- package/dist/hoi4_agent_tools/core/worker-exit.js +80 -0
- package/dist/hoi4_agent_tools/core/workspace.js +84 -8
- package/dist/hoi4_agent_tools/decision/ai.js +1 -1
- package/dist/hoi4_agent_tools/decision/service.js +28 -0
- package/dist/hoi4_agent_tools/decision/source-inventory.js +7 -5
- package/dist/hoi4_agent_tools/event/job-operations.js +9 -5
- package/dist/hoi4_agent_tools/event/operations.js +14 -1
- package/dist/hoi4_agent_tools/focus/job-operations.js +1 -2
- package/dist/hoi4_agent_tools/focus/tool-operations.js +1 -1
- package/dist/hoi4_agent_tools/gui/assets.js +3 -0
- package/dist/hoi4_agent_tools/gui/dds.js +8 -4
- package/dist/hoi4_agent_tools/gui/layout.js +45 -5
- package/dist/hoi4_agent_tools/gui/native-state.js +244 -0
- package/dist/hoi4_agent_tools/gui/renderer.js +12 -7
- package/dist/hoi4_agent_tools/gui/source-graph.js +17 -4
- package/dist/hoi4_agent_tools/gui/studio.js +33 -21
- package/dist/hoi4_agent_tools/gui/validators.js +8 -6
- package/dist/hoi4_agent_tools/map/bmp.js +1 -1
- package/dist/hoi4_agent_tools/map/derived.js +748 -0
- package/dist/hoi4_agent_tools/map/job-operations.js +7 -3
- package/dist/hoi4_agent_tools/map/model.js +103 -0
- package/dist/hoi4_agent_tools/map/operations.js +427 -58
- package/dist/hoi4_agent_tools/map/references.js +483 -0
- package/dist/hoi4_agent_tools/map/service.js +136 -61
- package/dist/hoi4_agent_tools/map/tool-operations.js +2 -1
- package/dist/hoi4_agent_tools/map/validation.js +3 -0
- package/dist/hoi4_agent_tools/map/world.js +1280 -0
- package/dist/hoi4_agent_tools/mcp/prompts/probability.js +2 -2
- package/dist/hoi4_agent_tools/mcp/server/argument-errors.js +72 -0
- package/dist/hoi4_agent_tools/mcp/server/create-modern-operations.js +23 -5
- package/dist/hoi4_agent_tools/mcp/server/create.js +7 -0
- package/dist/hoi4_agent_tools/mcp/server/instructions.js +1 -1
- package/dist/hoi4_agent_tools/mcp/server/request-lifecycle.js +24 -30
- package/dist/hoi4_agent_tools/mcp/server/tool-listing.js +45 -0
- package/dist/hoi4_agent_tools/mcp/tools/analysis.js +28 -8
- package/dist/hoi4_agent_tools/mcp/tools/event.js +19 -30
- package/dist/hoi4_agent_tools/mcp/tools/focus.js +5 -13
- package/dist/hoi4_agent_tools/mcp/tools/gui.js +6 -6
- package/dist/hoi4_agent_tools/mcp/tools/job.js +4 -4
- package/dist/hoi4_agent_tools/mcp/tools/map.js +15 -18
- package/dist/hoi4_agent_tools/mcp/tools/probability.js +42 -48
- package/dist/hoi4_agent_tools/mcp/tools/reference.js +26 -9
- package/dist/hoi4_agent_tools/mcp/tools/technology.js +17 -30
- package/dist/hoi4_agent_tools/mcp/transports/http.js +28 -4
- package/dist/hoi4_agent_tools/mcp/transports/stdio-lifetime.js +31 -8
- package/dist/hoi4_agent_tools/probability/job-operations.js +4 -9
- package/dist/hoi4_agent_tools/probability/operations.js +175 -13
- package/dist/hoi4_agent_tools/probability/source-analysis.js +24 -1
- package/dist/hoi4_agent_tools/reference/error-log.js +359 -0
- package/dist/hoi4_agent_tools/reference/known-mistakes.js +307 -0
- package/dist/hoi4_agent_tools/reference/known-mistakes.json +154 -0
- package/dist/hoi4_agent_tools/reference/mod-index.js +101 -0
- package/dist/hoi4_agent_tools/reference/operations.js +88 -19
- package/dist/hoi4_agent_tools/reference/player-text.js +4 -3
- package/dist/hoi4_agent_tools/reference/script-validation.js +19 -1
- package/dist/hoi4_agent_tools/reference/service.js +6 -0
- package/dist/hoi4_agent_tools/runtime.js +14 -1
- package/dist/hoi4_agent_tools/schemas/event.js +78 -19
- package/dist/hoi4_agent_tools/schemas/focus-requests.js +10 -0
- package/dist/hoi4_agent_tools/schemas/lenient-arguments.js +266 -0
- package/dist/hoi4_agent_tools/schemas/map-requests.js +63 -20
- package/dist/hoi4_agent_tools/schemas/map.js +122 -3
- package/dist/hoi4_agent_tools/schemas/mod-tools.js +181 -0
- package/dist/hoi4_agent_tools/schemas/probability-requests.js +119 -9
- package/dist/hoi4_agent_tools/schemas/reference.js +13 -0
- package/dist/hoi4_agent_tools/schemas/scenarios.js +1 -3
- package/dist/hoi4_agent_tools/schemas/script-validation.js +13 -6
- package/dist/hoi4_agent_tools/schemas/technology.js +79 -21
- package/dist/hoi4_agent_tools/technology/job-operations.js +3 -4
- package/dist/hoi4_agent_tools/technology/operations.js +14 -1
- package/dist/hoi4_agent_tools/version.js +1 -1
- package/docs/README.md +7 -4
- package/docs/analysis.md +15 -2
- package/docs/configuration.md +281 -0
- package/docs/development.md +26 -2
- package/docs/events.md +29 -6
- package/docs/examples.md +13 -28
- package/docs/focus.md +127 -5
- package/docs/gui.md +105 -21
- package/docs/helper-expansion.md +9 -9
- package/docs/http.md +3 -1
- package/docs/jobs.md +23 -14
- package/docs/map.md +199 -7
- package/docs/mechanics.md +6 -2
- package/docs/probability.md +46 -16
- package/docs/reference.md +73 -5
- package/docs/setup.md +45 -14
- package/docs/technology.md +28 -4
- package/docs/third-party-notices.md +11 -0
- package/docs/tools.md +150 -42
- package/docs/troubleshooting.md +61 -0
- package/examples/clients/codex-config.toml +5 -2
- package/examples/clients/generic.json +2 -2
- package/examples/config/config.json +7 -0
- package/npm-shrinkwrap.json +151 -156
- package/package.json +13 -8
- package/schemas/configuration.schema.json +144 -128
- package/schemas/continuous-focus-palette.schema.json +1 -1
- package/schemas/custom-weighted-pool.schema.json +1 -1
- package/schemas/decision-inspect-input.schema.json +1 -1
- package/schemas/error-log-input.schema.json +55 -0
- package/schemas/event-feature-manifest.schema.json +1 -1
- package/schemas/focus-plan.schema.json +1 -1
- package/schemas/focus-planning-sidecar.schema.json +1 -1
- package/schemas/gui-animation-source.schema.json +1 -1
- package/schemas/gui-helper.schema.json +1 -1
- package/schemas/gui-scenario.schema.json +1 -1
- package/schemas/helper-expansion-request.schema.json +1 -1
- package/schemas/helper-expansion-summary.schema.json +1 -1
- package/schemas/impact-inspect-input.schema.json +1 -1
- package/schemas/map-operation.schema.json +597 -126
- package/schemas/mechanic-test-input.schema.json +1 -1
- package/schemas/mod-index-input.schema.json +51 -0
- package/schemas/operation-result.schema.json +2 -2
- package/schemas/package-check-input.schema.json +1 -1
- package/schemas/package-manifest.schema.json +1 -1
- package/schemas/probability-analysis-result.schema.json +1 -1
- package/schemas/probability-compare-input.schema.json +1 -1
- package/schemas/probability-evaluate-input.schema.json +1 -1
- package/schemas/probability-inspect-input.schema.json +1 -1
- package/schemas/probability-render-input.schema.json +1 -1
- package/schemas/probability-scenario-set.schema.json +1 -1
- package/schemas/probability-sequence-input.schema.json +1 -1
- package/schemas/probability-simulate-input.schema.json +1 -1
- package/schemas/probability-sweep-input.schema.json +1 -1
- package/schemas/reference-context-input.schema.json +1 -1
- package/schemas/reference-read-input.schema.json +1 -1
- package/schemas/reference-search-input.schema.json +1 -1
- package/schemas/scenario-suite.schema.json +3 -5
- package/schemas/scenario-test-input.schema.json +3 -5
- package/schemas/script-validate-input.schema.json +1 -1
- package/schemas/shared-scenario.schema.json +1 -1
- package/schemas/source-lookup-input.schema.json +1 -1
- package/server.json +3 -3
|
@@ -11,7 +11,7 @@ const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u);
|
|
|
11
11
|
const entryKindSchema = z.enum(['source-document', 'index-segment']);
|
|
12
12
|
const unsignedEntrySchema = z
|
|
13
13
|
.object({
|
|
14
|
-
version: z.literal(
|
|
14
|
+
version: z.literal(2),
|
|
15
15
|
kind: entryKindSchema,
|
|
16
16
|
scopeHash: digestSchema,
|
|
17
17
|
address: digestSchema,
|
|
@@ -23,6 +23,15 @@ const unsignedEntrySchema = z
|
|
|
23
23
|
.strict();
|
|
24
24
|
const entrySchema = unsignedEntrySchema.extend({ authenticationTag: digestSchema }).strict();
|
|
25
25
|
const entryFilePattern = /^(?<address>[a-f0-9]{64})\.json$/u;
|
|
26
|
+
/**
|
|
27
|
+
* What the authentication tag covers: every field but the payload bytes, which the tag binds
|
|
28
|
+
* through their SHA-256. A reader hashes the payload once and compares it, so the megabytes of
|
|
29
|
+
* a parsed document do not pass through canonical JSON and the HMAC as well.
|
|
30
|
+
*/
|
|
31
|
+
function entryIdentity(entry) {
|
|
32
|
+
const { payload: _payload, ...identity } = entry;
|
|
33
|
+
return { kind: 'analysis-cache-entry.v2', entry: identity };
|
|
34
|
+
}
|
|
26
35
|
function sourceAddress(key) {
|
|
27
36
|
return hashCanonical({
|
|
28
37
|
version: 'persistent-source-document.v1',
|
|
@@ -114,10 +123,7 @@ export class PersistentAnalysisCache {
|
|
|
114
123
|
const payload = await this.read(scopeHash, 'source-document', documentAddress, signal);
|
|
115
124
|
if (payload !== undefined) {
|
|
116
125
|
try {
|
|
117
|
-
documents.importEncoded(key, payload
|
|
118
|
-
bytes: file.bytes,
|
|
119
|
-
sourcePath: file.displayPath,
|
|
120
|
-
});
|
|
126
|
+
documents.importEncoded(key, payload);
|
|
121
127
|
}
|
|
122
128
|
catch {
|
|
123
129
|
this.#invalidEntries += 1;
|
|
@@ -172,22 +178,23 @@ export class PersistentAnalysisCache {
|
|
|
172
178
|
}
|
|
173
179
|
const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
|
|
174
180
|
const documentAddress = sourceAddress(key);
|
|
175
|
-
|
|
176
|
-
if (
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
if (
|
|
181
|
+
// Stored entries are skipped before their payload is exported, which copies it.
|
|
182
|
+
if (!sourceEntries.has(documentAddress) ||
|
|
183
|
+
this.#invalidAddresses.has(`${scopeHash}\0source-document\0${documentAddress}`)) {
|
|
184
|
+
const document = documents.exportEncoded(key);
|
|
185
|
+
if (document !== undefined &&
|
|
186
|
+
(await retain('source-document', documentAddress, document))) {
|
|
180
187
|
sourceEntries.add(documentAddress);
|
|
181
188
|
}
|
|
182
189
|
}
|
|
183
190
|
for (const provinceDefinition of [false, true]) {
|
|
184
191
|
const address = indexSegmentAddress(file, provinceDefinition);
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
(indexEntries.has(address) &&
|
|
188
|
-
!this.#invalidAddresses.has(`${scopeHash}\0index-segment\0${address}`)))
|
|
192
|
+
if (indexEntries.has(address) &&
|
|
193
|
+
!this.#invalidAddresses.has(`${scopeHash}\0index-segment\0${address}`))
|
|
189
194
|
continue;
|
|
190
|
-
|
|
195
|
+
const segment = segments.exportEncoded(address);
|
|
196
|
+
if (segment !== undefined &&
|
|
197
|
+
(await retain('index-segment', address, Buffer.from(segment)))) {
|
|
191
198
|
indexEntries.add(address);
|
|
192
199
|
}
|
|
193
200
|
}
|
|
@@ -254,7 +261,7 @@ export class PersistentAnalysisCache {
|
|
|
254
261
|
if (entry.scopeHash !== scopeHash ||
|
|
255
262
|
entry.kind !== kind ||
|
|
256
263
|
entry.address !== address ||
|
|
257
|
-
!this.state.verifyJournal(
|
|
264
|
+
!this.state.verifyJournal(entryIdentity(entry), authenticationTag))
|
|
258
265
|
throw new Error('Persistent analysis cache authentication or identity is invalid');
|
|
259
266
|
const payload = Buffer.from(entry.payload, 'base64');
|
|
260
267
|
if (payload.length > this.maxSingleBytes || sha256Bytes(payload) !== entry.payloadHash)
|
|
@@ -289,7 +296,7 @@ export class PersistentAnalysisCache {
|
|
|
289
296
|
const target = await containedGeneratedPath(directory, `${digestSchema.parse(address)}.json`);
|
|
290
297
|
const invalidAddress = `${scopeHash}\0${kind}\0${address}`;
|
|
291
298
|
const unsigned = {
|
|
292
|
-
version:
|
|
299
|
+
version: 2,
|
|
293
300
|
kind,
|
|
294
301
|
scopeHash,
|
|
295
302
|
address,
|
|
@@ -300,10 +307,7 @@ export class PersistentAnalysisCache {
|
|
|
300
307
|
};
|
|
301
308
|
const bytes = Buffer.from(canonicalJson({
|
|
302
309
|
...unsigned,
|
|
303
|
-
authenticationTag: this.state.authenticateJournal(
|
|
304
|
-
kind: 'analysis-cache-entry.v1',
|
|
305
|
-
entry: unsigned,
|
|
306
|
-
}),
|
|
310
|
+
authenticationTag: this.state.authenticateJournal(entryIdentity(unsigned)),
|
|
307
311
|
}));
|
|
308
312
|
if (bytes.length > Math.ceil((this.maxSingleBytes * 4) / 3) + 4096) {
|
|
309
313
|
this.#oversized += 1;
|
|
@@ -11,6 +11,7 @@ const HEARTBEAT_MS = 60_000;
|
|
|
11
11
|
/** A record this old without an update belongs to a process that is blocked or gone. */
|
|
12
12
|
const STALE_MS = 5 * HEARTBEAT_MS;
|
|
13
13
|
const MAX_RECORDS = 512;
|
|
14
|
+
const ABANDONED_MS = 24 * STALE_MS;
|
|
14
15
|
const recordSchema = z
|
|
15
16
|
.object({
|
|
16
17
|
pid: z.number().int().positive(),
|
|
@@ -78,8 +79,13 @@ export class ServerRegistry {
|
|
|
78
79
|
const beat = () => void mkdir(root, { recursive: true })
|
|
79
80
|
.then(async () => {
|
|
80
81
|
const temporary = `${file}.${Date.now()}.tmp`;
|
|
81
|
-
|
|
82
|
-
|
|
82
|
+
try {
|
|
83
|
+
await writeFile(temporary, `${JSON.stringify(this.current())}\n`);
|
|
84
|
+
await rename(temporary, file);
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
await unlink(temporary).catch(() => undefined);
|
|
88
|
+
}
|
|
83
89
|
})
|
|
84
90
|
.catch(() => undefined);
|
|
85
91
|
beat();
|
|
@@ -109,9 +115,14 @@ export class ServerRegistry {
|
|
|
109
115
|
const servers = [];
|
|
110
116
|
let removed = 0;
|
|
111
117
|
if (root !== undefined) {
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
|
|
118
|
+
const entries = await readdir(root).catch(() => []);
|
|
119
|
+
// Temporary heartbeat files left by a failed rename.
|
|
120
|
+
for (const name of entries) {
|
|
121
|
+
const temporary = /^\d+\.json\.(\d+)\.tmp$/u.exec(name);
|
|
122
|
+
if (temporary !== null && Date.now() - Number(temporary[1]) > STALE_MS)
|
|
123
|
+
await unlink(path.join(root, name)).catch(() => undefined);
|
|
124
|
+
}
|
|
125
|
+
const names = entries.filter((name) => /^\d+\.json$/u.test(name)).slice(0, MAX_RECORDS);
|
|
115
126
|
for (const name of names) {
|
|
116
127
|
const file = path.join(root, name);
|
|
117
128
|
const parsed = recordSchema.safeParse(await readFile(file, 'utf8')
|
|
@@ -128,6 +139,12 @@ export class ServerRegistry {
|
|
|
128
139
|
continue;
|
|
129
140
|
}
|
|
130
141
|
const age = Date.now() - Date.parse(record.updatedAt);
|
|
142
|
+
// A record silent for this long belongs to an exited process whose pid was reused.
|
|
143
|
+
if (age > ABANDONED_MS) {
|
|
144
|
+
removed += 1;
|
|
145
|
+
await unlink(file).catch(() => undefined);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
131
148
|
servers.push({ ...record, state: age > STALE_MS ? 'stale' : 'live' });
|
|
132
149
|
}
|
|
133
150
|
}
|
|
@@ -20,11 +20,13 @@ export class SharedRequestCapacity {
|
|
|
20
20
|
stateRoot;
|
|
21
21
|
capacity;
|
|
22
22
|
deletePendingLimitMs;
|
|
23
|
+
leaseMaxAgeMs;
|
|
23
24
|
now;
|
|
24
25
|
constructor(stateRoot, capacity = 4, options = {}) {
|
|
25
26
|
this.stateRoot = stateRoot;
|
|
26
27
|
this.capacity = capacity;
|
|
27
28
|
this.deletePendingLimitMs = options.deletePendingLimitMs ?? 60_000;
|
|
29
|
+
this.leaseMaxAgeMs = options.leaseMaxAgeMs;
|
|
28
30
|
this.now = options.now ?? Date.now;
|
|
29
31
|
}
|
|
30
32
|
async run(signal, action) {
|
|
@@ -34,6 +36,7 @@ export class SharedRequestCapacity {
|
|
|
34
36
|
handoffToProcess: () => {
|
|
35
37
|
return Promise.reject(new ServiceError('REQUEST_LEASE_STORAGE_REQUIRED', 'Worker ownership transfer requires persistent capacity storage'));
|
|
36
38
|
},
|
|
39
|
+
reclaimFromProcess: () => Promise.resolve(),
|
|
37
40
|
});
|
|
38
41
|
const root = await containedGeneratedPath(await canonicalPath(this.stateRoot, signal), 'request-capacity', sha256Bytes(hostname().toLowerCase()).slice(0, 16));
|
|
39
42
|
await mkdir(root, { recursive: true });
|
|
@@ -112,6 +115,14 @@ export class SharedRequestCapacity {
|
|
|
112
115
|
owner = next;
|
|
113
116
|
transferredPid = pid;
|
|
114
117
|
},
|
|
118
|
+
reclaimFromProcess: async () => {
|
|
119
|
+
if (transferredPid === undefined)
|
|
120
|
+
return;
|
|
121
|
+
const next = path.join(slot, `${process.pid}-${randomUUID()}.lease`);
|
|
122
|
+
await retryWindowsSharing(() => rename(owner, next));
|
|
123
|
+
owner = next;
|
|
124
|
+
transferredPid = undefined;
|
|
125
|
+
},
|
|
115
126
|
});
|
|
116
127
|
}
|
|
117
128
|
finally {
|
|
@@ -137,6 +148,18 @@ export class SharedRequestCapacity {
|
|
|
137
148
|
await delay(sharingRetryDelay(waits) * (0.75 + Math.random() / 2), undefined, { signal });
|
|
138
149
|
}
|
|
139
150
|
}
|
|
151
|
+
async leaseExpired(lease) {
|
|
152
|
+
if (this.leaseMaxAgeMs === undefined)
|
|
153
|
+
return false;
|
|
154
|
+
try {
|
|
155
|
+
return this.now() - (await lstat(lease)).mtimeMs > this.leaseMaxAgeMs;
|
|
156
|
+
}
|
|
157
|
+
catch (error) {
|
|
158
|
+
if (error.code === 'ENOENT')
|
|
159
|
+
return false;
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
140
163
|
async reap(slot) {
|
|
141
164
|
try {
|
|
142
165
|
const metadata = await lstat(slot);
|
|
@@ -150,7 +173,9 @@ export class SharedRequestCapacity {
|
|
|
150
173
|
else {
|
|
151
174
|
for (const name of owners) {
|
|
152
175
|
const match = /^([1-9][0-9]*)-[0-9a-f-]{36}\.lease$/u.exec(name);
|
|
153
|
-
if (match === null
|
|
176
|
+
if (match === null)
|
|
177
|
+
return;
|
|
178
|
+
if (processAlive(Number(match[1])) && !(await this.leaseExpired(path.join(slot, name))))
|
|
154
179
|
return;
|
|
155
180
|
// Unique owner names prevent a concurrent reaper from deleting a new lease.
|
|
156
181
|
await unlink(path.join(slot, name)).catch((error) => {
|
|
@@ -3,7 +3,7 @@ import { sha256Bytes } from '../canonical.js';
|
|
|
3
3
|
export function sourceDocumentCacheKey(bytes, sourcePath) {
|
|
4
4
|
return `${sha256Bytes(bytes)}:${sourcePath}`;
|
|
5
5
|
}
|
|
6
|
-
function decodedDocument(encoded
|
|
6
|
+
function decodedDocument(encoded) {
|
|
7
7
|
const value = deserialize(encoded);
|
|
8
8
|
if (value === null ||
|
|
9
9
|
typeof value !== 'object' ||
|
|
@@ -20,11 +20,6 @@ function decodedDocument(encoded, expected) {
|
|
|
20
20
|
!('bytes' in value) ||
|
|
21
21
|
!(value.bytes instanceof Uint8Array))
|
|
22
22
|
throw new Error('Persistent source document is structurally invalid');
|
|
23
|
-
if (expected !== undefined &&
|
|
24
|
-
(value.path !== expected.sourcePath ||
|
|
25
|
-
value.sourceRevision !== sha256Bytes(expected.bytes) ||
|
|
26
|
-
!Buffer.from(value.bytes).equals(Buffer.from(expected.bytes))))
|
|
27
|
-
throw new Error('Persistent source document does not match its source identity');
|
|
28
23
|
return value;
|
|
29
24
|
}
|
|
30
25
|
/** Puts between halvings of every entry's hit count, so entries no longer read become evictable. */
|
|
@@ -64,13 +59,23 @@ export class SourceDocumentCache {
|
|
|
64
59
|
this.#misses += 1;
|
|
65
60
|
return undefined;
|
|
66
61
|
}
|
|
62
|
+
let document;
|
|
63
|
+
try {
|
|
64
|
+
document = decodedDocument(entry.encoded);
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
// An imported entry is decoded here, on first use; one that cannot be decoded is a miss.
|
|
68
|
+
this.#remove(key);
|
|
69
|
+
this.#misses += 1;
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
67
72
|
this.#hits += 1;
|
|
68
73
|
if (entry.hits === 0)
|
|
69
74
|
this.#coldBytes -= entrySize(key, entry.encoded);
|
|
70
75
|
entry.hits += 1;
|
|
71
76
|
this.#entries.delete(key);
|
|
72
77
|
this.#entries.set(key, entry);
|
|
73
|
-
return
|
|
78
|
+
return document;
|
|
74
79
|
}
|
|
75
80
|
/** Availability check without deserialization or cache-hit accounting. */
|
|
76
81
|
has(key) {
|
|
@@ -96,9 +101,14 @@ export class SourceDocumentCache {
|
|
|
96
101
|
}
|
|
97
102
|
this.retain(key, serialize(document));
|
|
98
103
|
}
|
|
99
|
-
/**
|
|
100
|
-
|
|
101
|
-
|
|
104
|
+
/**
|
|
105
|
+
* Import one authenticated same-runtime entry without counting it as a process-local hit.
|
|
106
|
+
* The caller has verified the entry against the key's source bytes (the key is their hash and
|
|
107
|
+
* the path), so the document is decoded when it is first read instead of for every file at
|
|
108
|
+
* load, which cost as much as parsing the files again; `get` treats an undecodable entry as
|
|
109
|
+
* a miss.
|
|
110
|
+
*/
|
|
111
|
+
importEncoded(key, encoded) {
|
|
102
112
|
this.retain(key, Buffer.from(encoded));
|
|
103
113
|
}
|
|
104
114
|
exportEncoded(key) {
|
|
@@ -1,4 +1,12 @@
|
|
|
1
1
|
import { SOURCE_LINE_LIMIT, SOURCE_TOKEN_LIMIT, SourceDiagnosticCollector } from './limits.js';
|
|
2
|
+
const unicodeSpace = /\s/u;
|
|
3
|
+
/** The same characters as the `\s` class; the regular expression only runs for non-ASCII text. */
|
|
4
|
+
function isSpace(character) {
|
|
5
|
+
const code = character.charCodeAt(0);
|
|
6
|
+
if (code < 128)
|
|
7
|
+
return code === 32 || (code >= 9 && code <= 13);
|
|
8
|
+
return code > 159 && unicodeSpace.test(character);
|
|
9
|
+
}
|
|
2
10
|
const operatorStarts = new Set(['=', '<', '>', '!']);
|
|
3
11
|
function startsOperator(text, index) {
|
|
4
12
|
const character = text[index] ?? '';
|
|
@@ -141,8 +149,8 @@ export function lexClausewitz(text, sourcePath, options = {}) {
|
|
|
141
149
|
const startLine = line;
|
|
142
150
|
const startColumn = column;
|
|
143
151
|
const current = text[index] ?? '';
|
|
144
|
-
if (
|
|
145
|
-
while (index < text.length &&
|
|
152
|
+
if (isSpace(current)) {
|
|
153
|
+
while (index < text.length && isSpace(text[index] ?? ''))
|
|
146
154
|
advance();
|
|
147
155
|
add('whitespace', start, startLine, startColumn);
|
|
148
156
|
continue;
|
|
@@ -236,7 +244,7 @@ export function lexClausewitz(text, sourcePath, options = {}) {
|
|
|
236
244
|
}
|
|
237
245
|
while (index < text.length) {
|
|
238
246
|
const character = text[index] ?? '';
|
|
239
|
-
if (
|
|
247
|
+
if (isSpace(character) ||
|
|
240
248
|
character === '#' ||
|
|
241
249
|
character === '{' ||
|
|
242
250
|
character === '}' ||
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { lstatSync, readFileSync, realpathSync, rmSync } from 'node:fs';
|
|
3
|
+
import { lstat, open, readFile, readdir, realpath, rm } from 'node:fs/promises';
|
|
4
|
+
import { hostname } from 'node:os';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { z } from 'zod/v4';
|
|
7
|
+
const OWNER_FILE = '.hoi4-temp-owner.json';
|
|
8
|
+
const SWEEP_LOCK = '.hoi4-temp-sweep.lock';
|
|
9
|
+
export const TEMPORARY_MAX_BYTES = 256 * 1024 * 1024;
|
|
10
|
+
export const TEMPORARY_STALE_MS = 3 * 60 * 60 * 1000;
|
|
11
|
+
// Dated event evaluation directories contain retained benchmark evidence, not scratch files.
|
|
12
|
+
const scratchName = /^hoi4-(?:reference-source|event-eval)-[A-Za-z0-9]{6}$/u;
|
|
13
|
+
const ownerSchema = z
|
|
14
|
+
.object({
|
|
15
|
+
version: z.literal(1),
|
|
16
|
+
pid: z.number().int().positive(),
|
|
17
|
+
host: z.string().min(1).max(256),
|
|
18
|
+
token: z.uuid(),
|
|
19
|
+
})
|
|
20
|
+
.strict();
|
|
21
|
+
function alive(pid) {
|
|
22
|
+
try {
|
|
23
|
+
process.kill(pid, 0);
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
return error.code !== 'ESRCH';
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
async function owner(directory, name = OWNER_FILE) {
|
|
31
|
+
const file = path.join(directory, name);
|
|
32
|
+
try {
|
|
33
|
+
const metadata = await lstat(file);
|
|
34
|
+
if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > 4096)
|
|
35
|
+
return undefined;
|
|
36
|
+
const result = ownerSchema.safeParse(JSON.parse(await readFile(file, 'utf8')));
|
|
37
|
+
return result.success ? result.data : undefined;
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
return error.code === 'ENOENT' ? null : undefined;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
async function inventory(directory) {
|
|
44
|
+
const result = { bytes: 0, modified: 0, locked: false };
|
|
45
|
+
const pending = [directory];
|
|
46
|
+
let entries = 0;
|
|
47
|
+
while (pending.length > 0) {
|
|
48
|
+
const current = pending.pop();
|
|
49
|
+
const metadata = await lstat(current);
|
|
50
|
+
if (++entries > 100_000)
|
|
51
|
+
throw new Error('Temporary directory inventory limit exceeded');
|
|
52
|
+
result.modified = Math.max(result.modified, metadata.mtimeMs, metadata.birthtimeMs);
|
|
53
|
+
if (metadata.isSymbolicLink())
|
|
54
|
+
continue;
|
|
55
|
+
if (metadata.isFile())
|
|
56
|
+
result.bytes += metadata.size;
|
|
57
|
+
else if (metadata.isDirectory()) {
|
|
58
|
+
for (const name of await readdir(current)) {
|
|
59
|
+
if (name === OWNER_FILE)
|
|
60
|
+
continue;
|
|
61
|
+
if (/\.(?:lock|lease)$/u.test(name))
|
|
62
|
+
result.locked = true;
|
|
63
|
+
pending.push(path.join(current, name));
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return result;
|
|
68
|
+
}
|
|
69
|
+
function overlaps(left, right) {
|
|
70
|
+
const normalize = (value) => process.platform === 'win32' ? path.resolve(value).toLowerCase() : path.resolve(value);
|
|
71
|
+
const a = normalize(left);
|
|
72
|
+
const b = normalize(right);
|
|
73
|
+
const contains = (parent, child) => {
|
|
74
|
+
const relative = path.relative(parent, child);
|
|
75
|
+
return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
|
|
76
|
+
};
|
|
77
|
+
return contains(a, b) || contains(b, a);
|
|
78
|
+
}
|
|
79
|
+
/** Trusted startup maintenance only; never follows a junction or accepts an MCP path. */
|
|
80
|
+
export async function sweepTemporaryDirectories(parent, options = {}) {
|
|
81
|
+
const now = options.now ?? Date.now();
|
|
82
|
+
const staleMs = options.staleMs ?? TEMPORARY_STALE_MS;
|
|
83
|
+
const maxBytes = options.maxBytes ?? TEMPORARY_MAX_BYTES;
|
|
84
|
+
const currentUid = options.currentUid ?? (process.platform === 'win32' ? undefined : process.geteuid?.());
|
|
85
|
+
if (!Number.isFinite(now) ||
|
|
86
|
+
!Number.isSafeInteger(staleMs) ||
|
|
87
|
+
staleMs < 0 ||
|
|
88
|
+
!Number.isSafeInteger(maxBytes) ||
|
|
89
|
+
maxBytes < 0 ||
|
|
90
|
+
(currentUid !== undefined && (!Number.isSafeInteger(currentUid) || currentUid < 0)))
|
|
91
|
+
throw new RangeError('Invalid temporary storage limits');
|
|
92
|
+
const root = await realpath(parent);
|
|
93
|
+
// A Windows runner reaches %TEMP% through an 8.3 alias; match protected roots by both spellings.
|
|
94
|
+
const protectedRoots = (await Promise.all((options.protectedRoots ?? []).map(async (protectedRoot) => {
|
|
95
|
+
try {
|
|
96
|
+
return [protectedRoot, await realpath(protectedRoot)];
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return [protectedRoot];
|
|
100
|
+
}
|
|
101
|
+
}))).flat();
|
|
102
|
+
const candidates = [];
|
|
103
|
+
for (const name of (await readdir(root)).filter((name) => scratchName.test(name)).sort()) {
|
|
104
|
+
const directory = path.join(root, name);
|
|
105
|
+
if (protectedRoots.some((protectedRoot) => overlaps(directory, protectedRoot)))
|
|
106
|
+
continue;
|
|
107
|
+
try {
|
|
108
|
+
const metadata = await lstat(directory);
|
|
109
|
+
if (!metadata.isDirectory() ||
|
|
110
|
+
metadata.isSymbolicLink() ||
|
|
111
|
+
(currentUid !== undefined && metadata.uid !== currentUid) ||
|
|
112
|
+
(await realpath(directory)) !== directory)
|
|
113
|
+
continue;
|
|
114
|
+
const record = await owner(directory);
|
|
115
|
+
// An interrupted sweep must not leave a permanent lock. Malformed or live locks
|
|
116
|
+
// stay protected; only a confirmed dead local owner can relinquish its lock.
|
|
117
|
+
const previousSweep = await owner(directory, SWEEP_LOCK);
|
|
118
|
+
const deadSweep = previousSweep?.host === hostname() &&
|
|
119
|
+
!(options.isAlive ?? alive)(previousSweep.pid) &&
|
|
120
|
+
(await owner(directory, SWEEP_LOCK))?.token === previousSweep.token;
|
|
121
|
+
if (deadSweep)
|
|
122
|
+
await rm(path.join(directory, SWEEP_LOCK));
|
|
123
|
+
const data = await inventory(directory);
|
|
124
|
+
const dead = record?.host === hostname() && !(options.isAlive ?? alive)(record.pid);
|
|
125
|
+
candidates.push({
|
|
126
|
+
directory,
|
|
127
|
+
...data,
|
|
128
|
+
eligible: !data.locked && (dead || deadSweep || (record === null && now - data.modified > staleMs)),
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
// Sharing violations, malformed leases and incomplete inventories fail closed.
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
let retainedBytes = candidates.reduce((sum, entry) => sum + entry.bytes, 0);
|
|
136
|
+
let removed = 0;
|
|
137
|
+
candidates.sort((a, b) => a.modified - b.modified || (a.directory < b.directory ? -1 : 1));
|
|
138
|
+
for (const entry of candidates) {
|
|
139
|
+
if (!entry.eligible || (now - entry.modified <= staleMs && retainedBytes <= maxBytes))
|
|
140
|
+
continue;
|
|
141
|
+
const lockPath = path.join(entry.directory, SWEEP_LOCK);
|
|
142
|
+
let lock;
|
|
143
|
+
let claimed = false;
|
|
144
|
+
const token = randomUUID();
|
|
145
|
+
try {
|
|
146
|
+
lock = await open(lockPath, 'wx', 0o600);
|
|
147
|
+
claimed = true;
|
|
148
|
+
await lock.writeFile(JSON.stringify({ version: 1, pid: process.pid, host: hostname(), token }));
|
|
149
|
+
const record = await owner(entry.directory);
|
|
150
|
+
if (record === undefined ||
|
|
151
|
+
(record !== null && (record.host !== hostname() || (options.isAlive ?? alive)(record.pid))))
|
|
152
|
+
continue;
|
|
153
|
+
// Check containment again immediately before recursive removal.
|
|
154
|
+
if ((await lstat(entry.directory)).isSymbolicLink() ||
|
|
155
|
+
(await realpath(entry.directory)) !== entry.directory)
|
|
156
|
+
continue;
|
|
157
|
+
if ((await owner(entry.directory, SWEEP_LOCK))?.token !== token)
|
|
158
|
+
continue;
|
|
159
|
+
await lock.close();
|
|
160
|
+
lock = undefined;
|
|
161
|
+
await rm(entry.directory, { recursive: true, force: true });
|
|
162
|
+
retainedBytes -= entry.bytes;
|
|
163
|
+
removed += 1;
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
// Windows may still hold a file open. Leave it for a later sweep.
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
await lock?.close().catch(() => undefined);
|
|
170
|
+
if (claimed && (await owner(entry.directory, SWEEP_LOCK))?.token === token)
|
|
171
|
+
await rm(lockPath, { force: true }).catch(() => undefined);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return { removed, retainedBytes };
|
|
175
|
+
}
|
|
176
|
+
/** Own only directories allocated by this process; exit cleanup never uses a prefix sweep. */
|
|
177
|
+
export class TemporaryDirectoryLifetime {
|
|
178
|
+
maxBytes;
|
|
179
|
+
#owned = new Map();
|
|
180
|
+
#reservedFiles = new Map();
|
|
181
|
+
#reservedBytes = 0;
|
|
182
|
+
#onExit = () => this.disposeSync();
|
|
183
|
+
constructor(maxBytes = TEMPORARY_MAX_BYTES) {
|
|
184
|
+
this.maxBytes = maxBytes;
|
|
185
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < 0)
|
|
186
|
+
throw new RangeError('Invalid temporary storage budget');
|
|
187
|
+
process.once('exit', this.#onExit);
|
|
188
|
+
}
|
|
189
|
+
/** Reject an oversized fixture before truncate allocates its disk blocks on Windows. */
|
|
190
|
+
assertFileSize(file, bytes) {
|
|
191
|
+
const requested = path.resolve(file);
|
|
192
|
+
// Fixture paths are canonical, so resolve an aliased parent before matching them.
|
|
193
|
+
let absolute = requested;
|
|
194
|
+
try {
|
|
195
|
+
absolute = path.join(realpathSync.native(path.dirname(requested)), path.basename(requested));
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
// A file in a directory that does not exist yet cannot be inside an owned fixture.
|
|
199
|
+
}
|
|
200
|
+
if (!this.paths().some((directory) => absolute.startsWith(`${directory}${path.sep}`)))
|
|
201
|
+
return;
|
|
202
|
+
if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.maxBytes)
|
|
203
|
+
throw new RangeError('Temporary fixture exceeds the disk budget; use a small injected limit');
|
|
204
|
+
const previous = this.#reservedFiles.get(absolute) ?? 0;
|
|
205
|
+
if (this.#reservedBytes - previous + bytes > this.maxBytes)
|
|
206
|
+
throw new RangeError('Temporary fixtures exceed the disk budget');
|
|
207
|
+
// Reserve before the caller writes, so concurrent allocations cannot oversubscribe.
|
|
208
|
+
// Failed writes remain reserved until teardown, which fails safely rather than undercounts.
|
|
209
|
+
this.#reservedBytes += bytes - previous;
|
|
210
|
+
this.#reservedFiles.set(absolute, bytes);
|
|
211
|
+
}
|
|
212
|
+
async own(directory) {
|
|
213
|
+
const canonical = await realpath(directory);
|
|
214
|
+
// The parent may be reached through an alias, such as a Windows 8.3 short name for the user
|
|
215
|
+
// profile; the fixture directory itself must be a real directory, not a link.
|
|
216
|
+
const resolved = path.resolve(directory);
|
|
217
|
+
const viaParent = path.join(await realpath(path.dirname(resolved)), path.basename(resolved));
|
|
218
|
+
if ((await lstat(directory)).isSymbolicLink() || canonical !== viaParent)
|
|
219
|
+
throw new Error('Temporary directory must be canonical and unlinked');
|
|
220
|
+
const sweepLock = path.join(canonical, SWEEP_LOCK);
|
|
221
|
+
if (await lstat(sweepLock).catch(() => undefined))
|
|
222
|
+
throw new Error('Temporary directory is being swept');
|
|
223
|
+
const token = randomUUID();
|
|
224
|
+
const handle = await open(path.join(canonical, OWNER_FILE), 'wx', 0o600);
|
|
225
|
+
try {
|
|
226
|
+
await handle.writeFile(JSON.stringify({ version: 1, pid: process.pid, host: hostname(), token }));
|
|
227
|
+
this.#owned.set(canonical, token);
|
|
228
|
+
if (await lstat(sweepLock).catch(() => undefined))
|
|
229
|
+
throw new Error('Temporary directory is being swept');
|
|
230
|
+
}
|
|
231
|
+
finally {
|
|
232
|
+
await handle.close();
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
paths() {
|
|
236
|
+
return [...this.#owned.keys()];
|
|
237
|
+
}
|
|
238
|
+
async release(requested) {
|
|
239
|
+
let directory = requested;
|
|
240
|
+
if (!this.#owned.has(directory)) {
|
|
241
|
+
// Owned paths are canonical; a caller may still hold an 8.3 or symlinked spelling.
|
|
242
|
+
try {
|
|
243
|
+
directory = await realpath(requested);
|
|
244
|
+
}
|
|
245
|
+
catch {
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
const token = this.#owned.get(directory);
|
|
250
|
+
if (token === undefined)
|
|
251
|
+
return;
|
|
252
|
+
if ((await owner(directory))?.token !== token) {
|
|
253
|
+
this.#owned.delete(directory);
|
|
254
|
+
this.#forgetFiles(directory);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
if ((await lstat(directory)).isSymbolicLink() || (await realpath(directory)) !== directory)
|
|
258
|
+
throw new Error('Temporary directory changed before cleanup');
|
|
259
|
+
await rm(directory, { recursive: true, force: true });
|
|
260
|
+
this.#owned.delete(directory);
|
|
261
|
+
this.#forgetFiles(directory);
|
|
262
|
+
}
|
|
263
|
+
#forgetFiles(directory) {
|
|
264
|
+
for (const [file, bytes] of this.#reservedFiles)
|
|
265
|
+
if (file.startsWith(`${directory}${path.sep}`)) {
|
|
266
|
+
this.#reservedBytes -= bytes;
|
|
267
|
+
this.#reservedFiles.delete(file);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
async dispose() {
|
|
271
|
+
for (const directory of this.paths())
|
|
272
|
+
await this.release(directory);
|
|
273
|
+
process.removeListener('exit', this.#onExit);
|
|
274
|
+
}
|
|
275
|
+
disposeSync() {
|
|
276
|
+
for (const [directory, token] of this.#owned) {
|
|
277
|
+
try {
|
|
278
|
+
if (lstatSync(directory).isSymbolicLink() || realpathSync.native(directory) !== directory)
|
|
279
|
+
continue;
|
|
280
|
+
const parsed = ownerSchema.safeParse(JSON.parse(readFileSync(path.join(directory, OWNER_FILE), 'utf8')));
|
|
281
|
+
if (parsed.success && parsed.data.token === token)
|
|
282
|
+
rmSync(directory, { recursive: true, force: true });
|
|
283
|
+
}
|
|
284
|
+
catch {
|
|
285
|
+
// A crash or sharing violation is recovered by the next stale sweep.
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
this.#owned.clear();
|
|
289
|
+
this.#reservedFiles.clear();
|
|
290
|
+
this.#reservedBytes = 0;
|
|
291
|
+
process.removeListener('exit', this.#onExit);
|
|
292
|
+
}
|
|
293
|
+
}
|