hoi4-agent-tools 3.6.0 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +124 -28
- package/dist/hoi4_agent_tools/core/configuration.js +2 -0
- package/dist/hoi4_agent_tools/core/engine.js +7 -7
- package/dist/hoi4_agent_tools/core/file-glob.js +164 -0
- package/dist/hoi4_agent_tools/core/index-segments.js +4 -0
- package/dist/hoi4_agent_tools/core/job-operation-registration.js +31 -0
- package/dist/hoi4_agent_tools/core/job-store.js +19 -40
- package/dist/hoi4_agent_tools/core/job-worker-host.js +84 -19
- package/dist/hoi4_agent_tools/core/job-worker.js +8 -16
- package/dist/hoi4_agent_tools/core/operation-tasks.js +2 -7
- package/dist/hoi4_agent_tools/core/persistent-analysis-cache.js +37 -6
- package/dist/hoi4_agent_tools/core/scanner.js +136 -47
- package/dist/hoi4_agent_tools/core/shared-request-capacity.js +27 -13
- package/dist/hoi4_agent_tools/core/source/cache.js +4 -0
- package/dist/hoi4_agent_tools/core/windows-sharing.js +42 -0
- package/dist/hoi4_agent_tools/core/workspace.js +114 -8
- package/dist/hoi4_agent_tools/event/artifact-validation.js +1 -1
- package/dist/hoi4_agent_tools/event/service.js +1 -1
- package/dist/hoi4_agent_tools/focus/service.js +2 -2
- package/dist/hoi4_agent_tools/gui/studio.js +16 -3
- package/dist/hoi4_agent_tools/map/model.js +81 -2
- package/dist/hoi4_agent_tools/map/service.js +3 -2
- package/dist/hoi4_agent_tools/map/validation.js +60 -4
- package/dist/hoi4_agent_tools/mcp/tools/reference.js +10 -1
- package/dist/hoi4_agent_tools/package-check/service.js +1 -1
- package/dist/hoi4_agent_tools/reference/lines.js +12 -0
- package/dist/hoi4_agent_tools/reference/operations.js +45 -2
- package/dist/hoi4_agent_tools/reference/ranking.js +309 -0
- package/dist/hoi4_agent_tools/reference/script-validation.js +267 -0
- package/dist/hoi4_agent_tools/reference/service.js +285 -177
- package/dist/hoi4_agent_tools/reference/source-locate.js +156 -0
- package/dist/hoi4_agent_tools/reference/source-lookup.js +125 -18
- package/dist/hoi4_agent_tools/reference/source-navigation.js +130 -0
- package/dist/hoi4_agent_tools/reference/suggestions.js +38 -0
- package/dist/hoi4_agent_tools/schemas/reference.js +135 -2
- package/dist/hoi4_agent_tools/schemas/script-validation.js +69 -0
- package/dist/hoi4_agent_tools/schemas/transaction.js +1 -1
- package/dist/hoi4_agent_tools/technology/service.js +1 -1
- package/dist/hoi4_agent_tools/version.js +1 -1
- package/docs/README.md +1 -0
- package/docs/development.md +6 -0
- package/docs/examples.md +3 -13
- package/docs/gui.md +1 -15
- package/docs/map.md +10 -0
- package/docs/reference.md +47 -4
- package/docs/research/reference-benchmark.md +80 -0
- package/docs/setup.md +15 -0
- package/docs/tools.md +37 -36
- package/npm-shrinkwrap.json +7101 -0
- package/package.json +8 -7
- package/schemas/configuration.schema.json +95 -88
- package/schemas/continuous-focus-palette.schema.json +1 -1
- package/schemas/custom-weighted-pool.schema.json +1 -1
- package/schemas/decision-inspect-input.schema.json +1 -1
- package/schemas/event-feature-manifest.schema.json +1 -1
- package/schemas/focus-plan.schema.json +1 -1
- package/schemas/focus-planning-sidecar.schema.json +1 -1
- package/schemas/gui-animation-source.schema.json +1 -1
- package/schemas/gui-helper.schema.json +1 -1
- package/schemas/gui-scenario.schema.json +1 -1
- package/schemas/helper-expansion-request.schema.json +1 -1
- package/schemas/helper-expansion-summary.schema.json +1 -1
- package/schemas/impact-inspect-input.schema.json +1 -1
- package/schemas/map-operation.schema.json +1 -1
- package/schemas/mechanic-test-input.schema.json +1 -1
- package/schemas/operation-result.schema.json +1 -1
- package/schemas/package-check-input.schema.json +1 -1
- package/schemas/package-manifest.schema.json +1 -1
- package/schemas/probability-analysis-result.schema.json +1 -1
- package/schemas/probability-compare-input.schema.json +1 -1
- package/schemas/probability-evaluate-input.schema.json +1 -1
- package/schemas/probability-inspect-input.schema.json +1 -1
- package/schemas/probability-render-input.schema.json +1 -1
- package/schemas/probability-scenario-set.schema.json +1 -1
- package/schemas/probability-sequence-input.schema.json +1 -1
- package/schemas/probability-simulate-input.schema.json +1 -1
- package/schemas/probability-sweep-input.schema.json +1 -1
- package/schemas/reference-context-input.schema.json +1 -1
- package/schemas/reference-read-input.schema.json +9 -1
- package/schemas/reference-search-input.schema.json +1 -1
- package/schemas/scenario-suite.schema.json +1 -1
- package/schemas/scenario-test-input.schema.json +1 -1
- package/schemas/script-validate-input.schema.json +92 -0
- package/schemas/shared-scenario.schema.json +1 -1
- package/schemas/source-lookup-input.schema.json +144 -15
- package/server.json +2 -2
|
@@ -10,6 +10,53 @@ import { containedGeneratedPath } from './workspace.js';
|
|
|
10
10
|
import { ServiceError } from './result.js';
|
|
11
11
|
const maximumCheckpointRecoveries = 2;
|
|
12
12
|
const maximumPreDispatchRecoveries = 2;
|
|
13
|
+
// Native and IPC codes are fixed vocabulary. Error messages can contain paths or source
|
|
14
|
+
// text, so only an allowlisted code and the fixed coordination stage are reported.
|
|
15
|
+
const nativeFailureCodes = new Set([
|
|
16
|
+
'EACCES',
|
|
17
|
+
'EAGAIN',
|
|
18
|
+
'EBADF',
|
|
19
|
+
'EBUSY',
|
|
20
|
+
'ECHILD',
|
|
21
|
+
'ECONNRESET',
|
|
22
|
+
'EEXIST',
|
|
23
|
+
'EIO',
|
|
24
|
+
'EMFILE',
|
|
25
|
+
'ENFILE',
|
|
26
|
+
'ENOENT',
|
|
27
|
+
'ENOMEM',
|
|
28
|
+
'ENOSPC',
|
|
29
|
+
'ENOTDIR',
|
|
30
|
+
'ENOTEMPTY',
|
|
31
|
+
'EPERM',
|
|
32
|
+
'EPIPE',
|
|
33
|
+
'ESRCH',
|
|
34
|
+
'ETIMEDOUT',
|
|
35
|
+
'ERR_IPC_CHANNEL_CLOSED',
|
|
36
|
+
'ERR_IPC_DISCONNECTED',
|
|
37
|
+
]);
|
|
38
|
+
function nativeFailureCode(error) {
|
|
39
|
+
if (typeof error !== 'object' || error === null || !('code' in error))
|
|
40
|
+
return undefined;
|
|
41
|
+
return typeof error.code === 'string' && nativeFailureCodes.has(error.code)
|
|
42
|
+
? error.code
|
|
43
|
+
: undefined;
|
|
44
|
+
}
|
|
45
|
+
/** Map a launcher failure to a stable, path-free job failure. */
|
|
46
|
+
export function workerFailure(error, stage) {
|
|
47
|
+
if (error instanceof ServiceError)
|
|
48
|
+
return { code: error.code, message: error.message };
|
|
49
|
+
const nativeCode = nativeFailureCode(error);
|
|
50
|
+
return nativeCode === undefined
|
|
51
|
+
? {
|
|
52
|
+
code: 'JOB_WORKER_FAILED',
|
|
53
|
+
message: `The isolated worker stopped without publishing a result (stage ${stage})`,
|
|
54
|
+
}
|
|
55
|
+
: {
|
|
56
|
+
code: `JOB_WORKER_${nativeCode}`,
|
|
57
|
+
message: `Worker coordination could not complete (${nativeCode}, stage ${stage}); inspect the retained job outcome`,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
13
60
|
/** Bounded, fixed-entry child execution; client cancellation/disconnection never kills a worker. */
|
|
14
61
|
export class JobWorkerHost {
|
|
15
62
|
engine;
|
|
@@ -69,10 +116,13 @@ export class JobWorkerHost {
|
|
|
69
116
|
throw new ServiceError('JOB_OPERATION_UNAVAILABLE', 'This domain has not yet been registered for worker execution');
|
|
70
117
|
const signal = new AbortController().signal;
|
|
71
118
|
const recovered = new Set();
|
|
119
|
+
const progress = { stage: 'admission' };
|
|
72
120
|
let preDispatchRecoveries = 0;
|
|
73
121
|
for (;;) {
|
|
74
122
|
try {
|
|
75
|
-
|
|
123
|
+
progress.stage = 'admission';
|
|
124
|
+
await this.scheduler.run(this.owner, 1024, signal, () => this.capacity.run(signal, (lease) => this.launch(workspaceId, id, lease, progress, principal)));
|
|
125
|
+
progress.stage = 'supervision';
|
|
76
126
|
const current = await this.jobs.get(workspaceId, id, principal);
|
|
77
127
|
if (['completed', 'cancelled', 'failed'].includes(current.status))
|
|
78
128
|
return current;
|
|
@@ -104,12 +154,7 @@ export class JobWorkerHost {
|
|
|
104
154
|
recovered.add(checkpoint);
|
|
105
155
|
continue;
|
|
106
156
|
}
|
|
107
|
-
return await this.jobs.failInterrupted(workspaceId, id,
|
|
108
|
-
code: error instanceof ServiceError ? error.code : 'JOB_WORKER_FAILED',
|
|
109
|
-
message: error instanceof ServiceError
|
|
110
|
-
? error.message
|
|
111
|
-
: 'The isolated worker stopped without publishing a result',
|
|
112
|
-
}, principal);
|
|
157
|
+
return await this.jobs.failInterrupted(workspaceId, id, workerFailure(error, progress.stage), principal);
|
|
113
158
|
}
|
|
114
159
|
}
|
|
115
160
|
}
|
|
@@ -126,7 +171,7 @@ export class JobWorkerHost {
|
|
|
126
171
|
await delay(100);
|
|
127
172
|
}
|
|
128
173
|
}
|
|
129
|
-
async launch(workspaceId, id, lease, principal) {
|
|
174
|
+
async launch(workspaceId, id, lease, progress, principal) {
|
|
130
175
|
// Resolve the same configured workspace and grants again immediately before dispatch.
|
|
131
176
|
await this.jobs.get(workspaceId, id, principal);
|
|
132
177
|
const sourceMode = import.meta.url.endsWith('.ts');
|
|
@@ -143,15 +188,18 @@ export class JobWorkerHost {
|
|
|
143
188
|
workspaceIds: [...new Set([...grant.workspaceIds, ...discoveredWorkspaceIds])],
|
|
144
189
|
}
|
|
145
190
|
: grant;
|
|
191
|
+
progress.stage = 'startup';
|
|
146
192
|
const child = spawn(process.execPath, [...(sourceMode ? ['--import', import.meta.resolve('tsx')] : []), entry], {
|
|
147
193
|
detached: true,
|
|
148
194
|
cwd: process.cwd(),
|
|
149
195
|
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
|
150
196
|
windowsHide: true,
|
|
151
197
|
});
|
|
198
|
+
let phase = 'waiting_for_ready';
|
|
199
|
+
let acknowledgmentTimer;
|
|
152
200
|
const initialized = new Promise((resolve, reject) => {
|
|
153
201
|
child.once('error', reject);
|
|
154
|
-
child.once('exit', () => reject(new ServiceError('JOB_WORKER_STARTUP_EXIT',
|
|
202
|
+
child.once('exit', (code, signal) => reject(new ServiceError('JOB_WORKER_STARTUP_EXIT', `The worker exited before completing its startup handshake (phase ${phase}, exit ${code ?? 'none'}, signal ${signal ?? 'none'})`)));
|
|
155
203
|
child.once('message', (value) => {
|
|
156
204
|
void (async () => {
|
|
157
205
|
if (typeof value !== 'object' ||
|
|
@@ -160,9 +208,27 @@ export class JobWorkerHost {
|
|
|
160
208
|
value.type !== 'ready' ||
|
|
161
209
|
child.pid === undefined)
|
|
162
210
|
throw new ServiceError('JOB_WORKER_PROTOCOL', 'The worker did not provide its readiness handshake');
|
|
211
|
+
progress.stage = 'handoff';
|
|
163
212
|
await lease.handoffToProcess(child.pid);
|
|
213
|
+
progress.stage = 'dispatch';
|
|
164
214
|
await new Promise((resolve, reject) => {
|
|
165
|
-
|
|
215
|
+
phase = 'waiting_for_acceptance';
|
|
216
|
+
acknowledgmentTimer = setTimeout(() => reject(new ServiceError('JOB_WORKER_ACCEPT_TIMEOUT', 'The worker did not acknowledge receipt within its startup window')), 30_000);
|
|
217
|
+
child.once('message', (accepted) => {
|
|
218
|
+
if (typeof accepted !== 'object' ||
|
|
219
|
+
accepted === null ||
|
|
220
|
+
!('type' in accepted) ||
|
|
221
|
+
accepted.type !== 'accepted') {
|
|
222
|
+
reject(new ServiceError('JOB_WORKER_PROTOCOL', 'The worker did not acknowledge the dispatched job'));
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
phase = 'accepted';
|
|
226
|
+
// Only close IPC after the child confirms receipt. The persistent job,
|
|
227
|
+
// rather than the launcher's lifetime, owns execution after this point.
|
|
228
|
+
if (child.connected)
|
|
229
|
+
child.disconnect();
|
|
230
|
+
resolve();
|
|
231
|
+
});
|
|
166
232
|
child.send({
|
|
167
233
|
configuration: {
|
|
168
234
|
...configuration,
|
|
@@ -178,14 +244,7 @@ export class JobWorkerHost {
|
|
|
178
244
|
jobId: id,
|
|
179
245
|
...(principal === undefined ? {} : { principal }),
|
|
180
246
|
}, (error) => {
|
|
181
|
-
if (error
|
|
182
|
-
// IPC is startup-only. Closing it after dispatch prevents launcher
|
|
183
|
-
// lifetime from becoming an implicit cancellation channel.
|
|
184
|
-
if (child.connected)
|
|
185
|
-
child.disconnect();
|
|
186
|
-
resolve();
|
|
187
|
-
}
|
|
188
|
-
else
|
|
247
|
+
if (error !== null)
|
|
189
248
|
reject(error);
|
|
190
249
|
});
|
|
191
250
|
});
|
|
@@ -199,7 +258,13 @@ export class JobWorkerHost {
|
|
|
199
258
|
});
|
|
200
259
|
});
|
|
201
260
|
});
|
|
202
|
-
|
|
261
|
+
try {
|
|
262
|
+
await initialized;
|
|
263
|
+
}
|
|
264
|
+
finally {
|
|
265
|
+
clearTimeout(acknowledgmentTimer);
|
|
266
|
+
}
|
|
267
|
+
progress.stage = 'supervision';
|
|
203
268
|
for (;;) {
|
|
204
269
|
const record = await this.jobs.get(workspaceId, id, principal);
|
|
205
270
|
if (['completed', 'failed', 'cancelled'].includes(record.status))
|
|
@@ -1,16 +1,10 @@
|
|
|
1
1
|
import { z } from 'zod/v4';
|
|
2
2
|
import { serverConfigurationSchema } from './configuration.js';
|
|
3
3
|
import { CoreEngine } from './engine.js';
|
|
4
|
-
import { JobExecutor
|
|
4
|
+
import { JobExecutor } from './job-executor.js';
|
|
5
5
|
import { JobService } from './job-service.js';
|
|
6
6
|
import { WorkspaceResolver } from './workspace.js';
|
|
7
|
-
import {
|
|
8
|
-
import { registerFocusJobs } from '../focus/job-operations.js';
|
|
9
|
-
import { registerGuiJobs } from '../gui/job-operations.js';
|
|
10
|
-
import { registerMapJobs } from '../map/job-operations.js';
|
|
11
|
-
import { registerProbabilityJobs } from '../probability/job-operations.js';
|
|
12
|
-
import { registerTechnologyJobs } from '../technology/job-operations.js';
|
|
13
|
-
import { registerAnalysisJobs } from './analysis-job-operations.js';
|
|
7
|
+
import { registerWorkerOperations } from './job-operation-registration.js';
|
|
14
8
|
// Fixed internal process entry point. It accepts only a trusted host's structured IPC
|
|
15
9
|
// message, never a script path, module name, callback, command, or transport request.
|
|
16
10
|
const inputSchema = z
|
|
@@ -38,19 +32,17 @@ process.once('message', (value) => {
|
|
|
38
32
|
clearTimeout(startup);
|
|
39
33
|
void (async () => {
|
|
40
34
|
const input = inputSchema.parse(value);
|
|
35
|
+
// The send callback confirms enqueueing, not receipt. Acknowledge before the host
|
|
36
|
+
// disconnects; losing that channel after receipt never cancels durable execution.
|
|
37
|
+
if (process.connected)
|
|
38
|
+
process.send?.({ type: 'accepted' }, () => undefined);
|
|
41
39
|
const resolver = await WorkspaceResolver.create(input.configuration);
|
|
42
40
|
const engine = new CoreEngine(resolver);
|
|
43
41
|
// Recovery is targeted by the job executor. Do not sweep or interfere with
|
|
44
42
|
// unrelated live rewrite journals when a worker starts.
|
|
45
43
|
const jobs = await JobService.create(resolver);
|
|
46
|
-
const
|
|
47
|
-
|
|
48
|
-
registerAnalysisJobs(operations, engine);
|
|
49
|
-
registerTechnologyJobs(operations, engine);
|
|
50
|
-
registerProbabilityJobs(operations, engine);
|
|
51
|
-
registerMapJobs(operations, engine);
|
|
52
|
-
registerGuiJobs(operations, engine);
|
|
53
|
-
registerFocusJobs(operations, engine);
|
|
44
|
+
const record = await jobs.get(input.workspaceId, input.jobId, input.principal);
|
|
45
|
+
const operations = await registerWorkerOperations(engine, record.request.toolName);
|
|
54
46
|
await new JobExecutor(engine, jobs, operations).run(input.workspaceId, input.jobId, input.principal);
|
|
55
47
|
})()
|
|
56
48
|
.catch(() => {
|
|
@@ -9,7 +9,7 @@ import { probabilityCompareRequestSchema, probabilityEvaluateRequestSchema, prob
|
|
|
9
9
|
import { technologyCompareRequestSchema, technologyInspectRequestSchema, technologyRenderRequestSchema, } from '../schemas/technology.js';
|
|
10
10
|
import { currentJobOwner, jobOwnerLiveness, jobRetentionExpired, } from './job-store.js';
|
|
11
11
|
import { JobService } from './job-service.js';
|
|
12
|
-
import { JobWorkerHost } from './job-worker-host.js';
|
|
12
|
+
import { JobWorkerHost, workerFailure } from './job-worker-host.js';
|
|
13
13
|
import { compareCodeUnits, hashCanonical, secureId } from './canonical.js';
|
|
14
14
|
import { ServiceError } from './result.js';
|
|
15
15
|
import { errorResult } from './operation-result.js';
|
|
@@ -378,12 +378,7 @@ export class OperationTaskService {
|
|
|
378
378
|
.then(() => undefined)
|
|
379
379
|
.catch(async (error) => {
|
|
380
380
|
const jobs = await this.jobs();
|
|
381
|
-
await jobs.failInterrupted(record.scope.workspaceId, record.id,
|
|
382
|
-
code: error instanceof ServiceError ? error.code : 'JOB_WORKER_FAILED',
|
|
383
|
-
message: error instanceof ServiceError
|
|
384
|
-
? error.message
|
|
385
|
-
: 'The isolated worker could not be started',
|
|
386
|
-
}, principal);
|
|
381
|
+
await jobs.failInterrupted(record.scope.workspaceId, record.id, workerFailure(error, 'admission'), principal);
|
|
387
382
|
})
|
|
388
383
|
.finally(() => {
|
|
389
384
|
this.launches.delete(key);
|
|
@@ -54,6 +54,8 @@ export class PersistentAnalysisCache {
|
|
|
54
54
|
maxBytes;
|
|
55
55
|
maxEntries;
|
|
56
56
|
maxSingleBytes;
|
|
57
|
+
batchMaxWrites;
|
|
58
|
+
batchMaxPayloadBytes;
|
|
57
59
|
#lock;
|
|
58
60
|
#hits = 0;
|
|
59
61
|
#misses = 0;
|
|
@@ -63,23 +65,30 @@ export class PersistentAnalysisCache {
|
|
|
63
65
|
#evictions = 0;
|
|
64
66
|
#invalidEntries = 0;
|
|
65
67
|
#oversized = 0;
|
|
68
|
+
#limitedBatches = 0;
|
|
66
69
|
#invalidAddresses = new Set();
|
|
67
|
-
constructor(state, root, maxBytes, maxEntries, maxSingleBytes) {
|
|
70
|
+
constructor(state, root, maxBytes, maxEntries, maxSingleBytes, batchMaxWrites, batchMaxPayloadBytes) {
|
|
68
71
|
this.state = state;
|
|
69
72
|
this.root = root;
|
|
70
73
|
this.maxBytes = maxBytes;
|
|
71
74
|
this.maxEntries = maxEntries;
|
|
72
75
|
this.maxSingleBytes = maxSingleBytes;
|
|
76
|
+
this.batchMaxWrites = batchMaxWrites;
|
|
77
|
+
this.batchMaxPayloadBytes = batchMaxPayloadBytes;
|
|
73
78
|
this.#lock = new SharedRequestCapacity(root, 1);
|
|
74
79
|
}
|
|
75
80
|
static async create(state, options = {}) {
|
|
76
81
|
const maxBytes = options.maxBytes ?? 268_435_456;
|
|
77
82
|
const maxEntries = options.maxEntries ?? 50_000;
|
|
78
83
|
const maxSingleBytes = options.maxSingleBytes ?? 16_777_216;
|
|
84
|
+
const batchMaxWrites = options.batchMaxWrites ?? 16;
|
|
85
|
+
const batchMaxPayloadBytes = options.batchMaxPayloadBytes ?? 1_048_576;
|
|
79
86
|
for (const [name, value] of [
|
|
80
87
|
['byte budget', maxBytes],
|
|
81
88
|
['entry budget', maxEntries],
|
|
82
89
|
['single-entry budget', maxSingleBytes],
|
|
90
|
+
['batch write budget', batchMaxWrites],
|
|
91
|
+
['batch payload budget', batchMaxPayloadBytes],
|
|
83
92
|
]) {
|
|
84
93
|
if (!Number.isSafeInteger(value) || value < 0)
|
|
85
94
|
throw new RangeError(`Persistent analysis cache ${name} must be a non-negative integer`);
|
|
@@ -88,7 +97,7 @@ export class PersistentAnalysisCache {
|
|
|
88
97
|
const root = await containedGeneratedPath(state.root, 'analysis-cache');
|
|
89
98
|
await mkdir(root, { recursive: true, mode: 0o700 });
|
|
90
99
|
await assertUnlinked(root);
|
|
91
|
-
return new PersistentAnalysisCache(state, root, maxBytes, maxEntries, maxSingleBytes);
|
|
100
|
+
return new PersistentAnalysisCache(state, root, maxBytes, maxEntries, maxSingleBytes, batchMaxWrites, batchMaxPayloadBytes);
|
|
92
101
|
}
|
|
93
102
|
async hydrate(scope, files, documents, segments, signal) {
|
|
94
103
|
signal?.throwIfAborted();
|
|
@@ -101,7 +110,7 @@ export class PersistentAnalysisCache {
|
|
|
101
110
|
signal?.throwIfAborted();
|
|
102
111
|
const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
|
|
103
112
|
const documentAddress = sourceAddress(key);
|
|
104
|
-
if (sourceEntries.has(documentAddress)) {
|
|
113
|
+
if (!documents.has(key) && sourceEntries.has(documentAddress)) {
|
|
105
114
|
const payload = await this.read(scopeHash, 'source-document', documentAddress, signal);
|
|
106
115
|
if (payload !== undefined) {
|
|
107
116
|
try {
|
|
@@ -118,7 +127,7 @@ export class PersistentAnalysisCache {
|
|
|
118
127
|
}
|
|
119
128
|
for (const provinceDefinition of [false, true]) {
|
|
120
129
|
const address = indexSegmentAddress(file, provinceDefinition);
|
|
121
|
-
if (!indexEntries.has(address))
|
|
130
|
+
if (segments.has(address) || !indexEntries.has(address))
|
|
122
131
|
continue;
|
|
123
132
|
const payload = await this.read(scopeHash, 'index-segment', address, signal);
|
|
124
133
|
if (payload === undefined)
|
|
@@ -140,15 +149,34 @@ export class PersistentAnalysisCache {
|
|
|
140
149
|
this.inventory(scopeHash, 'source-document', signal),
|
|
141
150
|
this.inventory(scopeHash, 'index-segment', signal),
|
|
142
151
|
]);
|
|
152
|
+
const batch = { writes: 0, payloadBytes: 0, limited: false };
|
|
153
|
+
const retain = async (kind, address, payload) => {
|
|
154
|
+
if (batch.writes >= this.batchMaxWrites ||
|
|
155
|
+
payload.length > this.batchMaxPayloadBytes - batch.payloadBytes) {
|
|
156
|
+
batch.limited = true;
|
|
157
|
+
return false;
|
|
158
|
+
}
|
|
159
|
+
const written = await this.write(scopeHash, kind, address, payload, signal);
|
|
160
|
+
if (written) {
|
|
161
|
+
batch.writes++;
|
|
162
|
+
batch.payloadBytes += payload.length;
|
|
163
|
+
}
|
|
164
|
+
return written;
|
|
165
|
+
};
|
|
143
166
|
for (const file of files) {
|
|
144
167
|
signal.throwIfAborted();
|
|
168
|
+
if (batch.writes >= this.batchMaxWrites ||
|
|
169
|
+
batch.payloadBytes >= this.batchMaxPayloadBytes) {
|
|
170
|
+
batch.limited = true;
|
|
171
|
+
break;
|
|
172
|
+
}
|
|
145
173
|
const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
|
|
146
174
|
const documentAddress = sourceAddress(key);
|
|
147
175
|
const document = documents.exportEncoded(key);
|
|
148
176
|
if (document !== undefined &&
|
|
149
177
|
(!sourceEntries.has(documentAddress) ||
|
|
150
178
|
this.#invalidAddresses.has(`${scopeHash}\0source-document\0${documentAddress}`))) {
|
|
151
|
-
if (await
|
|
179
|
+
if (await retain('source-document', documentAddress, document)) {
|
|
152
180
|
sourceEntries.add(documentAddress);
|
|
153
181
|
}
|
|
154
182
|
}
|
|
@@ -159,11 +187,13 @@ export class PersistentAnalysisCache {
|
|
|
159
187
|
(indexEntries.has(address) &&
|
|
160
188
|
!this.#invalidAddresses.has(`${scopeHash}\0index-segment\0${address}`)))
|
|
161
189
|
continue;
|
|
162
|
-
if (await
|
|
190
|
+
if (await retain('index-segment', address, Buffer.from(segment))) {
|
|
163
191
|
indexEntries.add(address);
|
|
164
192
|
}
|
|
165
193
|
}
|
|
166
194
|
}
|
|
195
|
+
if (batch.limited)
|
|
196
|
+
this.#limitedBatches++;
|
|
167
197
|
await this.prune(signal);
|
|
168
198
|
});
|
|
169
199
|
}
|
|
@@ -177,6 +207,7 @@ export class PersistentAnalysisCache {
|
|
|
177
207
|
evictions: this.#evictions,
|
|
178
208
|
invalidEntries: this.#invalidEntries,
|
|
179
209
|
oversized: this.#oversized,
|
|
210
|
+
limitedBatches: this.#limitedBatches,
|
|
180
211
|
};
|
|
181
212
|
}
|
|
182
213
|
async directory(scopeHash, kind) {
|
|
@@ -1,13 +1,59 @@
|
|
|
1
|
-
import { open } from 'node:fs/promises';
|
|
1
|
+
import { open, realpath, stat } from 'node:fs/promises';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
-
import
|
|
3
|
+
import { iterateFiles } from './file-glob.js';
|
|
4
4
|
import { compareCodeUnits, sha256Bytes } from './canonical.js';
|
|
5
5
|
import { DEFAULT_SCAN_MAX_BYTES, DEFAULT_SCAN_MAX_FILE_BYTES, DEFAULT_SCAN_MAX_FILES, } from './configuration.js';
|
|
6
6
|
import { ServiceError } from './result.js';
|
|
7
|
+
import { isWithin } from './workspace.js';
|
|
8
|
+
const SCAN_IO_CONCURRENCY = 8;
|
|
9
|
+
const SCAN_READ_BATCH_BYTES = 16_777_216;
|
|
10
|
+
export async function verifySource(root, source) {
|
|
11
|
+
const resolved = await realpath(source.absolutePath);
|
|
12
|
+
if (!isWithin(root.path, resolved))
|
|
13
|
+
throw new ServiceError('SCAN_ROOT_ESCAPE', 'Source path resolves outside its authorized root', {
|
|
14
|
+
file: source.relativePath,
|
|
15
|
+
});
|
|
16
|
+
const current = await stat(resolved);
|
|
17
|
+
const expected = source.metadata;
|
|
18
|
+
if (current.dev !== expected.dev ||
|
|
19
|
+
current.ino !== expected.ino ||
|
|
20
|
+
current.size !== expected.size ||
|
|
21
|
+
current.mtimeMs !== expected.mtimeMs ||
|
|
22
|
+
current.ctimeMs !== expected.ctimeMs)
|
|
23
|
+
throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source changed while it was being scanned; retry a focused query', { file: source.relativePath });
|
|
24
|
+
}
|
|
25
|
+
export async function prepareSource(root, relativePath, signal) {
|
|
26
|
+
signal?.throwIfAborted();
|
|
27
|
+
const absolutePath = path.join(root.path, relativePath);
|
|
28
|
+
const canonicalPath = await realpath(absolutePath);
|
|
29
|
+
if (!isWithin(root.path, canonicalPath))
|
|
30
|
+
throw new ServiceError('SCAN_ROOT_ESCAPE', 'Source path resolves outside its authorized root', {
|
|
31
|
+
file: relativePath,
|
|
32
|
+
});
|
|
33
|
+
const handle = await open(canonicalPath, 'r');
|
|
34
|
+
try {
|
|
35
|
+
const source = {
|
|
36
|
+
relativePath,
|
|
37
|
+
absolutePath,
|
|
38
|
+
canonicalPath,
|
|
39
|
+
handle,
|
|
40
|
+
metadata: await handle.stat(),
|
|
41
|
+
};
|
|
42
|
+
await verifySource(root, source);
|
|
43
|
+
signal?.throwIfAborted();
|
|
44
|
+
return source;
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
await handle.close();
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
7
51
|
function normalizeRelative(value) {
|
|
8
52
|
return value.replaceAll('\\', '/').replace(/^\.\//u, '');
|
|
9
53
|
}
|
|
10
54
|
function rootLabel(root) {
|
|
55
|
+
if (root.kind === 'dlc' && root.label !== undefined)
|
|
56
|
+
return root.label;
|
|
11
57
|
return root.kind === 'dependency' ? `dependency-${root.loadOrder}` : root.kind;
|
|
12
58
|
}
|
|
13
59
|
function hiddenByReplacePath(workspace, root, relativePath) {
|
|
@@ -86,6 +132,7 @@ export class WorkspaceScanner {
|
|
|
86
132
|
}
|
|
87
133
|
const roots = workspace.roots
|
|
88
134
|
.filter((root) => (root.kind === 'game' ||
|
|
135
|
+
root.kind === 'dlc' ||
|
|
89
136
|
root.kind === 'dependency' ||
|
|
90
137
|
root.kind === 'mod' ||
|
|
91
138
|
root.kind === 'fixture') &&
|
|
@@ -98,14 +145,12 @@ export class WorkspaceScanner {
|
|
|
98
145
|
options.signal?.throwIfAborted();
|
|
99
146
|
// Re-enumerate every root: installing content or changing load order can
|
|
100
147
|
// invalidate a vanilla inventory just as it can a mod inventory.
|
|
101
|
-
const matches =
|
|
148
|
+
const matches = iterateFiles(options.patterns, {
|
|
102
149
|
cwd: root.path,
|
|
103
|
-
onlyFiles: true,
|
|
104
|
-
unique: true,
|
|
105
|
-
dot: false,
|
|
106
|
-
followSymbolicLinks: false,
|
|
107
150
|
ignore: options.ignore ?? ['**/.hoi4-agent/**'],
|
|
151
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
108
152
|
});
|
|
153
|
+
const relativePaths = [];
|
|
109
154
|
for await (const match of matches) {
|
|
110
155
|
options.signal?.throwIfAborted();
|
|
111
156
|
enumeratedFiles += 1;
|
|
@@ -115,55 +160,99 @@ export class WorkspaceScanner {
|
|
|
115
160
|
limit: maxFiles,
|
|
116
161
|
});
|
|
117
162
|
}
|
|
118
|
-
const relativePath = normalizeRelative(
|
|
163
|
+
const relativePath = normalizeRelative(match);
|
|
119
164
|
if (hiddenByReplacePath(workspace, root, relativePath))
|
|
120
165
|
continue;
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
166
|
+
relativePaths.push(relativePath);
|
|
167
|
+
}
|
|
168
|
+
for (let offset = 0; offset < relativePaths.length; offset += SCAN_IO_CONCURRENCY) {
|
|
169
|
+
options.signal?.throwIfAborted();
|
|
170
|
+
const prepared = await Promise.allSettled(relativePaths
|
|
171
|
+
.slice(offset, offset + SCAN_IO_CONCURRENCY)
|
|
172
|
+
.map((relativePath) => prepareSource(root, relativePath, options.signal)));
|
|
173
|
+
const opened = prepared.flatMap((entry) => entry.status === 'fulfilled' ? [entry.value] : []);
|
|
124
174
|
try {
|
|
125
|
-
const
|
|
126
|
-
if (
|
|
127
|
-
|
|
128
|
-
const
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
175
|
+
const failed = prepared.find((entry) => entry.status === 'rejected');
|
|
176
|
+
if (failed?.status === 'rejected')
|
|
177
|
+
throw failed.reason;
|
|
178
|
+
const readable = opened.filter(({ metadata }) => metadata.isFile());
|
|
179
|
+
let reserved = 0;
|
|
180
|
+
for (const source of readable) {
|
|
181
|
+
if (source.metadata.size > maxBytes - totalBytes - reserved ||
|
|
182
|
+
source.metadata.size > this.serverMaxFileBytes)
|
|
183
|
+
throw new ServiceError('SCAN_BYTE_LIMIT', 'Scan exceeds the configured byte limit', {
|
|
184
|
+
file: source.relativePath,
|
|
185
|
+
fileBytes: source.metadata.size,
|
|
186
|
+
bytes: totalBytes + reserved,
|
|
187
|
+
limit: maxBytes,
|
|
188
|
+
perFileLimit: this.serverMaxFileBytes,
|
|
189
|
+
});
|
|
190
|
+
reserved += source.metadata.size;
|
|
191
|
+
}
|
|
192
|
+
let readOffset = 0;
|
|
193
|
+
while (readOffset < readable.length) {
|
|
194
|
+
const group = [];
|
|
195
|
+
let groupBytes = 0;
|
|
196
|
+
while (readOffset < readable.length) {
|
|
197
|
+
const source = readable[readOffset];
|
|
198
|
+
if (group.length > 0 && groupBytes + source.metadata.size > SCAN_READ_BATCH_BYTES)
|
|
199
|
+
break;
|
|
200
|
+
group.push(source);
|
|
201
|
+
groupBytes += source.metadata.size;
|
|
202
|
+
readOffset++;
|
|
203
|
+
}
|
|
204
|
+
const reads = await Promise.allSettled(group.map(async (source) => {
|
|
205
|
+
let bytes;
|
|
206
|
+
try {
|
|
207
|
+
bytes = await readBoundedFile(source.handle, source.metadata.size, options.signal);
|
|
208
|
+
}
|
|
209
|
+
catch (error) {
|
|
210
|
+
if (error instanceof ServiceError && error.code === 'SCAN_BYTE_LIMIT')
|
|
211
|
+
throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source grew while it was being scanned; retry a focused query', { file: source.relativePath });
|
|
212
|
+
throw error;
|
|
213
|
+
}
|
|
214
|
+
if (bytes.length !== source.metadata.size)
|
|
215
|
+
throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source changed while it was being scanned; retry a focused query', { file: source.relativePath });
|
|
216
|
+
await verifySource(root, source);
|
|
217
|
+
options.signal?.throwIfAborted();
|
|
218
|
+
return bytes;
|
|
219
|
+
}));
|
|
220
|
+
const readFailure = reads.find((entry) => entry.status === 'rejected');
|
|
221
|
+
if (readFailure?.status === 'rejected')
|
|
222
|
+
throw readFailure.reason;
|
|
223
|
+
for (const [index, read] of reads.entries()) {
|
|
224
|
+
if (read.status !== 'fulfilled')
|
|
225
|
+
continue;
|
|
226
|
+
const source = group[index];
|
|
227
|
+
const cacheKey = this.cacheKey(source.absolutePath);
|
|
228
|
+
const observed = read.value;
|
|
229
|
+
const sha256 = sha256Bytes(observed);
|
|
230
|
+
const cached = this.#sourceCache.get(cacheKey);
|
|
231
|
+
const bytes = cached?.sha256 === sha256 && cached.bytes.equals(observed)
|
|
232
|
+
? cached.bytes
|
|
233
|
+
: observed;
|
|
234
|
+
this.retainSource(cacheKey, { sha256, bytes });
|
|
235
|
+
totalBytes += bytes.length;
|
|
236
|
+
result.push({
|
|
237
|
+
absolutePath: source.absolutePath,
|
|
238
|
+
displayPath: `${rootLabel(root)}:${source.relativePath}`,
|
|
239
|
+
relativePath: source.relativePath,
|
|
240
|
+
rootKind: root.kind,
|
|
241
|
+
loadOrder: root.loadOrder,
|
|
242
|
+
size: bytes.length,
|
|
243
|
+
modifiedMs: source.metadata.mtimeMs,
|
|
244
|
+
sha256,
|
|
245
|
+
bytes,
|
|
246
|
+
});
|
|
247
|
+
}
|
|
137
248
|
}
|
|
138
|
-
const cached = this.#sourceCache.get(cacheKey);
|
|
139
|
-
// Metadata is not content identity (including on aliased/networked
|
|
140
|
-
// filesystems). Verify bytes before reusing parsed/indexed facts.
|
|
141
|
-
const observed = await readBoundedFile(handle, Math.min(remaining, this.serverMaxFileBytes), options.signal);
|
|
142
|
-
const sha256 = sha256Bytes(observed);
|
|
143
|
-
const bytes = cached?.sha256 === sha256 && cached.bytes.equals(observed) ? cached.bytes : observed;
|
|
144
|
-
this.retainSource(cacheKey, {
|
|
145
|
-
sha256,
|
|
146
|
-
bytes,
|
|
147
|
-
});
|
|
148
|
-
totalBytes += bytes.length;
|
|
149
|
-
const scanned = {
|
|
150
|
-
absolutePath,
|
|
151
|
-
displayPath: `${rootLabel(root)}:${relativePath}`,
|
|
152
|
-
relativePath,
|
|
153
|
-
rootKind: root.kind,
|
|
154
|
-
loadOrder: root.loadOrder,
|
|
155
|
-
size: bytes.length,
|
|
156
|
-
modifiedMs: metadata.mtimeMs,
|
|
157
|
-
sha256,
|
|
158
|
-
bytes,
|
|
159
|
-
};
|
|
160
|
-
result.push(scanned);
|
|
161
249
|
}
|
|
162
250
|
finally {
|
|
163
|
-
await handle.close();
|
|
251
|
+
await Promise.all(opened.map(({ handle }) => handle.close()));
|
|
164
252
|
}
|
|
165
253
|
}
|
|
166
254
|
}
|
|
255
|
+
options.signal?.throwIfAborted();
|
|
167
256
|
result.sort((left, right) => left.loadOrder - right.loadOrder || compareCodeUnits(left.relativePath, right.relativePath));
|
|
168
257
|
const groups = new Map();
|
|
169
258
|
for (const file of result) {
|