hoi4-agent-tools 3.6.0 → 3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/README.md +124 -28
  3. package/dist/hoi4_agent_tools/core/configuration.js +2 -0
  4. package/dist/hoi4_agent_tools/core/engine.js +7 -7
  5. package/dist/hoi4_agent_tools/core/file-glob.js +164 -0
  6. package/dist/hoi4_agent_tools/core/index-segments.js +4 -0
  7. package/dist/hoi4_agent_tools/core/job-operation-registration.js +31 -0
  8. package/dist/hoi4_agent_tools/core/job-store.js +19 -40
  9. package/dist/hoi4_agent_tools/core/job-worker-host.js +84 -19
  10. package/dist/hoi4_agent_tools/core/job-worker.js +8 -16
  11. package/dist/hoi4_agent_tools/core/operation-tasks.js +2 -7
  12. package/dist/hoi4_agent_tools/core/persistent-analysis-cache.js +37 -6
  13. package/dist/hoi4_agent_tools/core/scanner.js +136 -47
  14. package/dist/hoi4_agent_tools/core/shared-request-capacity.js +27 -13
  15. package/dist/hoi4_agent_tools/core/source/cache.js +4 -0
  16. package/dist/hoi4_agent_tools/core/windows-sharing.js +42 -0
  17. package/dist/hoi4_agent_tools/core/workspace.js +114 -8
  18. package/dist/hoi4_agent_tools/event/artifact-validation.js +1 -1
  19. package/dist/hoi4_agent_tools/event/service.js +1 -1
  20. package/dist/hoi4_agent_tools/focus/service.js +2 -2
  21. package/dist/hoi4_agent_tools/gui/studio.js +16 -3
  22. package/dist/hoi4_agent_tools/map/model.js +81 -2
  23. package/dist/hoi4_agent_tools/map/service.js +3 -2
  24. package/dist/hoi4_agent_tools/map/validation.js +60 -4
  25. package/dist/hoi4_agent_tools/mcp/tools/reference.js +10 -1
  26. package/dist/hoi4_agent_tools/package-check/service.js +1 -1
  27. package/dist/hoi4_agent_tools/reference/lines.js +12 -0
  28. package/dist/hoi4_agent_tools/reference/operations.js +45 -2
  29. package/dist/hoi4_agent_tools/reference/ranking.js +309 -0
  30. package/dist/hoi4_agent_tools/reference/script-validation.js +267 -0
  31. package/dist/hoi4_agent_tools/reference/service.js +285 -177
  32. package/dist/hoi4_agent_tools/reference/source-locate.js +156 -0
  33. package/dist/hoi4_agent_tools/reference/source-lookup.js +125 -18
  34. package/dist/hoi4_agent_tools/reference/source-navigation.js +130 -0
  35. package/dist/hoi4_agent_tools/reference/suggestions.js +38 -0
  36. package/dist/hoi4_agent_tools/schemas/reference.js +135 -2
  37. package/dist/hoi4_agent_tools/schemas/script-validation.js +69 -0
  38. package/dist/hoi4_agent_tools/schemas/transaction.js +1 -1
  39. package/dist/hoi4_agent_tools/technology/service.js +1 -1
  40. package/dist/hoi4_agent_tools/version.js +1 -1
  41. package/docs/README.md +1 -0
  42. package/docs/development.md +6 -0
  43. package/docs/examples.md +3 -13
  44. package/docs/gui.md +1 -15
  45. package/docs/map.md +10 -0
  46. package/docs/reference.md +47 -4
  47. package/docs/research/reference-benchmark.md +80 -0
  48. package/docs/setup.md +15 -0
  49. package/docs/tools.md +37 -36
  50. package/npm-shrinkwrap.json +7101 -0
  51. package/package.json +8 -7
  52. package/schemas/configuration.schema.json +95 -88
  53. package/schemas/continuous-focus-palette.schema.json +1 -1
  54. package/schemas/custom-weighted-pool.schema.json +1 -1
  55. package/schemas/decision-inspect-input.schema.json +1 -1
  56. package/schemas/event-feature-manifest.schema.json +1 -1
  57. package/schemas/focus-plan.schema.json +1 -1
  58. package/schemas/focus-planning-sidecar.schema.json +1 -1
  59. package/schemas/gui-animation-source.schema.json +1 -1
  60. package/schemas/gui-helper.schema.json +1 -1
  61. package/schemas/gui-scenario.schema.json +1 -1
  62. package/schemas/helper-expansion-request.schema.json +1 -1
  63. package/schemas/helper-expansion-summary.schema.json +1 -1
  64. package/schemas/impact-inspect-input.schema.json +1 -1
  65. package/schemas/map-operation.schema.json +1 -1
  66. package/schemas/mechanic-test-input.schema.json +1 -1
  67. package/schemas/operation-result.schema.json +1 -1
  68. package/schemas/package-check-input.schema.json +1 -1
  69. package/schemas/package-manifest.schema.json +1 -1
  70. package/schemas/probability-analysis-result.schema.json +1 -1
  71. package/schemas/probability-compare-input.schema.json +1 -1
  72. package/schemas/probability-evaluate-input.schema.json +1 -1
  73. package/schemas/probability-inspect-input.schema.json +1 -1
  74. package/schemas/probability-render-input.schema.json +1 -1
  75. package/schemas/probability-scenario-set.schema.json +1 -1
  76. package/schemas/probability-sequence-input.schema.json +1 -1
  77. package/schemas/probability-simulate-input.schema.json +1 -1
  78. package/schemas/probability-sweep-input.schema.json +1 -1
  79. package/schemas/reference-context-input.schema.json +1 -1
  80. package/schemas/reference-read-input.schema.json +9 -1
  81. package/schemas/reference-search-input.schema.json +1 -1
  82. package/schemas/scenario-suite.schema.json +1 -1
  83. package/schemas/scenario-test-input.schema.json +1 -1
  84. package/schemas/script-validate-input.schema.json +92 -0
  85. package/schemas/shared-scenario.schema.json +1 -1
  86. package/schemas/source-lookup-input.schema.json +144 -15
  87. package/server.json +2 -2
@@ -10,6 +10,53 @@ import { containedGeneratedPath } from './workspace.js';
10
10
  import { ServiceError } from './result.js';
11
11
  const maximumCheckpointRecoveries = 2;
12
12
  const maximumPreDispatchRecoveries = 2;
13
+ // Native and IPC codes are fixed vocabulary. Error messages can contain paths or source
14
+ // text, so only an allowlisted code and the fixed coordination stage are reported.
15
+ const nativeFailureCodes = new Set([
16
+ 'EACCES',
17
+ 'EAGAIN',
18
+ 'EBADF',
19
+ 'EBUSY',
20
+ 'ECHILD',
21
+ 'ECONNRESET',
22
+ 'EEXIST',
23
+ 'EIO',
24
+ 'EMFILE',
25
+ 'ENFILE',
26
+ 'ENOENT',
27
+ 'ENOMEM',
28
+ 'ENOSPC',
29
+ 'ENOTDIR',
30
+ 'ENOTEMPTY',
31
+ 'EPERM',
32
+ 'EPIPE',
33
+ 'ESRCH',
34
+ 'ETIMEDOUT',
35
+ 'ERR_IPC_CHANNEL_CLOSED',
36
+ 'ERR_IPC_DISCONNECTED',
37
+ ]);
38
+ function nativeFailureCode(error) {
39
+ if (typeof error !== 'object' || error === null || !('code' in error))
40
+ return undefined;
41
+ return typeof error.code === 'string' && nativeFailureCodes.has(error.code)
42
+ ? error.code
43
+ : undefined;
44
+ }
45
+ /** Map a launcher failure to a stable, path-free job failure. */
46
+ export function workerFailure(error, stage) {
47
+ if (error instanceof ServiceError)
48
+ return { code: error.code, message: error.message };
49
+ const nativeCode = nativeFailureCode(error);
50
+ return nativeCode === undefined
51
+ ? {
52
+ code: 'JOB_WORKER_FAILED',
53
+ message: `The isolated worker stopped without publishing a result (stage ${stage})`,
54
+ }
55
+ : {
56
+ code: `JOB_WORKER_${nativeCode}`,
57
+ message: `Worker coordination could not complete (${nativeCode}, stage ${stage}); inspect the retained job outcome`,
58
+ };
59
+ }
13
60
  /** Bounded, fixed-entry child execution; client cancellation/disconnection never kills a worker. */
14
61
  export class JobWorkerHost {
15
62
  engine;
@@ -69,10 +116,13 @@ export class JobWorkerHost {
69
116
  throw new ServiceError('JOB_OPERATION_UNAVAILABLE', 'This domain has not yet been registered for worker execution');
70
117
  const signal = new AbortController().signal;
71
118
  const recovered = new Set();
119
+ const progress = { stage: 'admission' };
72
120
  let preDispatchRecoveries = 0;
73
121
  for (;;) {
74
122
  try {
75
- await this.scheduler.run(this.owner, 1024, signal, () => this.capacity.run(signal, (lease) => this.launch(workspaceId, id, lease, principal)));
123
+ progress.stage = 'admission';
124
+ await this.scheduler.run(this.owner, 1024, signal, () => this.capacity.run(signal, (lease) => this.launch(workspaceId, id, lease, progress, principal)));
125
+ progress.stage = 'supervision';
76
126
  const current = await this.jobs.get(workspaceId, id, principal);
77
127
  if (['completed', 'cancelled', 'failed'].includes(current.status))
78
128
  return current;
@@ -104,12 +154,7 @@ export class JobWorkerHost {
104
154
  recovered.add(checkpoint);
105
155
  continue;
106
156
  }
107
- return await this.jobs.failInterrupted(workspaceId, id, {
108
- code: error instanceof ServiceError ? error.code : 'JOB_WORKER_FAILED',
109
- message: error instanceof ServiceError
110
- ? error.message
111
- : 'The isolated worker stopped without publishing a result',
112
- }, principal);
157
+ return await this.jobs.failInterrupted(workspaceId, id, workerFailure(error, progress.stage), principal);
113
158
  }
114
159
  }
115
160
  }
@@ -126,7 +171,7 @@ export class JobWorkerHost {
126
171
  await delay(100);
127
172
  }
128
173
  }
129
- async launch(workspaceId, id, lease, principal) {
174
+ async launch(workspaceId, id, lease, progress, principal) {
130
175
  // Resolve the same configured workspace and grants again immediately before dispatch.
131
176
  await this.jobs.get(workspaceId, id, principal);
132
177
  const sourceMode = import.meta.url.endsWith('.ts');
@@ -143,15 +188,18 @@ export class JobWorkerHost {
143
188
  workspaceIds: [...new Set([...grant.workspaceIds, ...discoveredWorkspaceIds])],
144
189
  }
145
190
  : grant;
191
+ progress.stage = 'startup';
146
192
  const child = spawn(process.execPath, [...(sourceMode ? ['--import', import.meta.resolve('tsx')] : []), entry], {
147
193
  detached: true,
148
194
  cwd: process.cwd(),
149
195
  stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
150
196
  windowsHide: true,
151
197
  });
198
+ let phase = 'waiting_for_ready';
199
+ let acknowledgmentTimer;
152
200
  const initialized = new Promise((resolve, reject) => {
153
201
  child.once('error', reject);
154
- child.once('exit', () => reject(new ServiceError('JOB_WORKER_STARTUP_EXIT', 'The worker exited before completing its readiness handshake')));
202
+ child.once('exit', (code, signal) => reject(new ServiceError('JOB_WORKER_STARTUP_EXIT', `The worker exited before completing its startup handshake (phase ${phase}, exit ${code ?? 'none'}, signal ${signal ?? 'none'})`)));
155
203
  child.once('message', (value) => {
156
204
  void (async () => {
157
205
  if (typeof value !== 'object' ||
@@ -160,9 +208,27 @@ export class JobWorkerHost {
160
208
  value.type !== 'ready' ||
161
209
  child.pid === undefined)
162
210
  throw new ServiceError('JOB_WORKER_PROTOCOL', 'The worker did not provide its readiness handshake');
211
+ progress.stage = 'handoff';
163
212
  await lease.handoffToProcess(child.pid);
213
+ progress.stage = 'dispatch';
164
214
  await new Promise((resolve, reject) => {
165
- child.once('disconnect', resolve);
215
+ phase = 'waiting_for_acceptance';
216
+ acknowledgmentTimer = setTimeout(() => reject(new ServiceError('JOB_WORKER_ACCEPT_TIMEOUT', 'The worker did not acknowledge receipt within its startup window')), 30_000);
217
+ child.once('message', (accepted) => {
218
+ if (typeof accepted !== 'object' ||
219
+ accepted === null ||
220
+ !('type' in accepted) ||
221
+ accepted.type !== 'accepted') {
222
+ reject(new ServiceError('JOB_WORKER_PROTOCOL', 'The worker did not acknowledge the dispatched job'));
223
+ return;
224
+ }
225
+ phase = 'accepted';
226
+ // Only close IPC after the child confirms receipt. The persistent job,
227
+ // rather than the launcher's lifetime, owns execution after this point.
228
+ if (child.connected)
229
+ child.disconnect();
230
+ resolve();
231
+ });
166
232
  child.send({
167
233
  configuration: {
168
234
  ...configuration,
@@ -178,14 +244,7 @@ export class JobWorkerHost {
178
244
  jobId: id,
179
245
  ...(principal === undefined ? {} : { principal }),
180
246
  }, (error) => {
181
- if (error === null) {
182
- // IPC is startup-only. Closing it after dispatch prevents launcher
183
- // lifetime from becoming an implicit cancellation channel.
184
- if (child.connected)
185
- child.disconnect();
186
- resolve();
187
- }
188
- else
247
+ if (error !== null)
189
248
  reject(error);
190
249
  });
191
250
  });
@@ -199,7 +258,13 @@ export class JobWorkerHost {
199
258
  });
200
259
  });
201
260
  });
202
- await initialized;
261
+ try {
262
+ await initialized;
263
+ }
264
+ finally {
265
+ clearTimeout(acknowledgmentTimer);
266
+ }
267
+ progress.stage = 'supervision';
203
268
  for (;;) {
204
269
  const record = await this.jobs.get(workspaceId, id, principal);
205
270
  if (['completed', 'failed', 'cancelled'].includes(record.status))
@@ -1,16 +1,10 @@
1
1
  import { z } from 'zod/v4';
2
2
  import { serverConfigurationSchema } from './configuration.js';
3
3
  import { CoreEngine } from './engine.js';
4
- import { JobExecutor, JobOperations } from './job-executor.js';
4
+ import { JobExecutor } from './job-executor.js';
5
5
  import { JobService } from './job-service.js';
6
6
  import { WorkspaceResolver } from './workspace.js';
7
- import { registerEventJobs } from '../event/job-operations.js';
8
- import { registerFocusJobs } from '../focus/job-operations.js';
9
- import { registerGuiJobs } from '../gui/job-operations.js';
10
- import { registerMapJobs } from '../map/job-operations.js';
11
- import { registerProbabilityJobs } from '../probability/job-operations.js';
12
- import { registerTechnologyJobs } from '../technology/job-operations.js';
13
- import { registerAnalysisJobs } from './analysis-job-operations.js';
7
+ import { registerWorkerOperations } from './job-operation-registration.js';
14
8
  // Fixed internal process entry point. It accepts only a trusted host's structured IPC
15
9
  // message, never a script path, module name, callback, command, or transport request.
16
10
  const inputSchema = z
@@ -38,19 +32,17 @@ process.once('message', (value) => {
38
32
  clearTimeout(startup);
39
33
  void (async () => {
40
34
  const input = inputSchema.parse(value);
35
+ // The send callback confirms enqueueing, not receipt. Acknowledge before the host
36
+ // disconnects; losing that channel after receipt never cancels durable execution.
37
+ if (process.connected)
38
+ process.send?.({ type: 'accepted' }, () => undefined);
41
39
  const resolver = await WorkspaceResolver.create(input.configuration);
42
40
  const engine = new CoreEngine(resolver);
43
41
  // Recovery is targeted by the job executor. Do not sweep or interfere with
44
42
  // unrelated live rewrite journals when a worker starts.
45
43
  const jobs = await JobService.create(resolver);
46
- const operations = new JobOperations();
47
- registerEventJobs(operations, engine);
48
- registerAnalysisJobs(operations, engine);
49
- registerTechnologyJobs(operations, engine);
50
- registerProbabilityJobs(operations, engine);
51
- registerMapJobs(operations, engine);
52
- registerGuiJobs(operations, engine);
53
- registerFocusJobs(operations, engine);
44
+ const record = await jobs.get(input.workspaceId, input.jobId, input.principal);
45
+ const operations = await registerWorkerOperations(engine, record.request.toolName);
54
46
  await new JobExecutor(engine, jobs, operations).run(input.workspaceId, input.jobId, input.principal);
55
47
  })()
56
48
  .catch(() => {
@@ -9,7 +9,7 @@ import { probabilityCompareRequestSchema, probabilityEvaluateRequestSchema, prob
9
9
  import { technologyCompareRequestSchema, technologyInspectRequestSchema, technologyRenderRequestSchema, } from '../schemas/technology.js';
10
10
  import { currentJobOwner, jobOwnerLiveness, jobRetentionExpired, } from './job-store.js';
11
11
  import { JobService } from './job-service.js';
12
- import { JobWorkerHost } from './job-worker-host.js';
12
+ import { JobWorkerHost, workerFailure } from './job-worker-host.js';
13
13
  import { compareCodeUnits, hashCanonical, secureId } from './canonical.js';
14
14
  import { ServiceError } from './result.js';
15
15
  import { errorResult } from './operation-result.js';
@@ -378,12 +378,7 @@ export class OperationTaskService {
378
378
  .then(() => undefined)
379
379
  .catch(async (error) => {
380
380
  const jobs = await this.jobs();
381
- await jobs.failInterrupted(record.scope.workspaceId, record.id, {
382
- code: error instanceof ServiceError ? error.code : 'JOB_WORKER_FAILED',
383
- message: error instanceof ServiceError
384
- ? error.message
385
- : 'The isolated worker could not be started',
386
- }, principal);
381
+ await jobs.failInterrupted(record.scope.workspaceId, record.id, workerFailure(error, 'admission'), principal);
387
382
  })
388
383
  .finally(() => {
389
384
  this.launches.delete(key);
@@ -54,6 +54,8 @@ export class PersistentAnalysisCache {
54
54
  maxBytes;
55
55
  maxEntries;
56
56
  maxSingleBytes;
57
+ batchMaxWrites;
58
+ batchMaxPayloadBytes;
57
59
  #lock;
58
60
  #hits = 0;
59
61
  #misses = 0;
@@ -63,23 +65,30 @@ export class PersistentAnalysisCache {
63
65
  #evictions = 0;
64
66
  #invalidEntries = 0;
65
67
  #oversized = 0;
68
+ #limitedBatches = 0;
66
69
  #invalidAddresses = new Set();
67
- constructor(state, root, maxBytes, maxEntries, maxSingleBytes) {
70
+ constructor(state, root, maxBytes, maxEntries, maxSingleBytes, batchMaxWrites, batchMaxPayloadBytes) {
68
71
  this.state = state;
69
72
  this.root = root;
70
73
  this.maxBytes = maxBytes;
71
74
  this.maxEntries = maxEntries;
72
75
  this.maxSingleBytes = maxSingleBytes;
76
+ this.batchMaxWrites = batchMaxWrites;
77
+ this.batchMaxPayloadBytes = batchMaxPayloadBytes;
73
78
  this.#lock = new SharedRequestCapacity(root, 1);
74
79
  }
75
80
  static async create(state, options = {}) {
76
81
  const maxBytes = options.maxBytes ?? 268_435_456;
77
82
  const maxEntries = options.maxEntries ?? 50_000;
78
83
  const maxSingleBytes = options.maxSingleBytes ?? 16_777_216;
84
+ const batchMaxWrites = options.batchMaxWrites ?? 16;
85
+ const batchMaxPayloadBytes = options.batchMaxPayloadBytes ?? 1_048_576;
79
86
  for (const [name, value] of [
80
87
  ['byte budget', maxBytes],
81
88
  ['entry budget', maxEntries],
82
89
  ['single-entry budget', maxSingleBytes],
90
+ ['batch write budget', batchMaxWrites],
91
+ ['batch payload budget', batchMaxPayloadBytes],
83
92
  ]) {
84
93
  if (!Number.isSafeInteger(value) || value < 0)
85
94
  throw new RangeError(`Persistent analysis cache ${name} must be a non-negative integer`);
@@ -88,7 +97,7 @@ export class PersistentAnalysisCache {
88
97
  const root = await containedGeneratedPath(state.root, 'analysis-cache');
89
98
  await mkdir(root, { recursive: true, mode: 0o700 });
90
99
  await assertUnlinked(root);
91
- return new PersistentAnalysisCache(state, root, maxBytes, maxEntries, maxSingleBytes);
100
+ return new PersistentAnalysisCache(state, root, maxBytes, maxEntries, maxSingleBytes, batchMaxWrites, batchMaxPayloadBytes);
92
101
  }
93
102
  async hydrate(scope, files, documents, segments, signal) {
94
103
  signal?.throwIfAborted();
@@ -101,7 +110,7 @@ export class PersistentAnalysisCache {
101
110
  signal?.throwIfAborted();
102
111
  const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
103
112
  const documentAddress = sourceAddress(key);
104
- if (sourceEntries.has(documentAddress)) {
113
+ if (!documents.has(key) && sourceEntries.has(documentAddress)) {
105
114
  const payload = await this.read(scopeHash, 'source-document', documentAddress, signal);
106
115
  if (payload !== undefined) {
107
116
  try {
@@ -118,7 +127,7 @@ export class PersistentAnalysisCache {
118
127
  }
119
128
  for (const provinceDefinition of [false, true]) {
120
129
  const address = indexSegmentAddress(file, provinceDefinition);
121
- if (!indexEntries.has(address))
130
+ if (segments.has(address) || !indexEntries.has(address))
122
131
  continue;
123
132
  const payload = await this.read(scopeHash, 'index-segment', address, signal);
124
133
  if (payload === undefined)
@@ -140,15 +149,34 @@ export class PersistentAnalysisCache {
140
149
  this.inventory(scopeHash, 'source-document', signal),
141
150
  this.inventory(scopeHash, 'index-segment', signal),
142
151
  ]);
152
+ const batch = { writes: 0, payloadBytes: 0, limited: false };
153
+ const retain = async (kind, address, payload) => {
154
+ if (batch.writes >= this.batchMaxWrites ||
155
+ payload.length > this.batchMaxPayloadBytes - batch.payloadBytes) {
156
+ batch.limited = true;
157
+ return false;
158
+ }
159
+ const written = await this.write(scopeHash, kind, address, payload, signal);
160
+ if (written) {
161
+ batch.writes++;
162
+ batch.payloadBytes += payload.length;
163
+ }
164
+ return written;
165
+ };
143
166
  for (const file of files) {
144
167
  signal.throwIfAborted();
168
+ if (batch.writes >= this.batchMaxWrites ||
169
+ batch.payloadBytes >= this.batchMaxPayloadBytes) {
170
+ batch.limited = true;
171
+ break;
172
+ }
145
173
  const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
146
174
  const documentAddress = sourceAddress(key);
147
175
  const document = documents.exportEncoded(key);
148
176
  if (document !== undefined &&
149
177
  (!sourceEntries.has(documentAddress) ||
150
178
  this.#invalidAddresses.has(`${scopeHash}\0source-document\0${documentAddress}`))) {
151
- if (await this.write(scopeHash, 'source-document', documentAddress, document, signal)) {
179
+ if (await retain('source-document', documentAddress, document)) {
152
180
  sourceEntries.add(documentAddress);
153
181
  }
154
182
  }
@@ -159,11 +187,13 @@ export class PersistentAnalysisCache {
159
187
  (indexEntries.has(address) &&
160
188
  !this.#invalidAddresses.has(`${scopeHash}\0index-segment\0${address}`)))
161
189
  continue;
162
- if (await this.write(scopeHash, 'index-segment', address, Buffer.from(segment), signal)) {
190
+ if (await retain('index-segment', address, Buffer.from(segment))) {
163
191
  indexEntries.add(address);
164
192
  }
165
193
  }
166
194
  }
195
+ if (batch.limited)
196
+ this.#limitedBatches++;
167
197
  await this.prune(signal);
168
198
  });
169
199
  }
@@ -177,6 +207,7 @@ export class PersistentAnalysisCache {
177
207
  evictions: this.#evictions,
178
208
  invalidEntries: this.#invalidEntries,
179
209
  oversized: this.#oversized,
210
+ limitedBatches: this.#limitedBatches,
180
211
  };
181
212
  }
182
213
  async directory(scopeHash, kind) {
@@ -1,13 +1,59 @@
1
- import { open } from 'node:fs/promises';
1
+ import { open, realpath, stat } from 'node:fs/promises';
2
2
  import path from 'node:path';
3
- import fg from 'fast-glob';
3
+ import { iterateFiles } from './file-glob.js';
4
4
  import { compareCodeUnits, sha256Bytes } from './canonical.js';
5
5
  import { DEFAULT_SCAN_MAX_BYTES, DEFAULT_SCAN_MAX_FILE_BYTES, DEFAULT_SCAN_MAX_FILES, } from './configuration.js';
6
6
  import { ServiceError } from './result.js';
7
+ import { isWithin } from './workspace.js';
8
+ const SCAN_IO_CONCURRENCY = 8;
9
+ const SCAN_READ_BATCH_BYTES = 16_777_216;
10
+ export async function verifySource(root, source) {
11
+ const resolved = await realpath(source.absolutePath);
12
+ if (!isWithin(root.path, resolved))
13
+ throw new ServiceError('SCAN_ROOT_ESCAPE', 'Source path resolves outside its authorized root', {
14
+ file: source.relativePath,
15
+ });
16
+ const current = await stat(resolved);
17
+ const expected = source.metadata;
18
+ if (current.dev !== expected.dev ||
19
+ current.ino !== expected.ino ||
20
+ current.size !== expected.size ||
21
+ current.mtimeMs !== expected.mtimeMs ||
22
+ current.ctimeMs !== expected.ctimeMs)
23
+ throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source changed while it was being scanned; retry a focused query', { file: source.relativePath });
24
+ }
25
+ export async function prepareSource(root, relativePath, signal) {
26
+ signal?.throwIfAborted();
27
+ const absolutePath = path.join(root.path, relativePath);
28
+ const canonicalPath = await realpath(absolutePath);
29
+ if (!isWithin(root.path, canonicalPath))
30
+ throw new ServiceError('SCAN_ROOT_ESCAPE', 'Source path resolves outside its authorized root', {
31
+ file: relativePath,
32
+ });
33
+ const handle = await open(canonicalPath, 'r');
34
+ try {
35
+ const source = {
36
+ relativePath,
37
+ absolutePath,
38
+ canonicalPath,
39
+ handle,
40
+ metadata: await handle.stat(),
41
+ };
42
+ await verifySource(root, source);
43
+ signal?.throwIfAborted();
44
+ return source;
45
+ }
46
+ catch (error) {
47
+ await handle.close();
48
+ throw error;
49
+ }
50
+ }
7
51
  function normalizeRelative(value) {
8
52
  return value.replaceAll('\\', '/').replace(/^\.\//u, '');
9
53
  }
10
54
  function rootLabel(root) {
55
+ if (root.kind === 'dlc' && root.label !== undefined)
56
+ return root.label;
11
57
  return root.kind === 'dependency' ? `dependency-${root.loadOrder}` : root.kind;
12
58
  }
13
59
  function hiddenByReplacePath(workspace, root, relativePath) {
@@ -86,6 +132,7 @@ export class WorkspaceScanner {
86
132
  }
87
133
  const roots = workspace.roots
88
134
  .filter((root) => (root.kind === 'game' ||
135
+ root.kind === 'dlc' ||
89
136
  root.kind === 'dependency' ||
90
137
  root.kind === 'mod' ||
91
138
  root.kind === 'fixture') &&
@@ -98,14 +145,12 @@ export class WorkspaceScanner {
98
145
  options.signal?.throwIfAborted();
99
146
  // Re-enumerate every root: installing content or changing load order can
100
147
  // invalidate a vanilla inventory just as it can a mod inventory.
101
- const matches = fg.stream(options.patterns, {
148
+ const matches = iterateFiles(options.patterns, {
102
149
  cwd: root.path,
103
- onlyFiles: true,
104
- unique: true,
105
- dot: false,
106
- followSymbolicLinks: false,
107
150
  ignore: options.ignore ?? ['**/.hoi4-agent/**'],
151
+ ...(options.signal === undefined ? {} : { signal: options.signal }),
108
152
  });
153
+ const relativePaths = [];
109
154
  for await (const match of matches) {
110
155
  options.signal?.throwIfAborted();
111
156
  enumeratedFiles += 1;
@@ -115,55 +160,99 @@ export class WorkspaceScanner {
115
160
  limit: maxFiles,
116
161
  });
117
162
  }
118
- const relativePath = normalizeRelative(String(match));
163
+ const relativePath = normalizeRelative(match);
119
164
  if (hiddenByReplacePath(workspace, root, relativePath))
120
165
  continue;
121
- const absolutePath = path.join(root.path, relativePath);
122
- const cacheKey = this.cacheKey(absolutePath);
123
- const handle = await open(absolutePath, 'r');
166
+ relativePaths.push(relativePath);
167
+ }
168
+ for (let offset = 0; offset < relativePaths.length; offset += SCAN_IO_CONCURRENCY) {
169
+ options.signal?.throwIfAborted();
170
+ const prepared = await Promise.allSettled(relativePaths
171
+ .slice(offset, offset + SCAN_IO_CONCURRENCY)
172
+ .map((relativePath) => prepareSource(root, relativePath, options.signal)));
173
+ const opened = prepared.flatMap((entry) => entry.status === 'fulfilled' ? [entry.value] : []);
124
174
  try {
125
- const metadata = await handle.stat();
126
- if (!metadata.isFile())
127
- continue;
128
- const remaining = maxBytes - totalBytes;
129
- if (metadata.size > remaining || metadata.size > this.serverMaxFileBytes) {
130
- throw new ServiceError('SCAN_BYTE_LIMIT', 'Scan exceeds the configured byte limit', {
131
- file: relativePath,
132
- fileBytes: metadata.size,
133
- bytes: totalBytes,
134
- limit: maxBytes,
135
- perFileLimit: this.serverMaxFileBytes,
136
- });
175
+ const failed = prepared.find((entry) => entry.status === 'rejected');
176
+ if (failed?.status === 'rejected')
177
+ throw failed.reason;
178
+ const readable = opened.filter(({ metadata }) => metadata.isFile());
179
+ let reserved = 0;
180
+ for (const source of readable) {
181
+ if (source.metadata.size > maxBytes - totalBytes - reserved ||
182
+ source.metadata.size > this.serverMaxFileBytes)
183
+ throw new ServiceError('SCAN_BYTE_LIMIT', 'Scan exceeds the configured byte limit', {
184
+ file: source.relativePath,
185
+ fileBytes: source.metadata.size,
186
+ bytes: totalBytes + reserved,
187
+ limit: maxBytes,
188
+ perFileLimit: this.serverMaxFileBytes,
189
+ });
190
+ reserved += source.metadata.size;
191
+ }
192
+ let readOffset = 0;
193
+ while (readOffset < readable.length) {
194
+ const group = [];
195
+ let groupBytes = 0;
196
+ while (readOffset < readable.length) {
197
+ const source = readable[readOffset];
198
+ if (group.length > 0 && groupBytes + source.metadata.size > SCAN_READ_BATCH_BYTES)
199
+ break;
200
+ group.push(source);
201
+ groupBytes += source.metadata.size;
202
+ readOffset++;
203
+ }
204
+ const reads = await Promise.allSettled(group.map(async (source) => {
205
+ let bytes;
206
+ try {
207
+ bytes = await readBoundedFile(source.handle, source.metadata.size, options.signal);
208
+ }
209
+ catch (error) {
210
+ if (error instanceof ServiceError && error.code === 'SCAN_BYTE_LIMIT')
211
+ throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source grew while it was being scanned; retry a focused query', { file: source.relativePath });
212
+ throw error;
213
+ }
214
+ if (bytes.length !== source.metadata.size)
215
+ throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source changed while it was being scanned; retry a focused query', { file: source.relativePath });
216
+ await verifySource(root, source);
217
+ options.signal?.throwIfAborted();
218
+ return bytes;
219
+ }));
220
+ const readFailure = reads.find((entry) => entry.status === 'rejected');
221
+ if (readFailure?.status === 'rejected')
222
+ throw readFailure.reason;
223
+ for (const [index, read] of reads.entries()) {
224
+ if (read.status !== 'fulfilled')
225
+ continue;
226
+ const source = group[index];
227
+ const cacheKey = this.cacheKey(source.absolutePath);
228
+ const observed = read.value;
229
+ const sha256 = sha256Bytes(observed);
230
+ const cached = this.#sourceCache.get(cacheKey);
231
+ const bytes = cached?.sha256 === sha256 && cached.bytes.equals(observed)
232
+ ? cached.bytes
233
+ : observed;
234
+ this.retainSource(cacheKey, { sha256, bytes });
235
+ totalBytes += bytes.length;
236
+ result.push({
237
+ absolutePath: source.absolutePath,
238
+ displayPath: `${rootLabel(root)}:${source.relativePath}`,
239
+ relativePath: source.relativePath,
240
+ rootKind: root.kind,
241
+ loadOrder: root.loadOrder,
242
+ size: bytes.length,
243
+ modifiedMs: source.metadata.mtimeMs,
244
+ sha256,
245
+ bytes,
246
+ });
247
+ }
137
248
  }
138
- const cached = this.#sourceCache.get(cacheKey);
139
- // Metadata is not content identity (including on aliased/networked
140
- // filesystems). Verify bytes before reusing parsed/indexed facts.
141
- const observed = await readBoundedFile(handle, Math.min(remaining, this.serverMaxFileBytes), options.signal);
142
- const sha256 = sha256Bytes(observed);
143
- const bytes = cached?.sha256 === sha256 && cached.bytes.equals(observed) ? cached.bytes : observed;
144
- this.retainSource(cacheKey, {
145
- sha256,
146
- bytes,
147
- });
148
- totalBytes += bytes.length;
149
- const scanned = {
150
- absolutePath,
151
- displayPath: `${rootLabel(root)}:${relativePath}`,
152
- relativePath,
153
- rootKind: root.kind,
154
- loadOrder: root.loadOrder,
155
- size: bytes.length,
156
- modifiedMs: metadata.mtimeMs,
157
- sha256,
158
- bytes,
159
- };
160
- result.push(scanned);
161
249
  }
162
250
  finally {
163
- await handle.close();
251
+ await Promise.all(opened.map(({ handle }) => handle.close()));
164
252
  }
165
253
  }
166
254
  }
255
+ options.signal?.throwIfAborted();
167
256
  result.sort((left, right) => left.loadOrder - right.loadOrder || compareCodeUnits(left.relativePath, right.relativePath));
168
257
  const groups = new Map();
169
258
  for (const file of result) {