hoi4-agent-tools 3.5.1 → 3.7.0

package/dist/hoi4_agent_tools/core/job-worker.js

@@ -1,16 +1,10 @@
 import { z } from 'zod/v4';
 import { serverConfigurationSchema } from './configuration.js';
 import { CoreEngine } from './engine.js';
-import { JobExecutor, JobOperations } from './job-executor.js';
+import { JobExecutor } from './job-executor.js';
 import { JobService } from './job-service.js';
 import { WorkspaceResolver } from './workspace.js';
-import { registerEventJobs } from '../event/job-operations.js';
-import { registerFocusJobs } from '../focus/job-operations.js';
-import { registerGuiJobs } from '../gui/job-operations.js';
-import { registerMapJobs } from '../map/job-operations.js';
-import { registerProbabilityJobs } from '../probability/job-operations.js';
-import { registerTechnologyJobs } from '../technology/job-operations.js';
-import { registerAnalysisJobs } from './analysis-job-operations.js';
+import { registerWorkerOperations } from './job-operation-registration.js';
 // Fixed internal process entry point. It accepts only a trusted host's structured IPC
 // message, never a script path, module name, callback, command, or transport request.
 const inputSchema = z
@@ -38,19 +32,17 @@ process.once('message', (value) => {
     clearTimeout(startup);
     void (async () => {
         const input = inputSchema.parse(value);
+        // The send callback confirms enqueueing, not receipt. Acknowledge before the host
+        // disconnects; losing that channel after receipt never cancels durable execution.
+        if (process.connected)
+            process.send?.({ type: 'accepted' }, () => undefined);
         const resolver = await WorkspaceResolver.create(input.configuration);
         const engine = new CoreEngine(resolver);
         // Recovery is targeted by the job executor. Do not sweep or interfere with
         // unrelated live rewrite journals when a worker starts.
         const jobs = await JobService.create(resolver);
-        const operations = new JobOperations();
-        registerEventJobs(operations, engine);
-        registerAnalysisJobs(operations, engine);
-        registerTechnologyJobs(operations, engine);
-        registerProbabilityJobs(operations, engine);
-        registerMapJobs(operations, engine);
-        registerGuiJobs(operations, engine);
-        registerFocusJobs(operations, engine);
+        const record = await jobs.get(input.workspaceId, input.jobId, input.principal);
+        const operations = await registerWorkerOperations(engine, record.request.toolName);
         await new JobExecutor(engine, jobs, operations).run(input.workspaceId, input.jobId, input.principal);
     })()
         .catch(() => {

package/dist/hoi4_agent_tools/core/persistent-analysis-cache.js

@@ -54,6 +54,8 @@ export class PersistentAnalysisCache {
     maxBytes;
     maxEntries;
     maxSingleBytes;
+    batchMaxWrites;
+    batchMaxPayloadBytes;
     #lock;
     #hits = 0;
     #misses = 0;
@@ -63,23 +65,30 @@ export class PersistentAnalysisCache {
     #evictions = 0;
     #invalidEntries = 0;
     #oversized = 0;
+    #limitedBatches = 0;
     #invalidAddresses = new Set();
-    constructor(state, root, maxBytes, maxEntries, maxSingleBytes) {
+    constructor(state, root, maxBytes, maxEntries, maxSingleBytes, batchMaxWrites, batchMaxPayloadBytes) {
         this.state = state;
         this.root = root;
         this.maxBytes = maxBytes;
         this.maxEntries = maxEntries;
         this.maxSingleBytes = maxSingleBytes;
+        this.batchMaxWrites = batchMaxWrites;
+        this.batchMaxPayloadBytes = batchMaxPayloadBytes;
         this.#lock = new SharedRequestCapacity(root, 1);
     }
     static async create(state, options = {}) {
         const maxBytes = options.maxBytes ?? 268_435_456;
         const maxEntries = options.maxEntries ?? 50_000;
         const maxSingleBytes = options.maxSingleBytes ?? 16_777_216;
+        const batchMaxWrites = options.batchMaxWrites ?? 16;
+        const batchMaxPayloadBytes = options.batchMaxPayloadBytes ?? 1_048_576;
         for (const [name, value] of [
             ['byte budget', maxBytes],
             ['entry budget', maxEntries],
             ['single-entry budget', maxSingleBytes],
+            ['batch write budget', batchMaxWrites],
+            ['batch payload budget', batchMaxPayloadBytes],
         ]) {
             if (!Number.isSafeInteger(value) || value < 0)
                 throw new RangeError(`Persistent analysis cache ${name} must be a non-negative integer`);
@@ -88,7 +97,7 @@ export class PersistentAnalysisCache {
         const root = await containedGeneratedPath(state.root, 'analysis-cache');
         await mkdir(root, { recursive: true, mode: 0o700 });
         await assertUnlinked(root);
-        return new PersistentAnalysisCache(state, root, maxBytes, maxEntries, maxSingleBytes);
+        return new PersistentAnalysisCache(state, root, maxBytes, maxEntries, maxSingleBytes, batchMaxWrites, batchMaxPayloadBytes);
     }
     async hydrate(scope, files, documents, segments, signal) {
         signal?.throwIfAborted();
@@ -101,7 +110,7 @@ export class PersistentAnalysisCache {
             signal?.throwIfAborted();
             const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
             const documentAddress = sourceAddress(key);
-            if (sourceEntries.has(documentAddress)) {
+            if (!documents.has(key) && sourceEntries.has(documentAddress)) {
                 const payload = await this.read(scopeHash, 'source-document', documentAddress, signal);
                 if (payload !== undefined) {
                     try {
@@ -118,7 +127,7 @@ export class PersistentAnalysisCache {
             }
             for (const provinceDefinition of [false, true]) {
                 const address = indexSegmentAddress(file, provinceDefinition);
-                if (!indexEntries.has(address))
+                if (segments.has(address) || !indexEntries.has(address))
                     continue;
                 const payload = await this.read(scopeHash, 'index-segment', address, signal);
                 if (payload === undefined)
@@ -140,15 +149,34 @@ export class PersistentAnalysisCache {
                 this.inventory(scopeHash, 'source-document', signal),
                 this.inventory(scopeHash, 'index-segment', signal),
             ]);
+            const batch = { writes: 0, payloadBytes: 0, limited: false };
+            const retain = async (kind, address, payload) => {
+                if (batch.writes >= this.batchMaxWrites ||
+                    payload.length > this.batchMaxPayloadBytes - batch.payloadBytes) {
+                    batch.limited = true;
+                    return false;
+                }
+                const written = await this.write(scopeHash, kind, address, payload, signal);
+                if (written) {
+                    batch.writes++;
+                    batch.payloadBytes += payload.length;
+                }
+                return written;
+            };
             for (const file of files) {
                 signal.throwIfAborted();
+                if (batch.writes >= this.batchMaxWrites ||
+                    batch.payloadBytes >= this.batchMaxPayloadBytes) {
+                    batch.limited = true;
+                    break;
+                }
                 const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
                 const documentAddress = sourceAddress(key);
                 const document = documents.exportEncoded(key);
                 if (document !== undefined &&
                     (!sourceEntries.has(documentAddress) ||
                         this.#invalidAddresses.has(`${scopeHash}\0source-document\0${documentAddress}`))) {
-                    if (await this.write(scopeHash, 'source-document', documentAddress, document, signal)) {
+                    if (await retain('source-document', documentAddress, document)) {
                         sourceEntries.add(documentAddress);
                     }
                 }
@@ -159,11 +187,13 @@ export class PersistentAnalysisCache {
                         (indexEntries.has(address) &&
                             !this.#invalidAddresses.has(`${scopeHash}\0index-segment\0${address}`)))
                         continue;
-                    if (await this.write(scopeHash, 'index-segment', address, Buffer.from(segment), signal)) {
+                    if (await retain('index-segment', address, Buffer.from(segment))) {
                         indexEntries.add(address);
                     }
                 }
             }
+            if (batch.limited)
+                this.#limitedBatches++;
             await this.prune(signal);
         });
     }
@@ -177,6 +207,7 @@ export class PersistentAnalysisCache {
             evictions: this.#evictions,
             invalidEntries: this.#invalidEntries,
             oversized: this.#oversized,
+            limitedBatches: this.#limitedBatches,
         };
     }
     async directory(scopeHash, kind) {

package/dist/hoi4_agent_tools/core/scanner.js

@@ -1,9 +1,53 @@
-import { open } from 'node:fs/promises';
+import { open, realpath, stat } from 'node:fs/promises';
 import path from 'node:path';
-import fg from 'fast-glob';
+import { iterateFiles } from './file-glob.js';
 import { compareCodeUnits, sha256Bytes } from './canonical.js';
 import { DEFAULT_SCAN_MAX_BYTES, DEFAULT_SCAN_MAX_FILE_BYTES, DEFAULT_SCAN_MAX_FILES, } from './configuration.js';
 import { ServiceError } from './result.js';
+import { isWithin } from './workspace.js';
+const SCAN_IO_CONCURRENCY = 8;
+const SCAN_READ_BATCH_BYTES = 16_777_216;
+export async function verifySource(root, source) {
+    const resolved = await realpath(source.absolutePath);
+    if (!isWithin(root.path, resolved))
+        throw new ServiceError('SCAN_ROOT_ESCAPE', 'Source path resolves outside its authorized root', {
+            file: source.relativePath,
+        });
+    const current = await stat(resolved);
+    const expected = source.metadata;
+    if (current.dev !== expected.dev ||
+        current.ino !== expected.ino ||
+        current.size !== expected.size ||
+        current.mtimeMs !== expected.mtimeMs ||
+        current.ctimeMs !== expected.ctimeMs)
+        throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source changed while it was being scanned; retry a focused query', { file: source.relativePath });
+}
+export async function prepareSource(root, relativePath, signal) {
+    signal?.throwIfAborted();
+    const absolutePath = path.join(root.path, relativePath);
+    const canonicalPath = await realpath(absolutePath);
+    if (!isWithin(root.path, canonicalPath))
+        throw new ServiceError('SCAN_ROOT_ESCAPE', 'Source path resolves outside its authorized root', {
+            file: relativePath,
+        });
+    const handle = await open(canonicalPath, 'r');
+    try {
+        const source = {
+            relativePath,
+            absolutePath,
+            canonicalPath,
+            handle,
+            metadata: await handle.stat(),
+        };
+        await verifySource(root, source);
+        signal?.throwIfAborted();
+        return source;
+    }
+    catch (error) {
+        await handle.close();
+        throw error;
+    }
+}
 function normalizeRelative(value) {
     return value.replaceAll('\\', '/').replace(/^\.\//u, '');
 }
@@ -98,14 +142,12 @@ export class WorkspaceScanner {
             options.signal?.throwIfAborted();
             // Re-enumerate every root: installing content or changing load order can
             // invalidate a vanilla inventory just as it can a mod inventory.
-            const matches = fg.stream(options.patterns, {
+            const matches = iterateFiles(options.patterns, {
                 cwd: root.path,
-                onlyFiles: true,
-                unique: true,
-                dot: false,
-                followSymbolicLinks: false,
                 ignore: options.ignore ?? ['**/.hoi4-agent/**'],
+                ...(options.signal === undefined ? {} : { signal: options.signal }),
             });
+            const relativePaths = [];
             for await (const match of matches) {
                 options.signal?.throwIfAborted();
                 enumeratedFiles += 1;
@@ -115,55 +157,99 @@ export class WorkspaceScanner {
                         limit: maxFiles,
                     });
                 }
-                const relativePath = normalizeRelative(String(match));
+                const relativePath = normalizeRelative(match);
                 if (hiddenByReplacePath(workspace, root, relativePath))
                     continue;
-                const absolutePath = path.join(root.path, relativePath);
-                const cacheKey = this.cacheKey(absolutePath);
-                const handle = await open(absolutePath, 'r');
+                relativePaths.push(relativePath);
+            }
+            for (let offset = 0; offset < relativePaths.length; offset += SCAN_IO_CONCURRENCY) {
+                options.signal?.throwIfAborted();
+                const prepared = await Promise.allSettled(relativePaths
+                    .slice(offset, offset + SCAN_IO_CONCURRENCY)
+                    .map((relativePath) => prepareSource(root, relativePath, options.signal)));
+                const opened = prepared.flatMap((entry) => entry.status === 'fulfilled' ? [entry.value] : []);
                 try {
-                    const metadata = await handle.stat();
-                    if (!metadata.isFile())
-                        continue;
-                    const remaining = maxBytes - totalBytes;
-                    if (metadata.size > remaining || metadata.size > this.serverMaxFileBytes) {
-                        throw new ServiceError('SCAN_BYTE_LIMIT', 'Scan exceeds the configured byte limit', {
-                            file: relativePath,
-                            fileBytes: metadata.size,
-                            bytes: totalBytes,
-                            limit: maxBytes,
-                            perFileLimit: this.serverMaxFileBytes,
-                        });
+                    const failed = prepared.find((entry) => entry.status === 'rejected');
+                    if (failed?.status === 'rejected')
+                        throw failed.reason;
+                    const readable = opened.filter(({ metadata }) => metadata.isFile());
+                    let reserved = 0;
+                    for (const source of readable) {
+                        if (source.metadata.size > maxBytes - totalBytes - reserved ||
+                            source.metadata.size > this.serverMaxFileBytes)
+                            throw new ServiceError('SCAN_BYTE_LIMIT', 'Scan exceeds the configured byte limit', {
+                                file: source.relativePath,
+                                fileBytes: source.metadata.size,
+                                bytes: totalBytes + reserved,
+                                limit: maxBytes,
+                                perFileLimit: this.serverMaxFileBytes,
+                            });
+                        reserved += source.metadata.size;
+                    }
+                    let readOffset = 0;
+                    while (readOffset < readable.length) {
+                        const group = [];
+                        let groupBytes = 0;
+                        while (readOffset < readable.length) {
+                            const source = readable[readOffset];
+                            if (group.length > 0 && groupBytes + source.metadata.size > SCAN_READ_BATCH_BYTES)
+                                break;
+                            group.push(source);
+                            groupBytes += source.metadata.size;
+                            readOffset++;
+                        }
+                        const reads = await Promise.allSettled(group.map(async (source) => {
+                            let bytes;
+                            try {
+                                bytes = await readBoundedFile(source.handle, source.metadata.size, options.signal);
+                            }
+                            catch (error) {
+                                if (error instanceof ServiceError && error.code === 'SCAN_BYTE_LIMIT')
+                                    throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source grew while it was being scanned; retry a focused query', { file: source.relativePath });
+                                throw error;
+                            }
+                            if (bytes.length !== source.metadata.size)
+                                throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source changed while it was being scanned; retry a focused query', { file: source.relativePath });
+                            await verifySource(root, source);
+                            options.signal?.throwIfAborted();
+                            return bytes;
+                        }));
+                        const readFailure = reads.find((entry) => entry.status === 'rejected');
+                        if (readFailure?.status === 'rejected')
+                            throw readFailure.reason;
+                        for (const [index, read] of reads.entries()) {
+                            if (read.status !== 'fulfilled')
+                                continue;
+                            const source = group[index];
+                            const cacheKey = this.cacheKey(source.absolutePath);
+                            const observed = read.value;
+                            const sha256 = sha256Bytes(observed);
+                            const cached = this.#sourceCache.get(cacheKey);
+                            const bytes = cached?.sha256 === sha256 && cached.bytes.equals(observed)
+                                ? cached.bytes
+                                : observed;
+                            this.retainSource(cacheKey, { sha256, bytes });
+                            totalBytes += bytes.length;
+                            result.push({
+                                absolutePath: source.absolutePath,
+                                displayPath: `${rootLabel(root)}:${source.relativePath}`,
+                                relativePath: source.relativePath,
+                                rootKind: root.kind,
+                                loadOrder: root.loadOrder,
+                                size: bytes.length,
+                                modifiedMs: source.metadata.mtimeMs,
+                                sha256,
+                                bytes,
+                            });
+                        }
                     }
-                    const cached = this.#sourceCache.get(cacheKey);
-                    // Metadata is not content identity (including on aliased/networked
-                    // filesystems). Verify bytes before reusing parsed/indexed facts.
-                    const observed = await readBoundedFile(handle, Math.min(remaining, this.serverMaxFileBytes), options.signal);
-                    const sha256 = sha256Bytes(observed);
-                    const bytes = cached?.sha256 === sha256 && cached.bytes.equals(observed) ? cached.bytes : observed;
-                    this.retainSource(cacheKey, {
-                        sha256,
-                        bytes,
-                    });
-                    totalBytes += bytes.length;
-                    const scanned = {
-                        absolutePath,
-                        displayPath: `${rootLabel(root)}:${relativePath}`,
-                        relativePath,
-                        rootKind: root.kind,
-                        loadOrder: root.loadOrder,
-                        size: bytes.length,
-                        modifiedMs: metadata.mtimeMs,
-                        sha256,
-                        bytes,
-                    };
-                    result.push(scanned);
                 }
                 finally {
-                    await handle.close();
+                    await Promise.all(opened.map(({ handle }) => handle.close()));
                 }
             }
         }
+        options.signal?.throwIfAborted();
         result.sort((left, right) => left.loadOrder - right.loadOrder || compareCodeUnits(left.relativePath, right.relativePath));
         const groups = new Map();
         for (const file of result) {

package/dist/hoi4_agent_tools/core/source/cache.js

@@ -51,6 +51,10 @@ export class SourceDocumentCache {
         this.#entries.set(key, encoded);
         return decodedDocument(encoded);
     }
+    /** Availability check without deserialization or cache-hit accounting. */
+    has(key) {
+        return this.#entries.has(key);
+    }
     put(key, document) {
         // Avoid allocating a large serialized copy just to discover that it cannot
         // be retained. This conservative estimate affects caching, never parsing.