hoi4-agent-tools 3.6.0 → 3.7.0

package/dist/hoi4_agent_tools/core/persistent-analysis-cache.js

@@ -54,6 +54,8 @@ export class PersistentAnalysisCache {
     maxBytes;
     maxEntries;
     maxSingleBytes;
+    batchMaxWrites;
+    batchMaxPayloadBytes;
     #lock;
     #hits = 0;
     #misses = 0;
@@ -63,23 +65,30 @@ export class PersistentAnalysisCache {
     #evictions = 0;
     #invalidEntries = 0;
     #oversized = 0;
+    #limitedBatches = 0;
     #invalidAddresses = new Set();
-    constructor(state, root, maxBytes, maxEntries, maxSingleBytes) {
+    constructor(state, root, maxBytes, maxEntries, maxSingleBytes, batchMaxWrites, batchMaxPayloadBytes) {
         this.state = state;
         this.root = root;
         this.maxBytes = maxBytes;
         this.maxEntries = maxEntries;
         this.maxSingleBytes = maxSingleBytes;
+        this.batchMaxWrites = batchMaxWrites;
+        this.batchMaxPayloadBytes = batchMaxPayloadBytes;
         this.#lock = new SharedRequestCapacity(root, 1);
     }
     static async create(state, options = {}) {
         const maxBytes = options.maxBytes ?? 268_435_456;
         const maxEntries = options.maxEntries ?? 50_000;
         const maxSingleBytes = options.maxSingleBytes ?? 16_777_216;
+        const batchMaxWrites = options.batchMaxWrites ?? 16;
+        const batchMaxPayloadBytes = options.batchMaxPayloadBytes ?? 1_048_576;
         for (const [name, value] of [
             ['byte budget', maxBytes],
             ['entry budget', maxEntries],
             ['single-entry budget', maxSingleBytes],
+            ['batch write budget', batchMaxWrites],
+            ['batch payload budget', batchMaxPayloadBytes],
         ]) {
             if (!Number.isSafeInteger(value) || value < 0)
                 throw new RangeError(`Persistent analysis cache ${name} must be a non-negative integer`);
@@ -88,7 +97,7 @@ export class PersistentAnalysisCache {
         const root = await containedGeneratedPath(state.root, 'analysis-cache');
         await mkdir(root, { recursive: true, mode: 0o700 });
         await assertUnlinked(root);
-        return new PersistentAnalysisCache(state, root, maxBytes, maxEntries, maxSingleBytes);
+        return new PersistentAnalysisCache(state, root, maxBytes, maxEntries, maxSingleBytes, batchMaxWrites, batchMaxPayloadBytes);
     }
     async hydrate(scope, files, documents, segments, signal) {
         signal?.throwIfAborted();
@@ -101,7 +110,7 @@ export class PersistentAnalysisCache {
             signal?.throwIfAborted();
             const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
             const documentAddress = sourceAddress(key);
-            if (sourceEntries.has(documentAddress)) {
+            if (!documents.has(key) && sourceEntries.has(documentAddress)) {
                 const payload = await this.read(scopeHash, 'source-document', documentAddress, signal);
                 if (payload !== undefined) {
                     try {
@@ -118,7 +127,7 @@ export class PersistentAnalysisCache {
             }
             for (const provinceDefinition of [false, true]) {
                 const address = indexSegmentAddress(file, provinceDefinition);
-                if (!indexEntries.has(address))
+                if (segments.has(address) || !indexEntries.has(address))
                     continue;
                 const payload = await this.read(scopeHash, 'index-segment', address, signal);
                 if (payload === undefined)
@@ -140,15 +149,34 @@ export class PersistentAnalysisCache {
                 this.inventory(scopeHash, 'source-document', signal),
                 this.inventory(scopeHash, 'index-segment', signal),
             ]);
+            const batch = { writes: 0, payloadBytes: 0, limited: false };
+            const retain = async (kind, address, payload) => {
+                if (batch.writes >= this.batchMaxWrites ||
+                    payload.length > this.batchMaxPayloadBytes - batch.payloadBytes) {
+                    batch.limited = true;
+                    return false;
+                }
+                const written = await this.write(scopeHash, kind, address, payload, signal);
+                if (written) {
+                    batch.writes++;
+                    batch.payloadBytes += payload.length;
+                }
+                return written;
+            };
             for (const file of files) {
                 signal.throwIfAborted();
+                if (batch.writes >= this.batchMaxWrites ||
+                    batch.payloadBytes >= this.batchMaxPayloadBytes) {
+                    batch.limited = true;
+                    break;
+                }
                 const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
                 const documentAddress = sourceAddress(key);
                 const document = documents.exportEncoded(key);
                 if (document !== undefined &&
                     (!sourceEntries.has(documentAddress) ||
                         this.#invalidAddresses.has(`${scopeHash}\0source-document\0${documentAddress}`))) {
-                    if (await this.write(scopeHash, 'source-document', documentAddress, document, signal)) {
+                    if (await retain('source-document', documentAddress, document)) {
                         sourceEntries.add(documentAddress);
                     }
                 }
@@ -159,11 +187,13 @@ export class PersistentAnalysisCache {
                         (indexEntries.has(address) &&
                             !this.#invalidAddresses.has(`${scopeHash}\0index-segment\0${address}`)))
                         continue;
-                    if (await this.write(scopeHash, 'index-segment', address, Buffer.from(segment), signal)) {
+                    if (await retain('index-segment', address, Buffer.from(segment))) {
                         indexEntries.add(address);
                     }
                 }
             }
+            if (batch.limited)
+                this.#limitedBatches++;
             await this.prune(signal);
         });
     }
@@ -177,6 +207,7 @@ export class PersistentAnalysisCache {
             evictions: this.#evictions,
             invalidEntries: this.#invalidEntries,
             oversized: this.#oversized,
+            limitedBatches: this.#limitedBatches,
         };
     }
     async directory(scopeHash, kind) {

package/dist/hoi4_agent_tools/core/scanner.js

@@ -1,9 +1,53 @@
-import { open } from 'node:fs/promises';
+import { open, realpath, stat } from 'node:fs/promises';
 import path from 'node:path';
-import fg from 'fast-glob';
+import { iterateFiles } from './file-glob.js';
 import { compareCodeUnits, sha256Bytes } from './canonical.js';
 import { DEFAULT_SCAN_MAX_BYTES, DEFAULT_SCAN_MAX_FILE_BYTES, DEFAULT_SCAN_MAX_FILES, } from './configuration.js';
 import { ServiceError } from './result.js';
+import { isWithin } from './workspace.js';
+const SCAN_IO_CONCURRENCY = 8;
+const SCAN_READ_BATCH_BYTES = 16_777_216;
+export async function verifySource(root, source) {
+    const resolved = await realpath(source.absolutePath);
+    if (!isWithin(root.path, resolved))
+        throw new ServiceError('SCAN_ROOT_ESCAPE', 'Source path resolves outside its authorized root', {
+            file: source.relativePath,
+        });
+    const current = await stat(resolved);
+    const expected = source.metadata;
+    if (current.dev !== expected.dev ||
+        current.ino !== expected.ino ||
+        current.size !== expected.size ||
+        current.mtimeMs !== expected.mtimeMs ||
+        current.ctimeMs !== expected.ctimeMs)
+        throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source changed while it was being scanned; retry a focused query', { file: source.relativePath });
+}
+export async function prepareSource(root, relativePath, signal) {
+    signal?.throwIfAborted();
+    const absolutePath = path.join(root.path, relativePath);
+    const canonicalPath = await realpath(absolutePath);
+    if (!isWithin(root.path, canonicalPath))
+        throw new ServiceError('SCAN_ROOT_ESCAPE', 'Source path resolves outside its authorized root', {
+            file: relativePath,
+        });
+    const handle = await open(canonicalPath, 'r');
+    try {
+        const source = {
+            relativePath,
+            absolutePath,
+            canonicalPath,
+            handle,
+            metadata: await handle.stat(),
+        };
+        await verifySource(root, source);
+        signal?.throwIfAborted();
+        return source;
+    }
+    catch (error) {
+        await handle.close();
+        throw error;
+    }
+}
 function normalizeRelative(value) {
     return value.replaceAll('\\', '/').replace(/^\.\//u, '');
 }
@@ -98,14 +142,12 @@ export class WorkspaceScanner {
             options.signal?.throwIfAborted();
             // Re-enumerate every root: installing content or changing load order can
             // invalidate a vanilla inventory just as it can a mod inventory.
-            const matches = fg.stream(options.patterns, {
+            const matches = iterateFiles(options.patterns, {
                 cwd: root.path,
-                onlyFiles: true,
-                unique: true,
-                dot: false,
-                followSymbolicLinks: false,
                 ignore: options.ignore ?? ['**/.hoi4-agent/**'],
+                ...(options.signal === undefined ? {} : { signal: options.signal }),
             });
+            const relativePaths = [];
             for await (const match of matches) {
                 options.signal?.throwIfAborted();
                 enumeratedFiles += 1;
@@ -115,55 +157,99 @@ export class WorkspaceScanner {
                         limit: maxFiles,
                     });
                 }
-                const relativePath = normalizeRelative(String(match));
+                const relativePath = normalizeRelative(match);
                 if (hiddenByReplacePath(workspace, root, relativePath))
                     continue;
-                const absolutePath = path.join(root.path, relativePath);
-                const cacheKey = this.cacheKey(absolutePath);
-                const handle = await open(absolutePath, 'r');
+                relativePaths.push(relativePath);
+            }
+            for (let offset = 0; offset < relativePaths.length; offset += SCAN_IO_CONCURRENCY) {
+                options.signal?.throwIfAborted();
+                const prepared = await Promise.allSettled(relativePaths
+                    .slice(offset, offset + SCAN_IO_CONCURRENCY)
+                    .map((relativePath) => prepareSource(root, relativePath, options.signal)));
+                const opened = prepared.flatMap((entry) => entry.status === 'fulfilled' ? [entry.value] : []);
                 try {
-                    const metadata = await handle.stat();
-                    if (!metadata.isFile())
-                        continue;
-                    const remaining = maxBytes - totalBytes;
-                    if (metadata.size > remaining || metadata.size > this.serverMaxFileBytes) {
-                        throw new ServiceError('SCAN_BYTE_LIMIT', 'Scan exceeds the configured byte limit', {
-                            file: relativePath,
-                            fileBytes: metadata.size,
-                            bytes: totalBytes,
-                            limit: maxBytes,
-                            perFileLimit: this.serverMaxFileBytes,
-                        });
+                    const failed = prepared.find((entry) => entry.status === 'rejected');
+                    if (failed?.status === 'rejected')
+                        throw failed.reason;
+                    const readable = opened.filter(({ metadata }) => metadata.isFile());
+                    let reserved = 0;
+                    for (const source of readable) {
+                        if (source.metadata.size > maxBytes - totalBytes - reserved ||
+                            source.metadata.size > this.serverMaxFileBytes)
+                            throw new ServiceError('SCAN_BYTE_LIMIT', 'Scan exceeds the configured byte limit', {
+                                file: source.relativePath,
+                                fileBytes: source.metadata.size,
+                                bytes: totalBytes + reserved,
+                                limit: maxBytes,
+                                perFileLimit: this.serverMaxFileBytes,
+                            });
+                        reserved += source.metadata.size;
+                    }
+                    let readOffset = 0;
+                    while (readOffset < readable.length) {
+                        const group = [];
+                        let groupBytes = 0;
+                        while (readOffset < readable.length) {
+                            const source = readable[readOffset];
+                            if (group.length > 0 && groupBytes + source.metadata.size > SCAN_READ_BATCH_BYTES)
+                                break;
+                            group.push(source);
+                            groupBytes += source.metadata.size;
+                            readOffset++;
+                        }
+                        const reads = await Promise.allSettled(group.map(async (source) => {
+                            let bytes;
+                            try {
+                                bytes = await readBoundedFile(source.handle, source.metadata.size, options.signal);
+                            }
+                            catch (error) {
+                                if (error instanceof ServiceError && error.code === 'SCAN_BYTE_LIMIT')
+                                    throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source grew while it was being scanned; retry a focused query', { file: source.relativePath });
+                                throw error;
+                            }
+                            if (bytes.length !== source.metadata.size)
+                                throw new ServiceError('SCAN_SOURCE_CHANGED', 'Source changed while it was being scanned; retry a focused query', { file: source.relativePath });
+                            await verifySource(root, source);
+                            options.signal?.throwIfAborted();
+                            return bytes;
+                        }));
+                        const readFailure = reads.find((entry) => entry.status === 'rejected');
+                        if (readFailure?.status === 'rejected')
+                            throw readFailure.reason;
+                        for (const [index, read] of reads.entries()) {
+                            if (read.status !== 'fulfilled')
+                                continue;
+                            const source = group[index];
+                            const cacheKey = this.cacheKey(source.absolutePath);
+                            const observed = read.value;
+                            const sha256 = sha256Bytes(observed);
+                            const cached = this.#sourceCache.get(cacheKey);
+                            const bytes = cached?.sha256 === sha256 && cached.bytes.equals(observed)
+                                ? cached.bytes
+                                : observed;
+                            this.retainSource(cacheKey, { sha256, bytes });
+                            totalBytes += bytes.length;
+                            result.push({
+                                absolutePath: source.absolutePath,
+                                displayPath: `${rootLabel(root)}:${source.relativePath}`,
+                                relativePath: source.relativePath,
+                                rootKind: root.kind,
+                                loadOrder: root.loadOrder,
+                                size: bytes.length,
+                                modifiedMs: source.metadata.mtimeMs,
+                                sha256,
+                                bytes,
+                            });
+                        }
                     }
-                    const cached = this.#sourceCache.get(cacheKey);
-                    // Metadata is not content identity (including on aliased/networked
-                    // filesystems). Verify bytes before reusing parsed/indexed facts.
-                    const observed = await readBoundedFile(handle, Math.min(remaining, this.serverMaxFileBytes), options.signal);
-                    const sha256 = sha256Bytes(observed);
-                    const bytes = cached?.sha256 === sha256 && cached.bytes.equals(observed) ? cached.bytes : observed;
-                    this.retainSource(cacheKey, {
-                        sha256,
-                        bytes,
-                    });
-                    totalBytes += bytes.length;
-                    const scanned = {
-                        absolutePath,
-                        displayPath: `${rootLabel(root)}:${relativePath}`,
-                        relativePath,
-                        rootKind: root.kind,
-                        loadOrder: root.loadOrder,
-                        size: bytes.length,
-                        modifiedMs: metadata.mtimeMs,
-                        sha256,
-                        bytes,
-                    };
-                    result.push(scanned);
                 }
                 finally {
-                    await handle.close();
+                    await Promise.all(opened.map(({ handle }) => handle.close()));
                 }
             }
         }
+        options.signal?.throwIfAborted();
         result.sort((left, right) => left.loadOrder - right.loadOrder || compareCodeUnits(left.relativePath, right.relativePath));
         const groups = new Map();
         for (const file of result) {

package/dist/hoi4_agent_tools/core/source/cache.js

@@ -51,6 +51,10 @@ export class SourceDocumentCache {
         this.#entries.set(key, encoded);
         return decodedDocument(encoded);
     }
+    /** Availability check without deserialization or cache-hit accounting. */
+    has(key) {
+        return this.#entries.has(key);
+    }
     put(key, document) {
         // Avoid allocating a large serialized copy just to discover that it cannot
         // be retained. This conservative estimate affects caching, never parsing.

package/dist/hoi4_agent_tools/map/service.js

@@ -1,5 +1,5 @@
 import path from 'node:path';
-import fg from 'fast-glob';
+import { escapeFileGlob } from '../core/file-glob.js';
 import { ArtifactStore, boundedSourceHashEvidence, publicArtifactLink, } from '../core/artifacts.js';
 import { canonicalJson, compareCodeUnits, hashCanonical, sha256Bytes } from '../core/canonical.js';
 import { sortDiagnostics } from '../core/diagnostics.js';
@@ -426,7 +426,7 @@ function provinceBitmapPatterns(defaultMaps, mapRoots) {
             const selectedPath = path.posix.normalize(path.posix.join(normalizedRoot, normalizedName));
             if (selectedPath !== normalizedRoot && !selectedPath.startsWith(`${normalizedRoot}/`))
                 continue;
-            patterns.add(fg.posix.escapePath(selectedPath));
+            patterns.add(escapeFileGlob(selectedPath));
         }
     }
     return [...patterns].sort((left, right) => compareCodeUnits(left, right));

package/dist/hoi4_agent_tools/mcp/tools/reference.js

@@ -1,6 +1,7 @@
 import { errorResult, strictOperationResultSchema } from '../../core/operation-result.js';
 import { ReferenceToolService } from '../../reference/operations.js';
 import { referenceContextRequestSchema, referenceReadRequestSchema, referenceSearchRequestSchema, sourceLookupRequestSchema, referenceContextDataSchema, referenceReadDataSchema, referenceSearchDataSchema, sourceLookupDataSchema, } from '../../schemas/reference.js';
+import { scriptValidateDataSchema, scriptValidateRequestSchema, } from '../../schemas/script-validation.js';
 const readOnly = {
     readOnlyHint: true,
     destructiveHint: false,
@@ -35,11 +36,19 @@ export const referenceTools = [
     {
         name: 'hoi4.source_lookup',
         title: 'Look up HOI4 source symbol',
-        description: 'Find exact Clausewitz definitions, override status, a bounded source block, and indexed usages in the configured game and mod.',
+        description: 'Find exact Clausewitz definitions and usages; navigate nested keyPath blocks or list bounded child structure with view=structure.',
         inputSchema: sourceLookupRequestSchema,
         outputSchema: strictOperationResultSchema(sourceLookupDataSchema),
         annotations: readOnly,
     },
+    {
+        name: 'hoi4.script_validate',
+        title: 'Check HOI4 script commands and scopes',
+        description: 'Check effect or trigger snippet syntax, native command kinds and declared scopes against cited installed documentation. Unknown helpers and native argument blocks remain explicit.',
+        inputSchema: scriptValidateRequestSchema,
+        outputSchema: strictOperationResultSchema(scriptValidateDataSchema),
+        annotations: readOnly,
+    },
 ];
 export function registerReferenceTools(server, engine, context) {
     const service = new ReferenceToolService(engine);

package/dist/hoi4_agent_tools/reference/lines.js

@@ -50,3 +50,15 @@ export function readBoundedLines(lines, firstLine, startLine, startColumn, maxLi
         nextColumn: lineIndex < lines.length ? columnIndex + 1 : null,
     };
 }
+/** Preserve continuation while bounding JSON escaping as well as raw UTF-8 text. */
+export function readBoundedJsonLines(lines, firstLine, startLine, startColumn, maxLines, maxBytes, maxJsonBytes) {
+    let budget = maxBytes;
+    for (;;) {
+        const result = readBoundedLines(lines, firstLine, startLine, startColumn, maxLines, budget);
+        if (Buffer.byteLength(JSON.stringify(result.text), 'utf8') <= maxJsonBytes)
+            return result;
+        budget = Math.max(4, Math.floor(budget / 2));
+        if (budget === 4 && maxJsonBytes < 26)
+            throw new RangeError('JSON text budget cannot retain a source character');
+    }
+}

package/dist/hoi4_agent_tools/reference/operations.js

@@ -4,11 +4,14 @@ import { toolResult } from '../core/operation-result.js';
 import { referenceContextRequestSchema, referenceReadRequestSchema, referenceSearchRequestSchema, sourceLookupRequestSchema, } from '../schemas/reference.js';
 import { ReferenceService } from './service.js';
 import { sourceLookup } from './source-lookup.js';
+import { scriptValidateDataSchema, scriptValidateRequestSchema, } from '../schemas/script-validation.js';
+import { validateScript } from './script-validation.js';
 const schemas = {
     'hoi4.reference_search': referenceSearchRequestSchema,
     'hoi4.reference_read': referenceReadRequestSchema,
     'hoi4.reference_context': referenceContextRequestSchema,
     'hoi4.source_lookup': sourceLookupRequestSchema,
+    'hoi4.script_validate': scriptValidateRequestSchema,
 };
 export function isReferenceTool(name) {
     return Object.hasOwn(schemas, name);
@@ -35,7 +38,47 @@ export class ReferenceToolService {
                 ? await this.references.read(workspace, referenceReadRequestSchema.parse(parsed), signal)
                 : name === 'hoi4.reference_context'
                     ? await this.references.context(workspace, referenceContextRequestSchema.parse(parsed), signal)
-                    : await sourceLookup(this.engine, workspaceId, sourceLookupRequestSchema.parse(parsed), context.principal, signal);
-        return toolResult(emptyServiceResult(workspaceId, data));
+                    : name === 'hoi4.script_validate'
+                        ? await validateScript(this.references, workspace, scriptValidateRequestSchema.parse(parsed), signal)
+                        : await sourceLookup(this.engine, workspaceId, sourceLookupRequestSchema.parse(parsed), context.principal, signal);
+        const result = emptyServiceResult(workspaceId, data);
+        if (name === 'hoi4.script_validate') {
+            const checked = scriptValidateDataSchema.parse(data);
+            result.code = checked.valid === null ? 'SCRIPT_CHECK_PARTIAL' : 'SCRIPT_CHECKED';
+            result.validation = {
+                passed: checked.valid === true,
+                checks: [
+                    {
+                        id: 'script-command-checks',
+                        passed: checked.valid === true,
+                        message: checked.valid === true
+                            ? 'Syntax, native command kinds and declared scopes match the selected documentation'
+                            : checked.valid === false
+                                ? 'Script checks found an error'
+                                : 'Script checks have unresolved coverage',
+                    },
+                ],
+            };
+            result.diagnostics = checked.findings
+                .filter(({ status }) => status !== 'supported')
+                .slice(0, 20)
+                .map((finding) => ({
+                code: finding.code,
+                severity: finding.status === 'error' ? 'error' : 'warning',
+                category: 'validation',
+                message: finding.message,
+                details: {
+                    command: finding.command,
+                    kind: finding.kind,
+                    scope: finding.scope,
+                    line: finding.line,
+                    column: finding.column,
+                },
+            }));
+            while (Buffer.byteLength(JSON.stringify(result.diagnostics), 'utf8') > 4096 &&
+                result.diagnostics.length > 1)
+                result.diagnostics.pop();
+        }
+        return toolResult(result);
     }
 }