hoi4-agent-tools 3.0.9 → 3.2.0

package/dist/hoi4_agent_tools/core/persistent-analysis-cache.js

@@ -0,0 +1,383 @@
+import { link, lstat, mkdir, open, opendir, readdir, unlink } from 'node:fs/promises';
+import path from 'node:path';
+import { z } from 'zod/v4';
+import { canonicalJson, hashCanonical, secureId, sha256Bytes } from './canonical.js';
+import { indexSegmentAddress } from './index-segments.js';
+import { SharedRequestCapacity } from './shared-request-capacity.js';
+import { sourceDocumentCacheKey } from './source/cache.js';
+import { containedGeneratedPath } from './workspace.js';
+import { PACKAGE_VERSION } from '../version.js';
+const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u);
+const entryKindSchema = z.enum(['source-document', 'index-segment']);
+const unsignedEntrySchema = z
+    .object({
+    version: z.literal(1),
+    kind: entryKindSchema,
+    scopeHash: digestSchema,
+    address: digestSchema,
+    toolVersion: z.literal(PACKAGE_VERSION),
+    runtime: z.string().min(1),
+    payloadHash: digestSchema,
+    payload: z.string(),
+})
+    .strict();
+const entrySchema = unsignedEntrySchema.extend({ authenticationTag: digestSchema }).strict();
+const entryFilePattern = /^(?<address>[a-f0-9]{64})\.json$/u;
+function sourceAddress(key) {
+    return hashCanonical({
+        version: 'persistent-source-document.v1',
+        toolVersion: PACKAGE_VERSION,
+        runtime: `${process.versions.node}\0${process.versions.v8}`,
+        key,
+    });
+}
+function cacheScopeHash(scope) {
+    return hashCanonical({ version: 'analysis-cache-scope.v1', ...scope });
+}
+async function assertUnlinked(candidate) {
+    try {
+        if ((await lstat(candidate)).isSymbolicLink())
+            throw new Error('Persistent analysis cache cannot use symbolic links or junctions');
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT')
+            throw error;
+    }
+}
+/**
+ * Internal same-version cache shared by authorized local processes.
+ * Entries are immutable, authenticated, scope-separated, and never public artifacts.
+ */
+export class PersistentAnalysisCache {
+    state;
+    root;
+    maxBytes;
+    maxEntries;
+    maxSingleBytes;
+    #lock;
+    #hits = 0;
+    #misses = 0;
+    #writes = 0;
+    #retainedEntries = 0;
+    #retainedBytes = 0;
+    #evictions = 0;
+    #invalidEntries = 0;
+    #oversized = 0;
+    #invalidAddresses = new Set();
+    constructor(state, root, maxBytes, maxEntries, maxSingleBytes) {
+        this.state = state;
+        this.root = root;
+        this.maxBytes = maxBytes;
+        this.maxEntries = maxEntries;
+        this.maxSingleBytes = maxSingleBytes;
+        this.#lock = new SharedRequestCapacity(root, 1);
+    }
+    static async create(state, options = {}) {
+        const maxBytes = options.maxBytes ?? 268_435_456;
+        const maxEntries = options.maxEntries ?? 50_000;
+        const maxSingleBytes = options.maxSingleBytes ?? 16_777_216;
+        for (const [name, value] of [
+            ['byte budget', maxBytes],
+            ['entry budget', maxEntries],
+            ['single-entry budget', maxSingleBytes],
+        ]) {
+            if (!Number.isSafeInteger(value) || value < 0)
+                throw new RangeError(`Persistent analysis cache ${name} must be a non-negative integer`);
+        }
+        await assertUnlinked(path.join(state.root, 'analysis-cache'));
+        const root = await containedGeneratedPath(state.root, 'analysis-cache');
+        await mkdir(root, { recursive: true, mode: 0o700 });
+        await assertUnlinked(root);
+        return new PersistentAnalysisCache(state, root, maxBytes, maxEntries, maxSingleBytes);
+    }
+    async hydrate(scope, files, documents, segments, signal) {
+        signal?.throwIfAborted();
+        const scopeHash = cacheScopeHash(scope);
+        const [sourceEntries, indexEntries] = await Promise.all([
+            this.inventory(scopeHash, 'source-document', signal),
+            this.inventory(scopeHash, 'index-segment', signal),
+        ]);
+        for (const file of files) {
+            signal?.throwIfAborted();
+            const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
+            const documentAddress = sourceAddress(key);
+            if (sourceEntries.has(documentAddress)) {
+                const payload = await this.read(scopeHash, 'source-document', documentAddress, signal);
+                if (payload !== undefined) {
+                    try {
+                        documents.importEncoded(key, payload, {
+                            bytes: file.bytes,
+                            sourcePath: file.displayPath,
+                        });
+                    }
+                    catch {
+                        this.#invalidEntries += 1;
+                        this.#invalidAddresses.add(`${scopeHash}\0source-document\0${documentAddress}`);
+                    }
+                }
+            }
+            for (const provinceDefinition of [false, true]) {
+                const address = indexSegmentAddress(file, provinceDefinition);
+                if (!indexEntries.has(address))
+                    continue;
+                const payload = await this.read(scopeHash, 'index-segment', address, signal);
+                if (payload === undefined)
+                    continue;
+                try {
+                    segments.importEncoded(address, payload.toString('utf8'));
+                }
+                catch {
+                    this.#invalidEntries += 1;
+                    this.#invalidAddresses.add(`${scopeHash}\0index-segment\0${address}`);
+                }
+            }
+        }
+    }
+    async persist(scope, files, documents, segments, signal = new AbortController().signal) {
+        const scopeHash = cacheScopeHash(scope);
+        await this.#lock.run(signal, async () => {
+            const [sourceEntries, indexEntries] = await Promise.all([
+                this.inventory(scopeHash, 'source-document', signal),
+                this.inventory(scopeHash, 'index-segment', signal),
+            ]);
+            for (const file of files) {
+                signal.throwIfAborted();
+                const key = sourceDocumentCacheKey(file.bytes, file.displayPath);
+                const documentAddress = sourceAddress(key);
+                const document = documents.exportEncoded(key);
+                if (document !== undefined &&
+                    (!sourceEntries.has(documentAddress) ||
+                        this.#invalidAddresses.has(`${scopeHash}\0source-document\0${documentAddress}`))) {
+                    if (await this.write(scopeHash, 'source-document', documentAddress, document, signal)) {
+                        sourceEntries.add(documentAddress);
+                    }
+                }
+                for (const provinceDefinition of [false, true]) {
+                    const address = indexSegmentAddress(file, provinceDefinition);
+                    const segment = segments.exportEncoded(address);
+                    if (segment === undefined ||
+                        (indexEntries.has(address) &&
+                            !this.#invalidAddresses.has(`${scopeHash}\0index-segment\0${address}`)))
+                        continue;
+                    if (await this.write(scopeHash, 'index-segment', address, Buffer.from(segment), signal)) {
+                        indexEntries.add(address);
+                    }
+                }
+            }
+            await this.prune(signal);
+        });
+    }
+    statistics() {
+        return {
+            hits: this.#hits,
+            misses: this.#misses,
+            writes: this.#writes,
+            retainedEntries: this.#retainedEntries,
+            retainedBytes: this.#retainedBytes,
+            evictions: this.#evictions,
+            invalidEntries: this.#invalidEntries,
+            oversized: this.#oversized,
+        };
+    }
+    async directory(scopeHash, kind) {
+        await assertUnlinked(this.root);
+        await assertUnlinked(path.join(this.root, scopeHash));
+        await assertUnlinked(path.join(this.root, scopeHash, kind));
+        return containedGeneratedPath(this.root, scopeHash, kind);
+    }
+    async inventory(scopeHash, kind, signal) {
+        const directory = await this.directory(scopeHash, kind);
+        const entries = new Set();
+        let opened;
+        try {
+            opened = await opendir(directory);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return entries;
+            throw error;
+        }
+        for await (const entry of opened) {
+            signal?.throwIfAborted();
+            const match = entryFilePattern.exec(entry.name);
+            if (entry.isFile() && !entry.isSymbolicLink() && match?.groups?.address !== undefined) {
+                entries.add(match.groups.address);
+            }
+        }
+        return entries;
+    }
+    async read(scopeHash, kind, address, signal) {
+        signal?.throwIfAborted();
+        const directory = await this.directory(scopeHash, kind);
+        const file = await containedGeneratedPath(directory, `${digestSchema.parse(address)}.json`);
+        const invalidAddress = `${scopeHash}\0${kind}\0${address}`;
+        let handle;
+        try {
+            await assertUnlinked(file);
+            handle = await open(file, 'r');
+            const metadata = await handle.stat();
+            if (!metadata.isFile() || metadata.size > Math.ceil((this.maxSingleBytes * 4) / 3) + 4096)
+                throw new Error('Persistent analysis cache entry exceeds its storage budget');
+            const parsed = entrySchema.parse(JSON.parse(await handle.readFile('utf8')));
+            const { authenticationTag, ...entry } = parsed;
+            if (entry.scopeHash !== scopeHash ||
+                entry.kind !== kind ||
+                entry.address !== address ||
+                !this.state.verifyJournal({ kind: 'analysis-cache-entry.v1', entry }, authenticationTag))
+                throw new Error('Persistent analysis cache authentication or identity is invalid');
+            const payload = Buffer.from(entry.payload, 'base64');
+            if (payload.length > this.maxSingleBytes || sha256Bytes(payload) !== entry.payloadHash)
+                throw new Error('Persistent analysis cache payload is invalid');
+            this.#hits += 1;
+            this.#invalidAddresses.delete(invalidAddress);
+            return payload;
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                this.#misses += 1;
+                return undefined;
+            }
+            this.#invalidEntries += 1;
+            this.#invalidAddresses.add(invalidAddress);
+            return undefined;
+        }
+        finally {
+            await handle?.close();
+        }
+    }
+    async write(scopeHash, kind, address, payload, signal) {
+        if (this.maxBytes === 0 ||
+            this.maxEntries === 0 ||
+            this.maxSingleBytes === 0 ||
+            payload.length > this.maxSingleBytes) {
+            this.#oversized += 1;
+            return false;
+        }
+        const directory = await this.directory(scopeHash, kind);
+        await mkdir(directory, { recursive: true, mode: 0o700 });
+        const target = await containedGeneratedPath(directory, `${digestSchema.parse(address)}.json`);
+        const invalidAddress = `${scopeHash}\0${kind}\0${address}`;
+        const unsigned = {
+            version: 1,
+            kind,
+            scopeHash,
+            address,
+            toolVersion: PACKAGE_VERSION,
+            runtime: `${process.versions.node}\0${process.versions.v8}`,
+            payloadHash: sha256Bytes(payload),
+            payload: payload.toString('base64'),
+        };
+        const bytes = Buffer.from(canonicalJson({
+            ...unsigned,
+            authenticationTag: this.state.authenticateJournal({
+                kind: 'analysis-cache-entry.v1',
+                entry: unsigned,
+            }),
+        }));
+        if (bytes.length > Math.ceil((this.maxSingleBytes * 4) / 3) + 4096) {
+            this.#oversized += 1;
+            return false;
+        }
+        const repair = this.#invalidAddresses.has(invalidAddress);
+        const existing = repair ? undefined : await this.read(scopeHash, kind, address, signal);
+        if (existing !== undefined) {
+            if (!existing.equals(payload))
+                this.#invalidEntries += 1;
+            return false;
+        }
+        if (repair || this.#invalidAddresses.has(invalidAddress)) {
+            await unlink(target).catch((error) => {
+                if (error.code !== 'ENOENT')
+                    throw error;
+            });
+        }
+        await assertUnlinked(target);
+        const temporary = await containedGeneratedPath(directory, `.${secureId('analysis')}.tmp`);
+        let handle;
+        try {
+            handle = await open(temporary, 'wx', 0o600);
+            await handle.writeFile(bytes);
+            await handle.sync();
+            await handle.close();
+            handle = undefined;
+            signal.throwIfAborted();
+            try {
+                await link(temporary, target);
+                this.#writes += 1;
+                this.#invalidAddresses.delete(invalidAddress);
+                return true;
+            }
+            catch (error) {
+                if (error.code !== 'EEXIST')
+                    throw error;
+                const winner = await this.read(scopeHash, kind, address, signal);
+                if (winner?.equals(payload) === true)
+                    return false;
+                this.#invalidEntries += 1;
+                return false;
+            }
+        }
+        finally {
+            await handle?.close();
+            await unlink(temporary).catch((error) => {
+                if (error.code !== 'ENOENT')
+                    throw error;
+            });
+        }
+    }
+    async prune(signal) {
+        const retained = [];
+        let scopes;
+        try {
+            scopes = await readdir(this.root);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return;
+            throw error;
+        }
+        for (const scopeHash of scopes) {
+            if (!digestSchema.safeParse(scopeHash).success)
+                continue;
+            for (const kind of entryKindSchema.options) {
+                const directory = await this.directory(scopeHash, kind);
+                let opened;
+                try {
+                    opened = await opendir(directory);
+                }
+                catch (error) {
+                    if (error.code === 'ENOENT')
+                        continue;
+                    throw error;
+                }
+                for await (const entry of opened) {
+                    signal.throwIfAborted();
+                    if (!entry.isFile() || entry.isSymbolicLink() || !entryFilePattern.test(entry.name))
+                        continue;
+                    const file = await containedGeneratedPath(directory, entry.name);
+                    const metadata = await lstat(file);
+                    retained.push({ file, bytes: metadata.size, modifiedMs: metadata.mtimeMs });
+                }
+            }
+        }
+        retained.sort((left, right) => left.modifiedMs - right.modifiedMs || left.file.localeCompare(right.file, 'en-US'));
+        let retainedBytes = retained.reduce((total, entry) => total + entry.bytes, 0);
+        let retainedEntries = retained.length;
+        for (const entry of retained) {
+            if (retainedBytes <= this.maxBytes && retainedEntries <= this.maxEntries)
+                break;
+            try {
+                await unlink(entry.file);
+                retainedBytes -= entry.bytes;
+                retainedEntries -= 1;
+                this.#evictions += 1;
+            }
+            catch (error) {
+                if (!['ENOENT', 'EPERM', 'EBUSY', 'EACCES'].includes(error.code ?? ''))
+                    throw error;
+            }
+        }
+        this.#retainedBytes = retainedBytes;
+        this.#retainedEntries = retainedEntries;
+    }
+}

package/dist/hoi4_agent_tools/core/result.js

@@ -1,3 +1,4 @@
+export const MAX_INLINE_ARTIFACT_LINKS = 32;
 export function emptyServiceResult(workspaceId, data) {
     return {
         status: 'ok',

package/dist/hoi4_agent_tools/core/scanner.js

@@ -1,7 +1,7 @@
 import { open } from 'node:fs/promises';
 import path from 'node:path';
 import fg from 'fast-glob';
-import { compareCodeUnits, hashCanonical, sha256Bytes } from './canonical.js';
+import { compareCodeUnits, sha256Bytes } from './canonical.js';
 import { DEFAULT_SCAN_MAX_BYTES, DEFAULT_SCAN_MAX_FILE_BYTES, DEFAULT_SCAN_MAX_FILES, } from './configuration.js';
 import { ServiceError } from './result.js';
 function normalizeRelative(value) {
@@ -27,17 +27,13 @@ export class WorkspaceScanner {
     serverMaxBytes;
     serverMaxFileBytes;
     #sourceCache = new Map();
-    #gameScanCache = new Map();
     #sourceCacheBytes = 0;
-    #gameScanCacheBytes = 0;
     #sourceCacheMaxBytes;
-    #gameScanCacheMaxBytes;
     constructor(serverMaxFiles = DEFAULT_SCAN_MAX_FILES, serverMaxBytes = DEFAULT_SCAN_MAX_BYTES, serverMaxFileBytes = DEFAULT_SCAN_MAX_FILE_BYTES) {
         this.serverMaxFiles = serverMaxFiles;
         this.serverMaxBytes = serverMaxBytes;
         this.serverMaxFileBytes = serverMaxFileBytes;
         this.#sourceCacheMaxBytes = Math.max(1, Math.min(134_217_728, serverMaxBytes));
-        this.#gameScanCacheMaxBytes = Math.min(134_217_728, serverMaxBytes);
     }
     cacheKey(absolutePath) {
         const resolved = path.resolve(absolutePath);
@@ -48,9 +44,11 @@ export class WorkspaceScanner {
         if (previous !== undefined)
             this.#sourceCacheBytes -= previous.bytes.length;
         this.#sourceCache.delete(key);
+        if (source.bytes.length > this.#sourceCacheMaxBytes)
+            return;
         this.#sourceCache.set(key, source);
         this.#sourceCacheBytes += source.bytes.length;
-        while (this.#sourceCacheBytes > this.#sourceCacheMaxBytes && this.#sourceCache.size > 1) {
+        while (this.#sourceCacheBytes > this.#sourceCacheMaxBytes || this.#sourceCache.size > 16_384) {
             const oldest = this.#sourceCache.entries().next().value;
             if (oldest === undefined)
                 break;
@@ -58,46 +56,20 @@ export class WorkspaceScanner {
             this.#sourceCacheBytes -= oldest[1].bytes.length;
         }
     }
-    gameScanKey(workspace, root, options) {
-        return hashCanonical({
-            workspaceId: workspace.id,
-            root: root.path,
-            patterns: [...options.patterns].sort(compareCodeUnits),
-            ignore: [...(options.ignore ?? ['**/.hoi4-agent/**'])].sort(compareCodeUnits),
-        });
-    }
-    retainGameScan(key, files) {
-        const bytes = files.reduce((total, file) => total + file.size, 0);
-        const previous = this.#gameScanCache.get(key);
-        if (previous !== undefined)
-            this.#gameScanCacheBytes -= previous.bytes;
-        this.#gameScanCache.delete(key);
-        if (bytes > this.#gameScanCacheMaxBytes)
-            return;
-        this.#gameScanCache.set(key, { files: files.map((file) => ({ ...file })), bytes });
-        this.#gameScanCacheBytes += bytes;
-        while (this.#gameScanCache.size > 8 || this.#gameScanCacheBytes > this.#gameScanCacheMaxBytes) {
-            const oldest = this.#gameScanCache.entries().next().value;
-            if (oldest === undefined)
-                break;
-            this.#gameScanCache.delete(oldest[0]);
-            this.#gameScanCacheBytes -= oldest[1].bytes;
-        }
-    }
     /** Release retained source buffers after an idle MCP analysis batch. */
     clearCaches() {
         this.#sourceCache.clear();
         this.#sourceCacheBytes = 0;
-        this.#gameScanCache.clear();
-        this.#gameScanCacheBytes = 0;
     }
     /** Cache accounting used by lifecycle and regression tests. */
     cacheUsage() {
         return {
             sourceBytes: this.#sourceCacheBytes,
             sourceEntries: this.#sourceCache.size,
-            gameScanBytes: this.#gameScanCacheBytes,
-            gameScanEntries: this.#gameScanCache.size,
+            // Retained for callers of the existing accounting interface. Inventories
+            // are no longer cached independently of verified source bytes.
+            gameScanBytes: 0,
+            gameScanEntries: 0,
         };
     }
     async scan(workspace, options) {
@@ -124,29 +96,8 @@ export class WorkspaceScanner {
         let enumeratedFiles = 0;
         for (const root of roots) {
             options.signal?.throwIfAborted();
-            const gameScanKey = root.kind === 'game' ? this.gameScanKey(workspace, root, options) : undefined;
-            const cachedGameFiles = gameScanKey === undefined ? undefined : this.#gameScanCache.get(gameScanKey);
-            if (cachedGameFiles !== undefined && gameScanKey !== undefined) {
-                this.#gameScanCache.delete(gameScanKey);
-                this.#gameScanCache.set(gameScanKey, cachedGameFiles);
-                enumeratedFiles += cachedGameFiles.files.length;
-                if (enumeratedFiles > maxFiles)
-                    throw new ServiceError('SCAN_FILE_LIMIT', 'Scan exceeds the configured file limit', {
-                        files: enumeratedFiles,
-                        limit: maxFiles,
-                    });
-                for (const file of cachedGameFiles.files) {
-                    totalBytes += file.size;
-                    if (totalBytes > maxBytes)
-                        throw new ServiceError('SCAN_BYTE_LIMIT', 'Scan exceeds the configured byte limit', {
-                            bytes: totalBytes,
-                            limit: maxBytes,
-                        });
-                    result.push({ ...file });
-                }
-                continue;
-            }
-            const rootFiles = [];
+            // Re-enumerate every root: installing content or changing load order can
+            // invalidate a vanilla inventory just as it can a mod inventory.
             const matches = fg.stream(options.patterns, {
                 cwd: root.path,
                 onlyFiles: true,
@@ -185,20 +136,12 @@ export class WorkspaceScanner {
                         });
                     }
                     const cached = this.#sourceCache.get(cacheKey);
-                    const retained = cached?.size === metadata.size &&
-                        cached.modifiedMs === metadata.mtimeMs &&
-                        cached.changedMs === metadata.ctimeMs &&
-                        cached.inode === metadata.ino
-                        ? cached
-                        : undefined;
-                    const bytes = retained?.bytes ??
-                        (await readBoundedFile(handle, Math.min(remaining, this.serverMaxFileBytes), options.signal));
-                    const sha256 = retained?.sha256 ?? sha256Bytes(bytes);
+                    // Metadata is not content identity (including on aliased/networked
+                    // filesystems). Verify bytes before reusing parsed/indexed facts.
+                    const observed = await readBoundedFile(handle, Math.min(remaining, this.serverMaxFileBytes), options.signal);
+                    const sha256 = sha256Bytes(observed);
+                    const bytes = cached?.sha256 === sha256 && cached.bytes.equals(observed) ? cached.bytes : observed;
                     this.retainSource(cacheKey, {
-                        size: metadata.size,
-                        modifiedMs: metadata.mtimeMs,
-                        changedMs: metadata.ctimeMs,
-                        inode: metadata.ino,
                         sha256,
                         bytes,
                     });
@@ -215,14 +158,11 @@ export class WorkspaceScanner {
                         bytes,
                     };
                     result.push(scanned);
-                    rootFiles.push(scanned);
                 }
                 finally {
                     await handle.close();
                 }
             }
-            if (gameScanKey !== undefined)
-                this.retainGameScan(gameScanKey, rootFiles);
         }
         result.sort((left, right) => left.loadOrder - right.loadOrder || compareCodeUnits(left.relativePath, right.relativePath));
         const groups = new Map();