hoi4-agent-tools 3.9.0 → 4.0.0

package/dist/hoi4_agent_tools/core/server-registry.js

@@ -0,0 +1,155 @@
+import { unlinkSync } from 'node:fs';
+import { mkdir, readdir, readFile, rename, unlink, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { z } from 'zod/v4';
+import { PACKAGE_VERSION } from '../version.js';
+/**
+ * Every server process sharing a server state root records a small heartbeat, so one status
+ * read shows how many servers and workers a machine runs and how much memory each holds.
+ */
+const HEARTBEAT_MS = 60_000;
+/** A record this old without an update belongs to a process that is blocked or gone. */
+const STALE_MS = 5 * HEARTBEAT_MS;
+const MAX_RECORDS = 512;
+const ABANDONED_MS = 24 * STALE_MS;
+const recordSchema = z
+    .object({
+    pid: z.number().int().positive(),
+    version: z.string().max(64),
+    transport: z.enum(['stdio', 'http']),
+    startedAt: z.string().max(64),
+    updatedAt: z.string().max(64),
+    lastActivityAt: z.string().max(64).optional(),
+    rssBytes: z.number().int().min(0),
+    heapUsedBytes: z.number().int().min(0),
+})
+    .strict();
+function directory(resolver) {
+    const state = resolver.serverState();
+    return state === undefined ? undefined : path.join(state.root, 'servers');
+}
+/** Whether a process exists; a permission refusal still proves it exists. */
+export function processAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === 'EPERM';
+    }
+}
+/** The registry of this server process, or a status-only one when no server started it. */
+export function serverRegistry(resolver) {
+    return ServerRegistry.active ?? new ServerRegistry(resolver, 'stdio');
+}
+export class ServerRegistry {
+    resolver;
+    transport;
+    lastActivity;
+    /** The registry whose heartbeats this process writes. */
+    static active;
+    #startedAt = new Date().toISOString();
+    #timer;
+    constructor(resolver, transport, lastActivity = () => undefined) {
+        this.resolver = resolver;
+        this.transport = transport;
+        this.lastActivity = lastActivity;
+    }
+    current() {
+        const memory = process.memoryUsage();
+        const activity = this.lastActivity();
+        return {
+            pid: process.pid,
+            version: PACKAGE_VERSION,
+            transport: this.transport,
+            startedAt: this.#startedAt,
+            updatedAt: new Date().toISOString(),
+            ...(activity === undefined ? {} : { lastActivityAt: new Date(activity).toISOString() }),
+            rssBytes: memory.rss,
+            heapUsedBytes: memory.heapUsed,
+        };
+    }
+    /** Start heartbeats; the record is removed when the process exits normally. */
+    start() {
+        ServerRegistry.active = this;
+        const root = directory(this.resolver);
+        if (root === undefined)
+            return;
+        const file = path.join(root, `${process.pid}.json`);
+        const beat = () => void mkdir(root, { recursive: true })
+            .then(async () => {
+            const temporary = `${file}.${Date.now()}.tmp`;
+            try {
+                await writeFile(temporary, `${JSON.stringify(this.current())}\n`);
+                await rename(temporary, file);
+            }
+            catch {
+                await unlink(temporary).catch(() => undefined);
+            }
+        })
+            .catch(() => undefined);
+        beat();
+        this.#timer = setInterval(beat, HEARTBEAT_MS);
+        this.#timer.unref();
+        process.once('exit', () => {
+            try {
+                unlinkSync(file);
+            }
+            catch {
+                // A record left behind is removed by the next status read once the pid is gone.
+            }
+        });
+    }
+    async status() {
+        const current = this.current();
+        const config = this.resolver.config();
+        const limits = {
+            maxConcurrentTools: config.maxConcurrentTools,
+            maxSharedTools: config.maxSharedTools,
+            jobDeadlineSeconds: config.jobDeadlineSeconds,
+            jobCancelGraceSeconds: config.jobCancelGraceSeconds,
+            stdioIdleExitMinutes: config.stdioIdleExitMinutes,
+            jobWorkerMaxHeapMiB: config.jobWorkerMaxHeapMiB,
+        };
+        const root = directory(this.resolver);
+        const servers = [];
+        let removed = 0;
+        if (root !== undefined) {
+            const entries = await readdir(root).catch(() => []);
+            // Temporary heartbeat files left by a failed rename.
+            for (const name of entries) {
+                const temporary = /^\d+\.json\.(\d+)\.tmp$/u.exec(name);
+                if (temporary !== null && Date.now() - Number(temporary[1]) > STALE_MS)
+                    await unlink(path.join(root, name)).catch(() => undefined);
+            }
+            const names = entries.filter((name) => /^\d+\.json$/u.test(name)).slice(0, MAX_RECORDS);
+            for (const name of names) {
+                const file = path.join(root, name);
+                const parsed = recordSchema.safeParse(await readFile(file, 'utf8')
+                    .then((text) => JSON.parse(text))
+                    .catch(() => undefined));
+                if (!parsed.success)
+                    continue;
+                const record = parsed.data;
+                if (record.pid === process.pid)
+                    continue;
+                if (!processAlive(record.pid)) {
+                    removed += 1;
+                    await unlink(file).catch(() => undefined);
+                    continue;
+                }
+                const age = Date.now() - Date.parse(record.updatedAt);
+                // A record silent for this long belongs to an exited process whose pid was reused.
+                if (age > ABANDONED_MS) {
+                    removed += 1;
+                    await unlink(file).catch(() => undefined);
+                    continue;
+                }
+                servers.push({ ...record, state: age > STALE_MS ? 'stale' : 'live' });
+            }
+        }
+        servers.push({ ...current, state: 'live' });
+        servers.sort((left, right) => right.rssBytes - left.rssBytes || left.pid - right.pid);
+        return { current, servers, removed, limits };
+    }
+}

package/dist/hoi4_agent_tools/core/server-status-resource.js

@@ -0,0 +1,22 @@
+import { canonicalJson } from './canonical.js';
+import { serverRegistry } from './server-registry.js';
+/** A fixed, listed resource: every server process sharing this server state and its memory. */
+export const serverStatusResource = {
+    uri: 'hoi4-agent://server/status',
+    name: 'server-status',
+    title: 'HOI4 Agent Tools server status',
+    description: 'Live server processes sharing this server state, with memory, last client activity and execution limits.',
+    mimeType: 'application/json',
+};
+export async function readServerStatus(engine) {
+    const status = await serverRegistry(engine.resolver).status();
+    return {
+        contents: [
+            {
+                uri: serverStatusResource.uri,
+                mimeType: serverStatusResource.mimeType,
+                text: `${canonicalJson(status)}\n`,
+            },
+        ],
+    };
+}

package/dist/hoi4_agent_tools/core/shared-request-capacity.js

@@ -20,11 +20,13 @@ export class SharedRequestCapacity {
     stateRoot;
     capacity;
     deletePendingLimitMs;
+    leaseMaxAgeMs;
     now;
     constructor(stateRoot, capacity = 4, options = {}) {
         this.stateRoot = stateRoot;
         this.capacity = capacity;
         this.deletePendingLimitMs = options.deletePendingLimitMs ?? 60_000;
+        this.leaseMaxAgeMs = options.leaseMaxAgeMs;
         this.now = options.now ?? Date.now;
     }
     async run(signal, action) {
@@ -34,6 +36,7 @@ export class SharedRequestCapacity {
                 handoffToProcess: () => {
                     return Promise.reject(new ServiceError('REQUEST_LEASE_STORAGE_REQUIRED', 'Worker ownership transfer requires persistent capacity storage'));
                 },
+                reclaimFromProcess: () => Promise.resolve(),
             });
         const root = await containedGeneratedPath(await canonicalPath(this.stateRoot, signal), 'request-capacity', sha256Bytes(hostname().toLowerCase()).slice(0, 16));
         await mkdir(root, { recursive: true });
@@ -112,6 +115,14 @@ export class SharedRequestCapacity {
                             owner = next;
                             transferredPid = pid;
                         },
+                        reclaimFromProcess: async () => {
+                            if (transferredPid === undefined)
+                                return;
+                            const next = path.join(slot, `${process.pid}-${randomUUID()}.lease`);
+                            await retryWindowsSharing(() => rename(owner, next));
+                            owner = next;
+                            transferredPid = undefined;
+                        },
                     });
                 }
                 finally {
@@ -137,6 +148,18 @@ export class SharedRequestCapacity {
             await delay(sharingRetryDelay(waits) * (0.75 + Math.random() / 2), undefined, { signal });
         }
     }
+    async leaseExpired(lease) {
+        if (this.leaseMaxAgeMs === undefined)
+            return false;
+        try {
+            return this.now() - (await lstat(lease)).mtimeMs > this.leaseMaxAgeMs;
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return false;
+            throw error;
+        }
+    }
     async reap(slot) {
         try {
             const metadata = await lstat(slot);
@@ -150,7 +173,9 @@ export class SharedRequestCapacity {
             else {
                 for (const name of owners) {
                     const match = /^([1-9][0-9]*)-[0-9a-f-]{36}\.lease$/u.exec(name);
-                    if (match === null || processAlive(Number(match[1])))
+                    if (match === null)
+                        return;
+                    if (processAlive(Number(match[1])) && !(await this.leaseExpired(path.join(slot, name))))
                         return;
                     // Unique owner names prevent a concurrent reaper from deleting a new lease.
                     await unlink(path.join(slot, name)).catch((error) => {

package/dist/hoi4_agent_tools/core/source/cache.js

@@ -3,7 +3,7 @@ import { sha256Bytes } from '../canonical.js';
 export function sourceDocumentCacheKey(bytes, sourcePath) {
     return `${sha256Bytes(bytes)}:${sourcePath}`;
 }
-function decodedDocument(encoded, expected) {
+function decodedDocument(encoded) {
     const value = deserialize(encoded);
     if (value === null ||
         typeof value !== 'object' ||
@@ -20,36 +20,62 @@ function decodedDocument(encoded, expected) {
         !('bytes' in value) ||
         !(value.bytes instanceof Uint8Array))
         throw new Error('Persistent source document is structurally invalid');
-    if (expected !== undefined &&
-        (value.path !== expected.sourcePath ||
-            value.sourceRevision !== sha256Bytes(expected.bytes) ||
-            !Buffer.from(value.bytes).equals(Buffer.from(expected.bytes))))
-        throw new Error('Persistent source document does not match its source identity');
     return value;
 }
-/** Process-local parsed facts; serialized entries cannot be mutated by their consumers. */
+/** Puts between halvings of every entry's hit count, so entries no longer read become evictable. */
+const HIT_AGING_INTERVAL = 4096;
+/** Keys of documents turned away from a full cache, remembered so a repeat can be admitted. */
+const DOORKEEPER_KEYS = 16_384;
+/**
+ * Process-local parsed facts; serialized entries cannot be mutated by their consumers.
+ *
+ * A workspace scan often parses more source than the cache holds. Plain LRU then evicts
+ * every document before its next use and pays serialization for nothing. A full cache
+ * therefore turns a document away the first time, remembering only its key, and admits it
+ * on a later miss by displacing entries that were never read; a rejected document is not
+ * serialized. Repeated scans keep what fits and hit it. Hit counts age, so documents that
+ * stop being read become evictable again.
+ */
 export class SourceDocumentCache {
     maxBytes;
     #entries = new Map();
     #bytes = 0;
+    /** Bytes held by entries without hits, which a new document may displace. */
+    #coldBytes = 0;
+    #puts = 0;
     #hits = 0;
     #misses = 0;
     #evictions = 0;
+    #rejections = 0;
+    #doorkeeper = new Set();
     constructor(maxBytes = 64 * 1024 * 1024) {
         this.maxBytes = maxBytes;
         if (!Number.isSafeInteger(maxBytes) || maxBytes < 0)
             throw new RangeError('Source cache size must be a non-negative integer');
     }
     get(key) {
-        const encoded = this.#entries.get(key);
-        if (encoded === undefined) {
+        const entry = this.#entries.get(key);
+        if (entry === undefined) {
+            this.#misses += 1;
+            return undefined;
+        }
+        let document;
+        try {
+            document = decodedDocument(entry.encoded);
+        }
+        catch {
+            // An imported entry is decoded here, on first use; one that cannot be decoded is a miss.
+            this.#remove(key);
             this.#misses += 1;
             return undefined;
         }
         this.#hits += 1;
+        if (entry.hits === 0)
+            this.#coldBytes -= entrySize(key, entry.encoded);
+        entry.hits += 1;
         this.#entries.delete(key);
-        this.#entries.set(key, encoded);
-        return decodedDocument(encoded);
+        this.#entries.set(key, entry);
+        return document;
     }
     /** Availability check without deserialization or cache-hit accounting. */
     has(key) {
@@ -61,51 +87,104 @@ export class SourceDocumentCache {
         const estimatedBytes = document.bytes.length * 12 + document.tokens.length * 512 + key.length * 2 + 128;
         if (this.maxBytes === 0 || estimatedBytes > this.maxBytes)
             return;
-        const encoded = serialize(document);
-        this.retain(key, encoded);
+        this.#age();
+        const previous = this.#entries.get(key);
+        const reusable = previous === undefined ? 0 : entrySize(key, previous.encoded);
+        const free = this.maxBytes - this.#bytes + reusable;
+        if (estimatedBytes > free) {
+            // Only a document missed before may displace entries, and only entries never read.
+            if (!this.#doorkeeper.delete(key) || estimatedBytes > free + this.#coldBytes) {
+                this.#remember(key);
+                this.#rejections += 1;
+                return;
+            }
+        }
+        this.retain(key, serialize(document));
     }
-    /** Import one authenticated same-runtime entry without counting it as a process-local hit. */
-    importEncoded(key, encoded, expected) {
-        decodedDocument(encoded, expected);
+    /**
+     * Import one authenticated same-runtime entry without counting it as a process-local hit.
+     * The caller has verified the entry against the key's source bytes (the key is their hash and
+     * the path), so the document is decoded when it is first read instead of for every file at
+     * load, which cost as much as parsing the files again; `get` treats an undecodable entry as
+     * a miss.
+     */
+    importEncoded(key, encoded) {
         this.retain(key, Buffer.from(encoded));
     }
     exportEncoded(key) {
-        const encoded = this.#entries.get(key);
-        return encoded === undefined ? undefined : Buffer.from(encoded);
+        const entry = this.#entries.get(key);
+        return entry === undefined ? undefined : Buffer.from(entry.encoded);
     }
     retain(key, encoded) {
-        const size = encoded.length + key.length * 2 + 128;
+        const size = entrySize(key, encoded);
         if (size > this.maxBytes)
             return;
-        const previous = this.#entries.get(key);
-        if (previous !== undefined)
-            this.#bytes -= previous.length + key.length * 2 + 128;
-        this.#entries.delete(key);
-        while (this.#bytes + size > this.maxBytes) {
-            const oldest = this.#entries.entries().next().value;
-            if (oldest === undefined)
+        this.#remove(key);
+        // Displace the oldest entries that were never read again; read entries stay.
+        for (const [oldKey, entry] of this.#entries) {
+            if (this.#bytes + size <= this.maxBytes)
                 break;
-            this.#entries.delete(oldest[0]);
-            this.#bytes -= oldest[1].length + oldest[0].length * 2 + 128;
+            if (entry.hits > 0)
+                continue;
+            this.#remove(oldKey);
             this.#evictions += 1;
         }
-        this.#entries.set(key, encoded);
+        if (this.#bytes + size > this.maxBytes) {
+            this.#rejections += 1;
+            return;
+        }
+        this.#entries.set(key, { encoded, hits: 0 });
         this.#bytes += size;
+        this.#coldBytes += size;
+    }
+    #remember(key) {
+        this.#doorkeeper.delete(key);
+        this.#doorkeeper.add(key);
+        if (this.#doorkeeper.size > DOORKEEPER_KEYS)
+            this.#doorkeeper.delete(this.#doorkeeper.values().next().value);
+    }
+    #remove(key) {
+        const entry = this.#entries.get(key);
+        if (entry === undefined)
+            return;
+        const size = entrySize(key, entry.encoded);
+        this.#entries.delete(key);
+        this.#bytes -= size;
+        if (entry.hits === 0)
+            this.#coldBytes -= size;
+    }
+    #age() {
+        this.#puts += 1;
+        if (this.#puts % HIT_AGING_INTERVAL !== 0)
+            return;
+        for (const [key, entry] of this.#entries) {
+            if (entry.hits === 0)
+                continue;
+            entry.hits = Math.floor(entry.hits / 2);
+            if (entry.hits === 0)
+                this.#coldBytes += entrySize(key, entry.encoded);
+        }
     }
     clear() {
         this.#entries.clear();
+        this.#doorkeeper.clear();
         this.#bytes = 0;
+        this.#coldBytes = 0;
     }
     statistics() {
         return {
             hits: this.#hits,
             misses: this.#misses,
             evictions: this.#evictions,
+            rejections: this.#rejections,
             retainedBytes: this.#bytes,
             retainedDocuments: this.#entries.size,
         };
     }
 }
+function entrySize(key, encoded) {
+    return encoded.length + key.length * 2 + 128;
+}
 // All domains call the same parser; cache keys include both bytes and source location.
 // Authorization remains in workspace reads, before any document reaches this cache.
 export const sourceDocuments = new SourceDocumentCache();

package/dist/hoi4_agent_tools/core/source/lexer.js

@@ -1,4 +1,12 @@
 import { SOURCE_LINE_LIMIT, SOURCE_TOKEN_LIMIT, SourceDiagnosticCollector } from './limits.js';
+const unicodeSpace = /\s/u;
+/** The same characters as the `\s` class; the regular expression only runs for non-ASCII text. */
+function isSpace(character) {
+    const code = character.charCodeAt(0);
+    if (code < 128)
+        return code === 32 || (code >= 9 && code <= 13);
+    return code > 159 && unicodeSpace.test(character);
+}
 const operatorStarts = new Set(['=', '<', '>', '!']);
 function startsOperator(text, index) {
     const character = text[index] ?? '';
@@ -141,8 +149,8 @@ export function lexClausewitz(text, sourcePath, options = {}) {
         const startLine = line;
         const startColumn = column;
         const current = text[index] ?? '';
-        if (/\s/u.test(current)) {
-            while (index < text.length && /\s/u.test(text[index] ?? ''))
+        if (isSpace(current)) {
+            while (index < text.length && isSpace(text[index] ?? ''))
                 advance();
             add('whitespace', start, startLine, startColumn);
             continue;
@@ -236,7 +244,7 @@ export function lexClausewitz(text, sourcePath, options = {}) {
         }
         while (index < text.length) {
             const character = text[index] ?? '';
-            if (/\s/u.test(character) ||
+            if (isSpace(character) ||
                 character === '#' ||
                 character === '{' ||
                 character === '}' ||