hoi4-agent-tools 3.0.9 → 3.2.0

package/dist/hoi4_agent_tools/core/semantic-dependencies.js

@@ -0,0 +1,233 @@
+import { canonicalJson, compareCodeUnits, hashCanonical } from './canonical.js';
+import { ReverseSourceDependencies } from './index-segments.js';
+import { PACKAGE_VERSION } from '../version.js';
+const MAX_PROOF_BYTES = 32 * 1024 * 1024;
+const MAX_FRAGMENT_PROOFS = 10_000;
+const MAX_SCOPE_PROOFS = 32;
+/** Source identity includes provenance and interpretation, not just identical source bytes. */
+export function semanticSourceIdentity(file) {
+    return hashCanonical({
+        version: 'semantic-source.v1',
+        toolVersion: PACKAGE_VERSION,
+        absolutePath: file.absolutePath,
+        path: file.displayPath,
+        relativePath: file.relativePath,
+        rootKind: file.rootKind,
+        loadOrder: file.loadOrder,
+        shadowedBy: file.shadowedBy ?? null,
+        hash: file.sha256,
+    });
+}
+function valueIdentity(value) {
+    if (value instanceof Set)
+        return [...value].map(valueIdentity);
+    if (value instanceof Map)
+        return [...value].map(([key, child]) => [
+            valueIdentity(key),
+            valueIdentity(child),
+        ]);
+    return value ?? null;
+}
+/** Observe actual typed catalog lookups, including unsuccessful lookups for later additions. */
+export class SemanticCatalog {
+    bindings = new Map();
+    reads;
+    bind(name, kind, values) {
+        if (this.bindings.has(name))
+            throw new Error('Semantic catalog binding names must be unique');
+        this.bindings.set(name, { kind, values });
+        const record = (id) => this.reads?.set(canonicalJson([name, id]), { name, kind, id });
+        return new Proxy(values, {
+            get(target, property) {
+                if (property === 'get' || property === 'has')
+                    return (id) => {
+                        record(id);
+                        return property === 'get'
+                            ? target.get(id)
+                            : target.has(id);
+                    };
+                if (property === 'size') {
+                    record(null);
+                    return target.size;
+                }
+                if (['set', 'add', 'delete', 'clear'].includes(String(property)))
+                    return () => {
+                        throw new Error('Semantic catalogs are read-only');
+                    };
+                if (property === Symbol.iterator ||
+                    ['entries', 'keys', 'values', 'forEach'].includes(String(property))) {
+                    record(null);
+                    const method = Reflect.get(target, property, target);
+                    return method.bind(target);
+                }
+                return Reflect.get(target, property, target);
+            },
+        });
+    }
+    collect(analyze) {
+        const previous = this.reads;
+        this.reads = new Map();
+        try {
+            const value = analyze();
+            const reads = [...this.reads.entries()]
+                .sort(([a], [b]) => compareCodeUnits(a, b))
+                .map(([, read]) => read);
+            return { value, reads, signature: this.signature(reads) };
+        }
+        finally {
+            this.reads = previous;
+        }
+    }
+    signature(reads) {
+        return hashCanonical(reads.map(({ name, kind, id }) => {
+            const binding = this.bindings.get(name);
+            if (binding?.kind !== kind)
+                return [name, kind, id, 'missing-binding'];
+            const values = binding.values;
+            return [
+                name,
+                kind,
+                id,
+                id === null
+                    ? valueIdentity(values)
+                    : [values.has(id), values instanceof Map ? valueIdentity(values.get(id)) : null],
+            ];
+        }));
+    }
+}
+class DependencyProofs {
+    entries = new Map();
+    scopes = new Map();
+    entryBytes = 0;
+    scopeBytes = 0;
+    put(key, proof) {
+        const bytes = canonicalJson(proof).length * 2 + key.length * 2 + 128;
+        const previous = this.entries.get(key);
+        if (previous !== undefined) {
+            this.entries.delete(key);
+            this.entryBytes -= previous.bytes;
+        }
+        if (bytes > MAX_PROOF_BYTES)
+            return;
+        this.entries.set(key, { proof, bytes });
+        this.entryBytes += bytes;
+        while (this.entries.size > MAX_FRAGMENT_PROOFS || this.entryBytes > MAX_PROOF_BYTES) {
+            const [oldest, entry] = this.entries.entries().next().value;
+            this.entries.delete(oldest);
+            this.entryBytes -= entry.bytes;
+        }
+    }
+    remember(scope, proof) {
+        const previous = this.scopes.get(scope);
+        if (previous !== undefined) {
+            this.scopes.delete(scope);
+            this.scopeBytes -= previous.bytes;
+        }
+        if (proof.bytes > MAX_PROOF_BYTES)
+            return;
+        this.scopes.set(scope, proof);
+        this.scopeBytes += proof.bytes;
+        while (this.scopes.size > MAX_SCOPE_PROOFS || this.scopeBytes > MAX_PROOF_BYTES) {
+            const [oldest, entry] = this.scopes.entries().next().value;
+            this.scopes.delete(oldest);
+            this.scopeBytes -= entry.bytes;
+        }
+    }
+}
+const proofsByCache = new WeakMap();
+export function clearSemanticDependencies(cache) {
+    proofsByCache.delete(cache);
+}
+/** Reconcile both old and new typed definitions so removal, overlay and rename edges survive. */
+export function beginSemanticAnalysis(cache, snapshot, domain, workspaceIdentity) {
+    const scope = hashCanonical({
+        domain,
+        workspaceId: snapshot.workspaceId,
+        workspaceIdentity: workspaceIdentity ?? null,
+        roots: [
+            ...new Set(snapshot.files.map(({ absolutePath, relativePath }) => absolutePath.slice(0, -relativePath.length))),
+        ].sort(compareCodeUnits),
+    });
+    let proofs = cache === undefined ? undefined : proofsByCache.get(cache);
+    if (proofs === undefined) {
+        proofs = new DependencyProofs();
+        if (cache !== undefined)
+            proofsByCache.set(cache, proofs);
+    }
+    const files = new Map(snapshot.files.map((file) => [file.displayPath, semanticSourceIdentity(file)]));
+    const symbols = snapshot.index.symbols.map(({ kind, id, path }) => ({ kind, id, path }));
+    const previous = proofs.scopes.get(scope);
+    const baselineRevision = previous?.baselineRevision ?? snapshot.revision;
+    const invalidations = new Map([...(previous?.invalidations ?? [])].filter(([path]) => files.has(path)));
+    let affectedFiles = new Set();
+    if (previous !== undefined) {
+        const changed = [...new Set([...previous.files.keys(), ...files.keys()])].filter((path) => previous.files.get(path) !== files.get(path));
+        if (changed.length > 0) {
+            const references = [
+                ...snapshot.index.references,
+            ];
+            for (const { proof } of proofs.entries.values())
+                if (proof.scope === scope)
+                    for (const { kind, id } of proof.reads)
+                        if (id !== null)
+                            references.push({ path: proof.path, toKind: kind, to: id });
+            affectedFiles = new Set(new ReverseSourceDependencies([...previous.symbols, ...symbols], references).affectedFiles(changed));
+            for (const path of affectedFiles)
+                if (files.has(path))
+                    invalidations.set(path, snapshot.revision);
+        }
+    }
+    const bytes = [...files].reduce((total, [path, address]) => total + 128 + (path.length + address.length) * 2, 0) +
+        symbols.reduce((total, { kind, id, path }) => total + 128 + (kind.length + id.length + path.length) * 2, 0) +
+        [...invalidations].reduce((total, [path, revision]) => total + 128 + (path.length + revision.length) * 2, 0);
+    proofs.remember(scope, { files, symbols, invalidations, bytes, baselineRevision });
+    return {
+        scope,
+        proofs,
+        affectedFiles,
+        complete: snapshot.complete,
+        invalidations,
+        baselineRevision,
+    };
+}
+export function semanticFragment(session, cache, file, legacyKey, catalog, analyze) {
+    if (cache === undefined)
+        return analyze();
+    const address = `${session.scope}:${semanticSourceIdentity(file)}`;
+    const previous = session.proofs.entries.get(address)?.proof;
+    const dependencyRevision = session.invalidations.get(file.displayPath) ?? session.baselineRevision;
+    if (!session.affectedFiles.has(file.displayPath)) {
+        if (previous?.environment === session.complete &&
+            previous.dependencyRevision === dependencyRevision &&
+            previous.signature === catalog.signature(previous.reads)) {
+            const value = cache.get(previous.key);
+            if (value !== undefined)
+                return value;
+        }
+        else if (previous === undefined) {
+            // Preserve custom graph-builder cache adapters; internally produced entries use exact read proofs.
+            const value = cache.get(legacyKey);
+            if (value !== undefined)
+                return value;
+        }
+    }
+    const { value, reads, signature } = catalog.collect(analyze);
+    const key = hashCanonical({
+        version: 'semantic-fragment.v1',
+        address,
+        signature,
+        complete: session.complete,
+        dependencyRevision,
+    });
+    cache.set(key, value, file.bytes.length);
+    session.proofs.put(address, {
+        scope: session.scope,
+        path: file.displayPath,
+        reads,
+        signature,
+        key,
+        environment: session.complete,
+        dependencyRevision,
+    });
+    return value;
+}

package/dist/hoi4_agent_tools/core/shared-request-capacity.js

@@ -1,5 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import { lstat, mkdir, readdir, rmdir, unlink, writeFile } from 'node:fs/promises';
+import { lstat, mkdir, readdir, rename, rmdir, unlink, writeFile } from 'node:fs/promises';
 import { hostname } from 'node:os';
 import path from 'node:path';
 import { setTimeout as delay } from 'node:timers/promises';
@@ -15,7 +15,6 @@ function processAlive(pid) {
         return error.code !== 'ESRCH';
     }
 }
-/** Bounds heavy work across task processes sharing the same private server state. */
 export class SharedRequestCapacity {
     stateRoot;
     capacity;
@@ -26,7 +25,11 @@ export class SharedRequestCapacity {
     async run(signal, action) {
         signal.throwIfAborted();
         if (this.stateRoot === undefined)
-            return action();
+            return action({
+                handoffToProcess: () => {
+                    return Promise.reject(new ServiceError('REQUEST_LEASE_STORAGE_REQUIRED', 'Worker ownership transfer requires persistent capacity storage'));
+                },
+            });
         const root = await containedGeneratedPath(await canonicalPath(this.stateRoot, signal), 'request-capacity', sha256Bytes(hostname().toLowerCase()).slice(0, 16));
         await mkdir(root, { recursive: true });
         const lease = `${process.pid}-${randomUUID()}.lease`;
@@ -74,18 +77,47 @@ export class SharedRequestCapacity {
                     await rmdir(slot).catch(() => undefined);
                     throw error;
                 }
+                let transferredPid;
                 try {
                     signal.throwIfAborted();
-                    return await action();
+                    return await action({
+                        handoffToProcess: async (pid) => {
+                            if (transferredPid !== undefined)
+                                throw new ServiceError('REQUEST_LEASE_ALREADY_TRANSFERRED', 'The capacity lease already belongs to a child process');
+                            if (!Number.isSafeInteger(pid) ||
+                                pid <= 0 ||
+                                pid === process.pid ||
+                                !processAlive(pid))
+                                throw new ServiceError('REQUEST_LEASE_OWNER_INVALID', 'Capacity can only transfer to another live process');
+                            const next = path.join(slot, `${pid}-${randomUUID()}.lease`);
+                            try {
+                                await rename(owner, next);
+                            }
+                            catch (error) {
+                                if (error.code === 'ENOENT')
+                                    throw new ServiceError('REQUEST_LEASE_HANDOFF_LOST', 'The capacity lease disappeared before worker dispatch');
+                                throw error;
+                            }
+                            owner = next;
+                            transferredPid = pid;
+                        },
+                    });
                 }
                 finally {
-                    await unlink(owner);
-                    await rmdir(slot).catch((error) => {
-                        const code = error.code;
-                        if (!['ENOENT', 'ENOTEMPTY'].includes(code ?? '') &&
-                            !(process.platform === 'win32' && ['EPERM', 'EBUSY'].includes(code ?? '')))
-                            throw error;
-                    });
+                    // A launcher can fail while its admitted child still runs. Keep that child's
+                    // lease until process-exit evidence permits a later reaper to reclaim it.
+                    if (transferredPid === undefined || !processAlive(transferredPid)) {
+                        await unlink(owner).catch((error) => {
+                            if (error.code !== 'ENOENT')
+                                throw error;
+                        });
+                        await rmdir(slot).catch((error) => {
+                            const code = error.code;
+                            if (!['ENOENT', 'ENOTEMPTY'].includes(code ?? '') &&
+                                !(process.platform === 'win32' && ['EPERM', 'EBUSY'].includes(code ?? '')))
+                                throw error;
+                        });
+                    }
                 }
             }
             await delay(100, undefined, { signal });

package/dist/hoi4_agent_tools/core/source/cache.js

@@ -0,0 +1,107 @@
+import { deserialize, serialize } from 'node:v8';
+import { sha256Bytes } from '../canonical.js';
+export function sourceDocumentCacheKey(bytes, sourcePath) {
+    return `${sha256Bytes(bytes)}:${sourcePath}`;
+}
+function decodedDocument(encoded, expected) {
+    const value = deserialize(encoded);
+    if (value === null ||
+        typeof value !== 'object' ||
+        !('path' in value) ||
+        typeof value.path !== 'string' ||
+        !('sourceRevision' in value) ||
+        typeof value.sourceRevision !== 'string' ||
+        !('tokens' in value) ||
+        !Array.isArray(value.tokens) ||
+        !('diagnostics' in value) ||
+        !Array.isArray(value.diagnostics) ||
+        !('root' in value) ||
+        value.root === undefined ||
+        !('bytes' in value) ||
+        !(value.bytes instanceof Uint8Array))
+        throw new Error('Persistent source document is structurally invalid');
+    if (expected !== undefined &&
+        (value.path !== expected.sourcePath ||
+            value.sourceRevision !== sha256Bytes(expected.bytes) ||
+            !Buffer.from(value.bytes).equals(Buffer.from(expected.bytes))))
+        throw new Error('Persistent source document does not match its source identity');
+    return value;
+}
+/** Process-local parsed facts; serialized entries cannot be mutated by their consumers. */
+export class SourceDocumentCache {
+    maxBytes;
+    #entries = new Map();
+    #bytes = 0;
+    #hits = 0;
+    #misses = 0;
+    #evictions = 0;
+    constructor(maxBytes = 64 * 1024 * 1024) {
+        this.maxBytes = maxBytes;
+        if (!Number.isSafeInteger(maxBytes) || maxBytes < 0)
+            throw new RangeError('Source cache size must be a non-negative integer');
+    }
+    get(key) {
+        const encoded = this.#entries.get(key);
+        if (encoded === undefined) {
+            this.#misses += 1;
+            return undefined;
+        }
+        this.#hits += 1;
+        this.#entries.delete(key);
+        this.#entries.set(key, encoded);
+        return decodedDocument(encoded);
+    }
+    put(key, document) {
+        // Avoid allocating a large serialized copy just to discover that it cannot
+        // be retained. This conservative estimate affects caching, never parsing.
+        const estimatedBytes = document.bytes.length * 12 + document.tokens.length * 512 + key.length * 2 + 128;
+        if (this.maxBytes === 0 || estimatedBytes > this.maxBytes)
+            return;
+        const encoded = serialize(document);
+        this.retain(key, encoded);
+    }
+    /** Import one authenticated same-runtime entry without counting it as a process-local hit. */
+    importEncoded(key, encoded, expected) {
+        decodedDocument(encoded, expected);
+        this.retain(key, Buffer.from(encoded));
+    }
+    exportEncoded(key) {
+        const encoded = this.#entries.get(key);
+        return encoded === undefined ? undefined : Buffer.from(encoded);
+    }
+    retain(key, encoded) {
+        const size = encoded.length + key.length * 2 + 128;
+        if (size > this.maxBytes)
+            return;
+        const previous = this.#entries.get(key);
+        if (previous !== undefined)
+            this.#bytes -= previous.length + key.length * 2 + 128;
+        this.#entries.delete(key);
+        while (this.#bytes + size > this.maxBytes) {
+            const oldest = this.#entries.entries().next().value;
+            if (oldest === undefined)
+                break;
+            this.#entries.delete(oldest[0]);
+            this.#bytes -= oldest[1].length + oldest[0].length * 2 + 128;
+            this.#evictions += 1;
+        }
+        this.#entries.set(key, encoded);
+        this.#bytes += size;
+    }
+    clear() {
+        this.#entries.clear();
+        this.#bytes = 0;
+    }
+    statistics() {
+        return {
+            hits: this.#hits,
+            misses: this.#misses,
+            evictions: this.#evictions,
+            retainedBytes: this.#bytes,
+            retainedDocuments: this.#entries.size,
+        };
+    }
+}
+// All domains call the same parser; cache keys include both bytes and source location.
+// Authorization remains in workspace reads, before any document reaches this cache.
+export const sourceDocuments = new SourceDocumentCache();

package/dist/hoi4_agent_tools/core/source/parser.js

@@ -1,6 +1,7 @@
 import { hasBlockingDiagnostics } from '../diagnostics.js';
 import { ServiceError } from '../result.js';
 import { sha256Bytes } from '../canonical.js';
+import { sourceDocumentCacheKey, sourceDocuments } from './cache.js';
 import { decodeSource, encodeSource } from './encoding.js';
 import { createSourceLineIndex, isTrivia, lexClausewitz, locationFor, } from './lexer.js';
 import { SOURCE_LINE_LIMIT, SOURCE_MAX_BYTES, SOURCE_MAX_NESTING, SourceDiagnosticCollector, } from './limits.js';
@@ -224,7 +225,16 @@ class Parser {
         return locationFor(this.sourcePath, this.lineIndex, token.start, token.end);
     }
 }
-export function parseClausewitz(bytes, sourcePath) {
+export function parseClausewitz(bytes, sourcePath, cache = sourceDocuments) {
+    const key = sourceDocumentCacheKey(bytes, sourcePath);
+    const cached = cache.get(key);
+    if (cached !== undefined)
+        return cached;
+    const document = parseClausewitzUncached(bytes, sourcePath);
+    cache.put(key, document);
+    return document;
+}
+function parseClausewitzUncached(bytes, sourcePath) {
     const decoded = decodeSource(bytes);
     const diagnostics = new SourceDiagnosticCollector();
     if (bytes.byteLength > SOURCE_MAX_BYTES) {

package/dist/hoi4_agent_tools/core/source-overlay.js

@@ -0,0 +1,125 @@
+import path from 'node:path';
+import { compareCodeUnits, hashCanonical, sha256Bytes } from './canonical.js';
+import { SymbolIndex } from './index.js';
+import { isWithin } from './workspace.js';
+const MAX_EDITS = 64;
+const MAX_EDIT_BYTES = 8 * 1024 * 1024;
+const MAX_TOTAL_BYTES = 32 * 1024 * 1024;
+const SOURCE_EXTENSION = /\.(?:txt|gui|gfx|yml|yaml)$/iu;
+function checkedRelativePath(relativePath) {
+    if (relativePath.length === 0 ||
+        relativePath.length > 4096 ||
+        relativePath.includes('\\') ||
+        relativePath.includes('\0') ||
+        path.posix.isAbsolute(relativePath) ||
+        /^[A-Za-z]:/u.test(relativePath))
+        throw new RangeError('Proposed source path is not workspace-relative');
+    const parts = relativePath.split('/');
+    if (parts.some((part) => part.length === 0 || part === '.' || part === '..'))
+        throw new RangeError('Proposed source path is not normalized');
+    if (!SOURCE_EXTENSION.test(relativePath))
+        throw new RangeError('Proposed source must be a supported text source');
+    return relativePath;
+}
+function revision(files) {
+    return hashCanonical(files.map(({ absolutePath, displayPath, relativePath, rootKind, loadOrder, shadowedBy, sha256 }) => ({
+        absolutePath,
+        displayPath,
+        relativePath,
+        rootKind,
+        loadOrder,
+        shadowedBy: shadowedBy ?? null,
+        sha256,
+    })));
+}
+/** Builds a source-only proposal in memory under the authorized mod root. */
+export function buildProposedSourceOverlay(baseline, workspace, edits) {
+    if (edits.length < 1 || edits.length > MAX_EDITS)
+        throw new RangeError(`Proposed overlay requires 1-${MAX_EDITS} edits`);
+    if (baseline.workspaceId !== workspace.id)
+        throw new Error('Proposed overlay workspace does not match its baseline snapshot');
+    const modRoot = workspace.roots.find(({ kind }) => kind === 'mod');
+    if (modRoot?.path !== workspace.modRoot)
+        throw new Error('Authorized mod root is unavailable');
+    const files = baseline.files.map((file) => {
+        const copy = { ...file };
+        delete copy.shadowedBy;
+        return copy;
+    });
+    const changes = [];
+    const seen = new Set();
+    let totalBytes = 0;
+    for (const edit of edits) {
+        const relativePath = checkedRelativePath(edit.relativePath);
+        const key = relativePath.toLowerCase();
+        if (seen.has(key))
+            throw new Error(`Duplicate proposed source path: ${relativePath}`);
+        seen.add(key);
+        const existingIndex = files.findIndex((file) => file.rootKind === 'mod' && file.relativePath.toLowerCase() === relativePath.toLowerCase());
+        const existing = existingIndex < 0 ? undefined : files[existingIndex];
+        const sourcePath = existing?.relativePath ?? relativePath;
+        const absolutePath = path.resolve(modRoot.path, ...sourcePath.split('/'));
+        if (!isWithin(modRoot.path, absolutePath) || absolutePath === modRoot.path)
+            throw new RangeError('Proposed source path escapes the authorized mod root');
+        const displayPath = existing?.displayPath ?? `mod:${sourcePath}`;
+        if (existing !== undefined)
+            files.splice(existingIndex, 1);
+        if (edit.content === null) {
+            if (existing === undefined)
+                throw new Error(`Proposed removal has no mod source: ${relativePath}`);
+            changes.push({
+                path: existing.displayPath,
+                change: 'removed',
+                beforeHash: existing.sha256,
+                afterHash: null,
+            });
+            continue;
+        }
+        const bytes = Buffer.from(edit.content, 'utf8');
+        totalBytes += bytes.length;
+        if (bytes.length > MAX_EDIT_BYTES || totalBytes > MAX_TOTAL_BYTES)
+            throw new RangeError('Proposed source content exceeds the overlay limit');
+        const sha256 = sha256Bytes(bytes);
+        files.push({
+            absolutePath,
+            displayPath,
+            relativePath: sourcePath,
+            rootKind: 'mod',
+            loadOrder: modRoot.loadOrder,
+            size: bytes.length,
+            modifiedMs: 0,
+            sha256,
+            bytes,
+        });
+        changes.push({
+            path: displayPath,
+            change: existing === undefined ? 'added' : 'replaced',
+            beforeHash: existing?.sha256 ?? null,
+            afterHash: sha256,
+        });
+    }
+    files.sort((left, right) => left.loadOrder - right.loadOrder || compareCodeUnits(left.relativePath, right.relativePath));
+    const groups = new Map();
+    for (const file of files) {
+        const group = groups.get(file.relativePath.toLowerCase()) ?? [];
+        group.push(file);
+        groups.set(file.relativePath.toLowerCase(), group);
+    }
+    for (const group of groups.values()) {
+        group.sort((left, right) => right.loadOrder - left.loadOrder || compareCodeUnits(left.displayPath, right.displayPath));
+        for (const file of group.slice(1))
+            file.shadowedBy = group[0].displayPath;
+    }
+    const index = SymbolIndex.build(files);
+    const snapshot = {
+        workspaceId: baseline.workspaceId,
+        revision: revision(files),
+        files,
+        index,
+        complete: index.complete,
+        skippedSourceCount: index.skippedSourceCount,
+        skippedSources: index.skippedSources,
+        diagnostics: index.diagnostics,
+    };
+    return { baselineRevision: baseline.revision, snapshot, changes };
+}