hoi4-agent-tools 3.6.0 → 3.8.0

package/dist/hoi4_agent_tools/reference/service.js

@@ -1,101 +1,91 @@
-import { open, realpath, stat } from 'node:fs/promises';
+import { realpath } from 'node:fs/promises';
 import path from 'node:path';
-import fg from 'fast-glob';
+import { globFiles } from '../core/file-glob.js';
 import { compareCodeUnits, sha256Bytes } from '../core/canonical.js';
 import { ServiceError } from '../core/result.js';
 import { isWithin } from '../core/workspace.js';
-import { readBoundedLines } from './lines.js';
-import { readBoundedFile } from '../core/scanner.js';
+import { prepareSource, verifySource, readBoundedFile, } from '../core/scanner.js';
+import { decodeSource } from '../core/source/encoding.js';
+import { readBoundedJsonLines } from './lines.js';
+import { RankingIndex, sectionTerms } from './ranking.js';
 const MAX_FILES = 512;
 const MAX_FILE_BYTES = 2_000_000;
 const MAX_TOTAL_BYTES = 32_000_000;
 const MAX_SECTIONS = 20_000;
-const searchStopWords = new Set([
-    'a',
-    'an',
-    'and',
-    'at',
-    'be',
-    'can',
-    'come',
-    'do',
-    'does',
-    'for',
-    'from',
-    'how',
-    'in',
-    'is',
-    'of',
-    'the',
-    'to',
-    'where',
-    'with',
-    'work',
-]);
-function normalizeTerm(term) {
-    if (term === 'dated')
-        return 'date';
-    if (term.endsWith('ies') && term.length > 5)
-        return `${term.slice(0, -3)}y`;
-    if (term.endsWith('ses') && term.length > 5)
-        return term.slice(0, -2);
-    if (term.endsWith('s') && term.length > 4)
-        return term.slice(0, -1);
-    return term;
-}
-function queryTerms(query) {
-    return [
-        ...new Set((query.toLowerCase().match(/[a-z0-9_.:-]+/gu) ?? [])
-            .map(normalizeTerm)
-            .filter((term) => /[a-z0-9]/u.test(term) && !searchStopWords.has(term))),
-    ];
+const MAX_REFERENCE_METADATA_BYTES = 24_000;
+const PHRASE_CANDIDATES = 60;
+const EXACT_HEADING_SCORE = 1000;
+const sourceOrder = { game_doc: 0, wiki: 1, script_doc: 2 };
+function referenceText(bytes) {
+    if (bytes.length >= 2 &&
+        ((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff))) {
+        const payload = Buffer.from(bytes.subarray(2));
+        if (payload.length % 2 !== 0)
+            throw new ServiceError('REFERENCE_ENCODING_INVALID', 'UTF-16 reference text has an incomplete code unit');
+        if (bytes[0] === 0xfe)
+            payload.swap16();
+        return payload.toString('utf16le');
+    }
+    return decodeSource(bytes).text;
 }
-function containsTerm(text, term) {
-    if (term.length > 2)
-        return text.includes(term);
-    const escaped = term.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
-    return new RegExp(`(^|[^a-z0-9_])${escaped}($|[^a-z0-9_])`, 'u').test(text);
+/** The citation fields of a section, without its indexed text or term statistics. */
+export function publicReferenceSection(section) {
+    const { lines: _lines, terms: _terms, ...citation } = section;
+    return citation;
 }
-function snippet(lines, terms = []) {
-    let matching = -1;
-    let best = -1;
-    for (const [index, line] of lines.entries()) {
-        const count = terms.filter((term) => containsTerm(line.toLowerCase(), term)).length;
-        if (count > best) {
-            matching = index;
-            best = count;
-        }
-    }
-    const selected = lines
-        .slice(Math.max(0, matching), Math.max(0, matching) + 3)
+/** Up to three non-empty lines from `first`, trimmed to the 300-character excerpt bound. */
+function snippet(lines, first = 0) {
+    return lines
+        .slice(first, first + 3)
         .map((line) => line.trim())
         .filter(Boolean)
-        .join(' ');
-    return selected.slice(0, 300);
+        .join(' ')
+        .slice(0, 300);
+}
+// A Markdown table row whose first cell names a command, trigger, modifier, or on action.
+const identifierRow = /^\|\s*`?([A-Za-z_][A-Za-z0-9_@.:]*)`?\s*\|/u;
+const tableSeparator = /^\|\s*:?-{3,}/u;
+/** A closing fence repeats the opening character at least as often and has no info string. */
+function fenceMarker(line) {
+    const match = /^\s*(`{3,}|~{3,})(.*)$/u.exec(line);
+    if (match === null)
+        return undefined;
+    const marker = match[1];
+    return { character: marker[0], length: marker.length, info: match[2].trim() };
 }
 function sectionize(root, filePath, content, revision) {
     const lines = content.replace(/^\uFEFF/u, '').split(/\r?\n/u);
     const relative = path.relative(root.root, filePath).replaceAll('\\', '/');
     const title = path
         .basename(filePath)
-        .replace(/\.(md|log)$/u, '')
+        .replace(/\.(md|log)$/iu, '')
         .replaceAll('_', ' ')
         .replace(/ - Hearts of Iron 4 Wiki$/u, '');
     const headings = [];
-    let inFence = false;
+    const rows = [];
+    let fence;
     for (let index = 0; index < lines.length; index += 1) {
         const line = lines[index];
-        if (/^\s*(```|~~~)/u.test(line)) {
-            inFence = !inFence;
+        const marker = fenceMarker(line);
+        if (fence === undefined && marker !== undefined) {
+            fence = marker;
             continue;
         }
-        if (inFence)
+        if (fence !== undefined) {
+            if (marker?.character === fence.character &&
+                marker.length >= fence.length &&
+                marker.info === '')
+                fence = undefined;
             continue;
+        }
+        const row = identifierRow.exec(line);
+        if (row !== null && !tableSeparator.test(lines[index + 1] ?? ''))
+            rows.push({ heading: row[1], start: index });
         const match = /^(#{1,6})\s+(.+?)\s*#*\s*$/u.exec(line);
         if (match)
             headings.push({ heading: match[2].replace(/<a\b[^>]*><\/a>/gu, '').trim(), start: index });
         else if (root.kind === 'script_doc' &&
-            filePath.endsWith('.log') &&
+            filePath.toLowerCase().endsWith('.log') &&
             /^[A-Za-z_][A-Za-z0-9_.:-]*$/u.test(line.trim()) &&
             !/^\s/u.test(line) &&
             /^\s+\S/u.test(lines[index + 1] ?? '')) {
@@ -104,24 +94,30 @@ function sectionize(root, filePath, content, revision) {
     }
     if (headings.length === 0 || headings[0].start > 0)
         headings.unshift({ heading: title, start: 0 });
-    return headings.map((current, index) => {
-        const end = headings[index + 1]?.start ?? lines.length;
-        const sectionLines = lines.slice(current.start, end);
-        const id = sha256Bytes(Buffer.from(`${root.kind}\0${relative}\0${current.start + 1}\0${current.heading}`, 'utf8'));
+    const section = (heading, start, end) => {
+        const sectionLines = lines.slice(start, end);
+        const id = sha256Bytes(Buffer.from(`${root.kind}\0${relative}\0${start + 1}\0${heading}`, 'utf8'));
         return {
             id,
             revision,
             source: root.kind,
             title,
-            heading: current.heading.slice(0, 512),
+            heading: heading.slice(0, 512),
             path: filePath,
-            startLine: current.start + 1,
+            startLine: start + 1,
             endLine: end,
             excerpt: snippet(sectionLines),
             authority: root.authority,
             lines: sectionLines,
+            terms: sectionTerms(title, heading, sectionLines),
         };
-    });
+    };
+    // Heading sections come first, so a page's opening section still names the page. Table
+    // rows follow as one-line sections that cite one exact command or modifier entry.
+    return [
+        ...headings.map((current, index) => section(current.heading, current.start, headings[index + 1]?.start ?? lines.length)),
+        ...rows.map((row) => section(row.heading, row.start, row.start + 1)),
+    ];
 }
 function sourceRoots(workspace) {
     const gameRoot = workspace.gameRoot ?? (workspace.registration.kind === 'game' ? workspace.modRoot : undefined);
@@ -208,6 +204,7 @@ const gameDocNames = {
 /** A workspace-private, read-only reference index over explicitly known local roots. */
 export class ReferenceService {
     #cache = new Map();
+    #indexes = new Map();
     #cacheBytes = 0;
     retain(key, value) {
         this.#cacheBytes -= this.#cache.get(key)?.bytes ?? 0;
@@ -222,16 +219,33 @@ export class ReferenceService {
             this.#cacheBytes -= oldest[1].bytes;
         }
     }
-    async inventory(workspace, signal) {
+    async inventory(workspace, signal, sources) {
         const sections = [];
         const coverage = {
-            game_doc: { files: 0, sections: 0, unavailable: true },
-            wiki: { files: 0, sections: 0, unavailable: true },
-            script_doc: { files: 0, sections: 0, unavailable: true },
+            game_doc: {
+                files: 0,
+                sections: 0,
+                unavailable: true,
+                included: sources === undefined || sources.includes('game_doc'),
+            },
+            wiki: {
+                files: 0,
+                sections: 0,
+                unavailable: true,
+                included: sources === undefined || sources.includes('wiki'),
+            },
+            script_doc: {
+                files: 0,
+                sections: 0,
+                unavailable: true,
+                included: sources === undefined || sources.includes('script_doc'),
+            },
         };
         let skipped = 0;
         let bytes = 0;
         for (const root of sourceRoots(workspace)) {
+            if (sources !== undefined && !sources.includes(root.kind))
+                continue;
             signal?.throwIfAborted();
             let canonicalRoot;
             try {
@@ -250,121 +264,205 @@ export class ReferenceService {
                 throw new ServiceError('REFERENCE_ROOT_ESCAPE', 'Reference root resolves outside its configured workspace');
             }
             coverage[root.kind].unavailable = false;
-            const names = await fg(root.patterns, {
+            const names = await globFiles(root.patterns, {
                 cwd: canonicalRoot,
-                onlyFiles: true,
-                followSymbolicLinks: false,
-                dot: false,
+                caseSensitive: false,
+                ...(signal === undefined ? {} : { signal }),
             });
             names.sort(compareCodeUnits);
-            for (const name of names) {
+            for (let offset = 0; offset < names.length; offset += 8) {
                 signal?.throwIfAborted();
                 if (coverage[root.kind].files >= MAX_FILES || sections.length >= MAX_SECTIONS) {
-                    skipped++;
-                    continue;
-                }
-                const candidate = path.join(canonicalRoot, name);
-                const canonical = await realpath(candidate);
-                if (!isWithin(canonicalRoot, canonical)) {
-                    skipped++;
-                    continue;
+                    skipped += names.length - offset;
+                    break;
                 }
-                const metadata = await stat(canonical);
-                if (metadata.size > MAX_FILE_BYTES || bytes + metadata.size > MAX_TOTAL_BYTES) {
-                    skipped++;
-                    continue;
-                }
-                bytes += metadata.size;
-                coverage[root.kind].files++;
-                const key = `${root.kind}:${canonicalRoot}:${canonical}`;
-                const handle = await open(canonical, 'r');
-                let content;
+                const readRoot = { path: canonicalRoot };
+                const prepared = await Promise.allSettled(names.slice(offset, offset + 8).map((name) => prepareSource(readRoot, name, signal)));
+                const opened = prepared.flatMap((entry) => entry.status === 'fulfilled' ? [entry.value] : []);
                 try {
-                    const opened = await handle.stat();
-                    const resolvedAfterOpen = await realpath(canonical);
-                    const current = await stat(resolvedAfterOpen);
-                    if (!opened.isFile() ||
-                        !isWithin(canonicalRoot, resolvedAfterOpen) ||
-                        opened.dev !== current.dev ||
-                        opened.ino !== current.ino) {
+                    signal?.throwIfAborted();
+                    for (const entry of prepared) {
+                        if (entry.status !== 'rejected')
+                            continue;
+                        if (entry.reason instanceof ServiceError && entry.reason.code === 'SCAN_ROOT_ESCAPE') {
+                            skipped++;
+                            continue;
+                        }
                         throw new ServiceError('REFERENCE_SOURCE_CHANGED', 'Reference path changed while opening it; retry the request');
                     }
-                    content = await readBoundedFile(handle, Math.min(MAX_FILE_BYTES, MAX_TOTAL_BYTES - bytes + metadata.size), signal);
+                    const admittedSources = [];
+                    for (const source of opened) {
+                        if (!source.metadata.isFile() ||
+                            source.metadata.size > MAX_FILE_BYTES ||
+                            bytes + source.metadata.size > MAX_TOTAL_BYTES ||
+                            coverage[root.kind].files >= MAX_FILES) {
+                            skipped++;
+                            continue;
+                        }
+                        bytes += source.metadata.size;
+                        coverage[root.kind].files++;
+                        admittedSources.push(source);
+                    }
+                    const reads = await Promise.allSettled(admittedSources.map(async (source) => {
+                        const content = await readBoundedFile(source.handle, source.metadata.size, signal);
+                        await verifySource(readRoot, source);
+                        if (content.length !== source.metadata.size)
+                            throw new ServiceError('REFERENCE_SOURCE_CHANGED', 'Reference changed while it was being read; retry the request');
+                        return content;
+                    }));
+                    signal?.throwIfAborted();
+                    const failed = reads.find((entry) => entry.status === 'rejected');
+                    if (failed?.status === 'rejected')
+                        throw new ServiceError('REFERENCE_SOURCE_CHANGED', 'Reference changed while it was being read; retry the request');
+                    for (const [index, read] of reads.entries()) {
+                        if (read.status !== 'fulfilled')
+                            continue;
+                        const canonical = admittedSources[index].canonicalPath;
+                        const content = read.value;
+                        const key = `${root.kind}:${canonicalRoot}:${canonical}`;
+                        const revision = sha256Bytes(content);
+                        let cached = this.#cache.get(key);
+                        if (cached?.revision !== revision)
+                            cached = {
+                                revision,
+                                bytes: content.length,
+                                sections: sectionize(root, canonical, referenceText(content), revision),
+                            };
+                        this.retain(key, cached);
+                        const admitted = cached.sections.slice(0, MAX_SECTIONS - sections.length);
+                        if (admitted.length < cached.sections.length)
+                            skipped++;
+                        sections.push(...admitted);
+                        coverage[root.kind].sections += admitted.length;
+                    }
                 }
                 finally {
-                    await handle.close();
-                }
-                bytes += content.length - metadata.size;
-                const revision = sha256Bytes(content);
-                let cached = this.#cache.get(key);
-                if (cached?.revision !== revision) {
-                    cached = {
-                        revision,
-                        bytes: content.length,
-                        sections: sectionize(root, canonical, content.toString('utf8'), revision),
-                    };
+                    await Promise.all(opened.map(({ handle }) => handle.close()));
                 }
-                this.retain(key, cached);
-                const admitted = cached.sections.slice(0, MAX_SECTIONS - sections.length);
-                if (admitted.length < cached.sections.length)
-                    skipped++;
-                sections.push(...admitted);
-                coverage[root.kind].sections += admitted.length;
             }
         }
         return { sections, coverage, skipped };
     }
     async search(workspace, input, signal) {
-        const inventory = await this.inventory(workspace, signal);
-        const matches = this.rank(inventory.sections, input.query, input.sources);
+        const inventory = await this.inventory(workspace, signal, input.sources);
+        const { ranked: matches, describe } = this.rank(inventory.sections, input.query);
+        const results = matches.slice(0, input.limit).map((match) => {
+            const { section, score, excerpt, matchLine } = describe(match);
+            return { ...this.publicSection(section), score, excerpt, matchLine };
+        });
+        const requested = results.length;
+        while (Buffer.byteLength(JSON.stringify(results), 'utf8') > MAX_REFERENCE_METADATA_BYTES &&
+            results.length > 1)
+            results.pop();
+        if (Buffer.byteLength(JSON.stringify(results), 'utf8') > MAX_REFERENCE_METADATA_BYTES)
+            throw new ServiceError('REFERENCE_RESPONSE_TOO_LARGE', 'Reference metadata exceeds the bounded reply');
         return {
-            results: matches.slice(0, input.limit).map(({ section, score, excerpt }) => ({
-                ...this.publicSection(section),
-                score,
-                excerpt,
-            })),
+            results,
             total: matches.length,
+            omitted: matches.length - results.length,
+            limitedByBytes: results.length < requested,
             coverage: inventory.coverage,
             skipped: inventory.skipped,
         };
     }
-    rank(sections, question, sources) {
-        const query = question.toLowerCase();
-        const terms = queryTerms(query);
-        return sections
-            .flatMap((section) => {
-            if (sources !== undefined && !sources.includes(section.source))
-                return [];
-            const title = section.title.toLowerCase();
-            const heading = section.heading.toLowerCase();
-            const body = section.lines.join('\n').toLowerCase();
-            let matched = 0;
-            let score = 0;
-            for (const term of terms) {
-                const inTitle = containsTerm(title, term);
-                const inHeading = containsTerm(heading, term);
-                const inBody = containsTerm(body, term);
-                if (!inTitle && !inHeading && !inBody)
-                    continue;
-                matched++;
-                score += (inTitle ? 24 : 0) + (inHeading ? 18 : 0) + (inBody ? 3 : 0);
-            }
-            if (matched < Math.max(1, Math.ceil(terms.length * 0.6)))
-                return [];
-            if (heading === query)
-                score += 100;
-            else if (heading.includes(query))
-                score += 70;
-            if (body.includes(query))
-                score += 30;
-            return [{ section, score, excerpt: snippet(section.lines, terms) }];
-        })
-            .sort((left, right) => right.score - left.score ||
+    /** One index per distinct set of section revisions; IDF depends on the whole inventory. */
+    rankingIndex(sections) {
+        const parts = [];
+        let previous;
+        for (const section of sections) {
+            if (previous?.path !== section.path || previous.revision !== section.revision)
+                parts.push(`${section.source}\0${section.path}\0${section.revision}`);
+            previous = section;
+        }
+        const key = sha256Bytes(Buffer.from(parts.join('\n'), 'utf8'));
+        const cached = this.#indexes.get(key);
+        if (cached !== undefined) {
+            this.#indexes.delete(key);
+            this.#indexes.set(key, cached);
+            return cached;
+        }
+        const index = new RankingIndex(sections.map(({ terms }) => terms));
+        this.#indexes.set(key, index);
+        while (this.#indexes.size > 4) {
+            const oldest = this.#indexes.keys().next().value;
+            if (oldest === undefined)
+                break;
+            this.#indexes.delete(oldest);
+        }
+        return index;
+    }
+    rank(sections, question) {
+        const index = this.rankingIndex(sections);
+        const plan = index.plan(question);
+        const scored = [];
+        for (const section of sections) {
+            const score = index.score(plan, section.terms, section.heading);
+            if (score !== undefined)
+                scored.push({ section, score: Math.round(score * 1000) / 1000 });
+        }
+        // Installed documentation precedes the wiki when both describe a command equally well.
+        const order = (left, right) => right.score - left.score ||
+            sourceOrder[left.section.source] - sourceOrder[right.section.source] ||
             compareCodeUnits(left.section.path, right.section.path) ||
-            left.section.startLine - right.section.startLine);
+            left.section.startLine - right.section.startLine;
+        scored.sort(order);
+        // Phrase adjacency is costlier to measure, so it only reorders the leading candidates.
+        const leading = scored.slice(0, PHRASE_CANDIDATES);
+        for (const entry of leading)
+            entry.score =
+                Math.round((entry.score + index.phraseBonus(plan, entry.section.lines)) * 1000) / 1000;
+        leading.sort(order);
+        scored.splice(0, leading.length, ...leading);
+        // An exact table row ranked above the section containing it is the sharper citation.
+        const rows = new Map();
+        const ranked = scored.filter(({ section }) => {
+            const above = rows.get(section.path) ?? [];
+            if (section.lines.length > 1 &&
+                above.some((line) => line > section.startLine && line < section.endLine + 1))
+                return false;
+            if (section.lines.length === 1)
+                rows.set(section.path, [...above, section.startLine]);
+            return true;
+        });
+        // The same command documented in several sources says little new after its first
+        // citation. Later copies wait behind distinct answers unless they match exactly.
+        const named = new Set();
+        const distinct = [];
+        const repeated = [];
+        for (const entry of ranked) {
+            const key = entry.section.heading.toLowerCase();
+            if (named.has(key) && entry.score < EXACT_HEADING_SCORE)
+                repeated.push(entry);
+            else
+                distinct.push(entry);
+            named.add(key);
+        }
+        ranked.splice(0, ranked.length, ...distinct, ...repeated);
+        return {
+            ranked,
+            describe: ({ section, score }) => {
+                const offset = index.bestLine(plan, section.lines);
+                return {
+                    section,
+                    score,
+                    excerpt: snippet(section.lines, offset),
+                    matchLine: section.startLine + offset,
+                };
+            },
+        };
     }
     async read(workspace, input, signal) {
-        const inventory = await this.inventory(workspace, signal);
+        let source = input.source;
+        if (source === undefined) {
+            for (const cached of this.#cache.values()) {
+                const section = cached.sections.find(({ id }) => id === input.id);
+                if (section !== undefined) {
+                    source = section.source;
+                    break;
+                }
+            }
+        }
+        const inventory = await this.inventory(workspace, signal, source === undefined ? undefined : [source]);
         const section = inventory.sections.find(({ id }) => id === input.id);
         if (section === undefined)
             throw new ServiceError('REFERENCE_SECTION_UNKNOWN', 'Reference section is not in this workspace');
@@ -373,7 +471,7 @@ export class ReferenceService {
                 currentRevision: section.revision,
             });
         const startLine = input.startLine ?? section.startLine;
-        const selected = readBoundedLines(section.lines, section.startLine, startLine, input.startColumn ?? 1, input.maxLines, 8_000);
+        const selected = readBoundedJsonLines(section.lines, section.startLine, startLine, input.startColumn ?? 1, input.maxLines, 8_000, 8_192);
         return {
             ...this.publicSection(section),
             sectionStartLine: section.startLine,
@@ -386,7 +484,8 @@ export class ReferenceService {
     async context(workspace, input, signal) {
         const inventory = await this.inventory(workspace, signal);
         const wanted = surfaceNames[input.surface];
-        const ranked = input.question === undefined ? [] : this.rank(inventory.sections, input.question);
+        const ranking = input.question === undefined ? undefined : this.rank(inventory.sections, input.question);
+        const ranked = ranking?.ranked ?? [];
         const required = [];
         const missing = [];
         const includeRequired = (name, section) => {
@@ -422,13 +521,23 @@ export class ReferenceService {
             return true;
         });
         const selected = prioritized.slice(0, input.limit);
+        const requestedCount = selected.length;
+        while (Buffer.byteLength(JSON.stringify(selected.map((section) => this.publicSection(section))), 'utf8') > MAX_REFERENCE_METADATA_BYTES &&
+            selected.length > 1)
+            selected.pop();
+        if (Buffer.byteLength(JSON.stringify(selected.map((section) => this.publicSection(section))), 'utf8') > MAX_REFERENCE_METADATA_BYTES)
+            throw new ServiceError('REFERENCE_RESPONSE_TOO_LARGE', 'Reference metadata exceeds the bounded reply');
         return {
             surface: input.surface,
-            sections: selected.map((section) => ({
-                ...this.publicSection(section),
-                excerpt: ranked.find((entry) => entry.section.id === section.id)?.excerpt ?? section.excerpt,
-            })),
-            omitted: Math.max(0, prioritized.length - input.limit),
+            sections: selected.map((section) => {
+                const match = ranked.find((entry) => entry.section.id === section.id);
+                if (match === undefined || ranking === undefined)
+                    return this.publicSection(section);
+                const { excerpt, matchLine } = ranking.describe(match);
+                return { ...this.publicSection(section), excerpt, matchLine };
+            }),
+            omitted: Math.max(0, prioritized.length - selected.length),
+            limitedByBytes: selected.length < requestedCount,
             omittedSources: required
                 .filter(({ section }) => !selected.some((entry) => entry.path === section.path && entry.source === section.source))
                 .map(({ name }) => name),
@@ -438,7 +547,6 @@ export class ReferenceService {
         };
     }
     publicSection(section) {
-        const { lines: _lines, ...publicSection } = section;
-        return publicSection;
+        return publicReferenceSection(section);
     }
 }