hoi4-agent-tools 3.7.0 → 3.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +59 -57
- package/dist/hoi4_agent_tools/core/configuration.js +2 -0
- package/dist/hoi4_agent_tools/core/job-store.js +19 -40
- package/dist/hoi4_agent_tools/core/job-worker-host.js +57 -8
- package/dist/hoi4_agent_tools/core/operation-tasks.js +2 -7
- package/dist/hoi4_agent_tools/core/scanner.js +3 -0
- package/dist/hoi4_agent_tools/core/shared-request-capacity.js +27 -13
- package/dist/hoi4_agent_tools/core/transactions.js +31 -2
- package/dist/hoi4_agent_tools/core/windows-sharing.js +42 -0
- package/dist/hoi4_agent_tools/core/workspace.js +114 -8
- package/dist/hoi4_agent_tools/event/artifact-validation.js +1 -1
- package/dist/hoi4_agent_tools/event/service.js +1 -1
- package/dist/hoi4_agent_tools/focus/service.js +2 -2
- package/dist/hoi4_agent_tools/gui/studio.js +16 -3
- package/dist/hoi4_agent_tools/map/model.js +81 -2
- package/dist/hoi4_agent_tools/map/service.js +1 -0
- package/dist/hoi4_agent_tools/map/validation.js +60 -4
- package/dist/hoi4_agent_tools/mcp/tools/reference.js +1 -1
- package/dist/hoi4_agent_tools/package-check/service.js +1 -1
- package/dist/hoi4_agent_tools/reference/ranking.js +309 -0
- package/dist/hoi4_agent_tools/reference/script-validation.js +3 -13
- package/dist/hoi4_agent_tools/reference/service.js +152 -116
- package/dist/hoi4_agent_tools/reference/source-locate.js +156 -0
- package/dist/hoi4_agent_tools/reference/source-lookup.js +37 -6
- package/dist/hoi4_agent_tools/reference/suggestions.js +38 -0
- package/dist/hoi4_agent_tools/schemas/reference.js +80 -2
- package/dist/hoi4_agent_tools/schemas/transaction.js +1 -1
- package/dist/hoi4_agent_tools/technology/service.js +1 -1
- package/dist/hoi4_agent_tools/version.js +1 -1
- package/docs/development.md +6 -0
- package/docs/examples.md +3 -3
- package/docs/map.md +10 -0
- package/docs/reference.md +5 -3
- package/docs/research/reference-benchmark.md +39 -0
- package/docs/setup.md +15 -0
- package/docs/tools.md +37 -37
- package/npm-shrinkwrap.json +18 -5
- package/package.json +2 -2
- package/schemas/configuration.schema.json +95 -88
- package/schemas/continuous-focus-palette.schema.json +1 -1
- package/schemas/custom-weighted-pool.schema.json +1 -1
- package/schemas/decision-inspect-input.schema.json +1 -1
- package/schemas/event-feature-manifest.schema.json +1 -1
- package/schemas/focus-plan.schema.json +1 -1
- package/schemas/focus-planning-sidecar.schema.json +1 -1
- package/schemas/gui-animation-source.schema.json +1 -1
- package/schemas/gui-helper.schema.json +1 -1
- package/schemas/gui-scenario.schema.json +1 -1
- package/schemas/helper-expansion-request.schema.json +1 -1
- package/schemas/helper-expansion-summary.schema.json +1 -1
- package/schemas/impact-inspect-input.schema.json +1 -1
- package/schemas/map-operation.schema.json +1 -1
- package/schemas/mechanic-test-input.schema.json +1 -1
- package/schemas/operation-result.schema.json +1 -1
- package/schemas/package-check-input.schema.json +1 -1
- package/schemas/package-manifest.schema.json +1 -1
- package/schemas/probability-analysis-result.schema.json +1 -1
- package/schemas/probability-compare-input.schema.json +1 -1
- package/schemas/probability-evaluate-input.schema.json +1 -1
- package/schemas/probability-inspect-input.schema.json +1 -1
- package/schemas/probability-render-input.schema.json +1 -1
- package/schemas/probability-scenario-set.schema.json +1 -1
- package/schemas/probability-sequence-input.schema.json +1 -1
- package/schemas/probability-simulate-input.schema.json +1 -1
- package/schemas/probability-sweep-input.schema.json +1 -1
- package/schemas/reference-context-input.schema.json +1 -1
- package/schemas/reference-read-input.schema.json +1 -1
- package/schemas/reference-search-input.schema.json +1 -1
- package/schemas/scenario-suite.schema.json +1 -1
- package/schemas/scenario-test-input.schema.json +1 -1
- package/schemas/script-validate-input.schema.json +1 -1
- package/schemas/shared-scenario.schema.json +1 -1
- package/schemas/source-lookup-input.schema.json +113 -50
- package/server.json +2 -2
|
@@ -7,11 +7,15 @@ import { isWithin } from '../core/workspace.js';
|
|
|
7
7
|
import { prepareSource, verifySource, readBoundedFile, } from '../core/scanner.js';
|
|
8
8
|
import { decodeSource } from '../core/source/encoding.js';
|
|
9
9
|
import { readBoundedJsonLines } from './lines.js';
|
|
10
|
+
import { RankingIndex, sectionTerms } from './ranking.js';
|
|
10
11
|
const MAX_FILES = 512;
|
|
11
12
|
const MAX_FILE_BYTES = 2_000_000;
|
|
12
13
|
const MAX_TOTAL_BYTES = 32_000_000;
|
|
13
14
|
const MAX_SECTIONS = 20_000;
|
|
14
15
|
const MAX_REFERENCE_METADATA_BYTES = 24_000;
|
|
16
|
+
const PHRASE_CANDIDATES = 60;
|
|
17
|
+
const EXACT_HEADING_SCORE = 1000;
|
|
18
|
+
const sourceOrder = { game_doc: 0, wiki: 1, script_doc: 2 };
|
|
15
19
|
function referenceText(bytes) {
|
|
16
20
|
if (bytes.length >= 2 &&
|
|
17
21
|
((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff))) {
|
|
@@ -24,68 +28,30 @@ function referenceText(bytes) {
|
|
|
24
28
|
}
|
|
25
29
|
return decodeSource(bytes).text;
|
|
26
30
|
}
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
'at',
|
|
32
|
-
'be',
|
|
33
|
-
'can',
|
|
34
|
-
'come',
|
|
35
|
-
'do',
|
|
36
|
-
'does',
|
|
37
|
-
'for',
|
|
38
|
-
'from',
|
|
39
|
-
'how',
|
|
40
|
-
'in',
|
|
41
|
-
'is',
|
|
42
|
-
'of',
|
|
43
|
-
'the',
|
|
44
|
-
'to',
|
|
45
|
-
'where',
|
|
46
|
-
'with',
|
|
47
|
-
'work',
|
|
48
|
-
]);
|
|
49
|
-
function normalizeTerm(term) {
|
|
50
|
-
if (term === 'dated')
|
|
51
|
-
return 'date';
|
|
52
|
-
if (term.endsWith('ies') && term.length > 5)
|
|
53
|
-
return `${term.slice(0, -3)}y`;
|
|
54
|
-
if (term.endsWith('ses') && term.length > 5)
|
|
55
|
-
return term.slice(0, -2);
|
|
56
|
-
if (term.endsWith('s') && term.length > 4)
|
|
57
|
-
return term.slice(0, -1);
|
|
58
|
-
return term;
|
|
31
|
+
/** The citation fields of a section, without its indexed text or term statistics. */
|
|
32
|
+
export function publicReferenceSection(section) {
|
|
33
|
+
const { lines: _lines, terms: _terms, ...citation } = section;
|
|
34
|
+
return citation;
|
|
59
35
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
.filter((term) => /[a-z0-9]/u.test(term) && !searchStopWords.has(term))),
|
|
65
|
-
];
|
|
66
|
-
}
|
|
67
|
-
function containsTerm(text, term) {
|
|
68
|
-
if (term.length > 2)
|
|
69
|
-
return text.includes(term);
|
|
70
|
-
const escaped = term.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
|
|
71
|
-
return new RegExp(`(^|[^a-z0-9_])${escaped}($|[^a-z0-9_])`, 'u').test(text);
|
|
72
|
-
}
|
|
73
|
-
function snippet(lines, terms = []) {
|
|
74
|
-
let matching = -1;
|
|
75
|
-
let best = -1;
|
|
76
|
-
for (const [index, line] of lines.entries()) {
|
|
77
|
-
const count = terms.filter((term) => containsTerm(line.toLowerCase(), term)).length;
|
|
78
|
-
if (count > best) {
|
|
79
|
-
matching = index;
|
|
80
|
-
best = count;
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
const selected = lines
|
|
84
|
-
.slice(Math.max(0, matching), Math.max(0, matching) + 3)
|
|
36
|
+
/** Up to three non-empty lines from `first`, trimmed to the 300-character excerpt bound. */
|
|
37
|
+
function snippet(lines, first = 0) {
|
|
38
|
+
return lines
|
|
39
|
+
.slice(first, first + 3)
|
|
85
40
|
.map((line) => line.trim())
|
|
86
41
|
.filter(Boolean)
|
|
87
|
-
.join(' ')
|
|
88
|
-
|
|
42
|
+
.join(' ')
|
|
43
|
+
.slice(0, 300);
|
|
44
|
+
}
|
|
45
|
+
// A Markdown table row whose first cell names a command, trigger, modifier, or on action.
|
|
46
|
+
const identifierRow = /^\|\s*`?([A-Za-z_][A-Za-z0-9_@.:]*)`?\s*\|/u;
|
|
47
|
+
const tableSeparator = /^\|\s*:?-{3,}/u;
|
|
48
|
+
/** A closing fence repeats the opening character at least as often and has no info string. */
|
|
49
|
+
function fenceMarker(line) {
|
|
50
|
+
const match = /^\s*(`{3,}|~{3,})(.*)$/u.exec(line);
|
|
51
|
+
if (match === null)
|
|
52
|
+
return undefined;
|
|
53
|
+
const marker = match[1];
|
|
54
|
+
return { character: marker[0], length: marker.length, info: match[2].trim() };
|
|
89
55
|
}
|
|
90
56
|
function sectionize(root, filePath, content, revision) {
|
|
91
57
|
const lines = content.replace(/^\uFEFF/u, '').split(/\r?\n/u);
|
|
@@ -96,15 +62,25 @@ function sectionize(root, filePath, content, revision) {
|
|
|
96
62
|
.replaceAll('_', ' ')
|
|
97
63
|
.replace(/ - Hearts of Iron 4 Wiki$/u, '');
|
|
98
64
|
const headings = [];
|
|
99
|
-
|
|
65
|
+
const rows = [];
|
|
66
|
+
let fence;
|
|
100
67
|
for (let index = 0; index < lines.length; index += 1) {
|
|
101
68
|
const line = lines[index];
|
|
102
|
-
|
|
103
|
-
|
|
69
|
+
const marker = fenceMarker(line);
|
|
70
|
+
if (fence === undefined && marker !== undefined) {
|
|
71
|
+
fence = marker;
|
|
104
72
|
continue;
|
|
105
73
|
}
|
|
106
|
-
if (
|
|
74
|
+
if (fence !== undefined) {
|
|
75
|
+
if (marker?.character === fence.character &&
|
|
76
|
+
marker.length >= fence.length &&
|
|
77
|
+
marker.info === '')
|
|
78
|
+
fence = undefined;
|
|
107
79
|
continue;
|
|
80
|
+
}
|
|
81
|
+
const row = identifierRow.exec(line);
|
|
82
|
+
if (row !== null && !tableSeparator.test(lines[index + 1] ?? ''))
|
|
83
|
+
rows.push({ heading: row[1], start: index });
|
|
108
84
|
const match = /^(#{1,6})\s+(.+?)\s*#*\s*$/u.exec(line);
|
|
109
85
|
if (match)
|
|
110
86
|
headings.push({ heading: match[2].replace(/<a\b[^>]*><\/a>/gu, '').trim(), start: index });
|
|
@@ -118,24 +94,30 @@ function sectionize(root, filePath, content, revision) {
|
|
|
118
94
|
}
|
|
119
95
|
if (headings.length === 0 || headings[0].start > 0)
|
|
120
96
|
headings.unshift({ heading: title, start: 0 });
|
|
121
|
-
|
|
122
|
-
const
|
|
123
|
-
const
|
|
124
|
-
const id = sha256Bytes(Buffer.from(`${root.kind}\0${relative}\0${current.start + 1}\0${current.heading}`, 'utf8'));
|
|
97
|
+
const section = (heading, start, end) => {
|
|
98
|
+
const sectionLines = lines.slice(start, end);
|
|
99
|
+
const id = sha256Bytes(Buffer.from(`${root.kind}\0${relative}\0${start + 1}\0${heading}`, 'utf8'));
|
|
125
100
|
return {
|
|
126
101
|
id,
|
|
127
102
|
revision,
|
|
128
103
|
source: root.kind,
|
|
129
104
|
title,
|
|
130
|
-
heading:
|
|
105
|
+
heading: heading.slice(0, 512),
|
|
131
106
|
path: filePath,
|
|
132
|
-
startLine:
|
|
107
|
+
startLine: start + 1,
|
|
133
108
|
endLine: end,
|
|
134
109
|
excerpt: snippet(sectionLines),
|
|
135
110
|
authority: root.authority,
|
|
136
111
|
lines: sectionLines,
|
|
112
|
+
terms: sectionTerms(title, heading, sectionLines),
|
|
137
113
|
};
|
|
138
|
-
}
|
|
114
|
+
};
|
|
115
|
+
// Heading sections come first, so a page's opening section still names the page. Table
|
|
116
|
+
// rows follow as one-line sections that cite one exact command or modifier entry.
|
|
117
|
+
return [
|
|
118
|
+
...headings.map((current, index) => section(current.heading, current.start, headings[index + 1]?.start ?? lines.length)),
|
|
119
|
+
...rows.map((row) => section(row.heading, row.start, row.start + 1)),
|
|
120
|
+
];
|
|
139
121
|
}
|
|
140
122
|
function sourceRoots(workspace) {
|
|
141
123
|
const gameRoot = workspace.gameRoot ?? (workspace.registration.kind === 'game' ? workspace.modRoot : undefined);
|
|
@@ -222,6 +204,7 @@ const gameDocNames = {
|
|
|
222
204
|
/** A workspace-private, read-only reference index over explicitly known local roots. */
|
|
223
205
|
export class ReferenceService {
|
|
224
206
|
#cache = new Map();
|
|
207
|
+
#indexes = new Map();
|
|
225
208
|
#cacheBytes = 0;
|
|
226
209
|
retain(key, value) {
|
|
227
210
|
this.#cacheBytes -= this.#cache.get(key)?.bytes ?? 0;
|
|
@@ -362,12 +345,11 @@ export class ReferenceService {
|
|
|
362
345
|
}
|
|
363
346
|
async search(workspace, input, signal) {
|
|
364
347
|
const inventory = await this.inventory(workspace, signal, input.sources);
|
|
365
|
-
const matches = this.rank(inventory.sections, input.query
|
|
366
|
-
const results = matches.slice(0, input.limit).map((
|
|
367
|
-
|
|
368
|
-
score,
|
|
369
|
-
|
|
370
|
-
}));
|
|
348
|
+
const { ranked: matches, describe } = this.rank(inventory.sections, input.query);
|
|
349
|
+
const results = matches.slice(0, input.limit).map((match) => {
|
|
350
|
+
const { section, score, excerpt, matchLine } = describe(match);
|
|
351
|
+
return { ...this.publicSection(section), score, excerpt, matchLine };
|
|
352
|
+
});
|
|
371
353
|
const requested = results.length;
|
|
372
354
|
while (Buffer.byteLength(JSON.stringify(results), 'utf8') > MAX_REFERENCE_METADATA_BYTES &&
|
|
373
355
|
results.length > 1)
|
|
@@ -383,40 +365,91 @@ export class ReferenceService {
|
|
|
383
365
|
skipped: inventory.skipped,
|
|
384
366
|
};
|
|
385
367
|
}
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
const
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
if (
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
368
|
+
/** One index per distinct set of section revisions; IDF depends on the whole inventory. */
|
|
369
|
+
rankingIndex(sections) {
|
|
370
|
+
const parts = [];
|
|
371
|
+
let previous;
|
|
372
|
+
for (const section of sections) {
|
|
373
|
+
if (previous?.path !== section.path || previous.revision !== section.revision)
|
|
374
|
+
parts.push(`${section.source}\0${section.path}\0${section.revision}`);
|
|
375
|
+
previous = section;
|
|
376
|
+
}
|
|
377
|
+
const key = sha256Bytes(Buffer.from(parts.join('\n'), 'utf8'));
|
|
378
|
+
const cached = this.#indexes.get(key);
|
|
379
|
+
if (cached !== undefined) {
|
|
380
|
+
this.#indexes.delete(key);
|
|
381
|
+
this.#indexes.set(key, cached);
|
|
382
|
+
return cached;
|
|
383
|
+
}
|
|
384
|
+
const index = new RankingIndex(sections.map(({ terms }) => terms));
|
|
385
|
+
this.#indexes.set(key, index);
|
|
386
|
+
while (this.#indexes.size > 4) {
|
|
387
|
+
const oldest = this.#indexes.keys().next().value;
|
|
388
|
+
if (oldest === undefined)
|
|
389
|
+
break;
|
|
390
|
+
this.#indexes.delete(oldest);
|
|
391
|
+
}
|
|
392
|
+
return index;
|
|
393
|
+
}
|
|
394
|
+
rank(sections, question) {
|
|
395
|
+
const index = this.rankingIndex(sections);
|
|
396
|
+
const plan = index.plan(question);
|
|
397
|
+
const scored = [];
|
|
398
|
+
for (const section of sections) {
|
|
399
|
+
const score = index.score(plan, section.terms, section.heading);
|
|
400
|
+
if (score !== undefined)
|
|
401
|
+
scored.push({ section, score: Math.round(score * 1000) / 1000 });
|
|
402
|
+
}
|
|
403
|
+
// Installed documentation precedes the wiki when both describe a command equally well.
|
|
404
|
+
const order = (left, right) => right.score - left.score ||
|
|
405
|
+
sourceOrder[left.section.source] - sourceOrder[right.section.source] ||
|
|
418
406
|
compareCodeUnits(left.section.path, right.section.path) ||
|
|
419
|
-
left.section.startLine - right.section.startLine
|
|
407
|
+
left.section.startLine - right.section.startLine;
|
|
408
|
+
scored.sort(order);
|
|
409
|
+
// Phrase adjacency is costlier to measure, so it only reorders the leading candidates.
|
|
410
|
+
const leading = scored.slice(0, PHRASE_CANDIDATES);
|
|
411
|
+
for (const entry of leading)
|
|
412
|
+
entry.score =
|
|
413
|
+
Math.round((entry.score + index.phraseBonus(plan, entry.section.lines)) * 1000) / 1000;
|
|
414
|
+
leading.sort(order);
|
|
415
|
+
scored.splice(0, leading.length, ...leading);
|
|
416
|
+
// An exact table row ranked above the section containing it is the sharper citation.
|
|
417
|
+
const rows = new Map();
|
|
418
|
+
const ranked = scored.filter(({ section }) => {
|
|
419
|
+
const above = rows.get(section.path) ?? [];
|
|
420
|
+
if (section.lines.length > 1 &&
|
|
421
|
+
above.some((line) => line > section.startLine && line < section.endLine + 1))
|
|
422
|
+
return false;
|
|
423
|
+
if (section.lines.length === 1)
|
|
424
|
+
rows.set(section.path, [...above, section.startLine]);
|
|
425
|
+
return true;
|
|
426
|
+
});
|
|
427
|
+
// The same command documented in several sources says little new after its first
|
|
428
|
+
// citation. Later copies wait behind distinct answers unless they match exactly.
|
|
429
|
+
const named = new Set();
|
|
430
|
+
const distinct = [];
|
|
431
|
+
const repeated = [];
|
|
432
|
+
for (const entry of ranked) {
|
|
433
|
+
const key = entry.section.heading.toLowerCase();
|
|
434
|
+
if (named.has(key) && entry.score < EXACT_HEADING_SCORE)
|
|
435
|
+
repeated.push(entry);
|
|
436
|
+
else
|
|
437
|
+
distinct.push(entry);
|
|
438
|
+
named.add(key);
|
|
439
|
+
}
|
|
440
|
+
ranked.splice(0, ranked.length, ...distinct, ...repeated);
|
|
441
|
+
return {
|
|
442
|
+
ranked,
|
|
443
|
+
describe: ({ section, score }) => {
|
|
444
|
+
const offset = index.bestLine(plan, section.lines);
|
|
445
|
+
return {
|
|
446
|
+
section,
|
|
447
|
+
score,
|
|
448
|
+
excerpt: snippet(section.lines, offset),
|
|
449
|
+
matchLine: section.startLine + offset,
|
|
450
|
+
};
|
|
451
|
+
},
|
|
452
|
+
};
|
|
420
453
|
}
|
|
421
454
|
async read(workspace, input, signal) {
|
|
422
455
|
let source = input.source;
|
|
@@ -451,7 +484,8 @@ export class ReferenceService {
|
|
|
451
484
|
async context(workspace, input, signal) {
|
|
452
485
|
const inventory = await this.inventory(workspace, signal);
|
|
453
486
|
const wanted = surfaceNames[input.surface];
|
|
454
|
-
const
|
|
487
|
+
const ranking = input.question === undefined ? undefined : this.rank(inventory.sections, input.question);
|
|
488
|
+
const ranked = ranking?.ranked ?? [];
|
|
455
489
|
const required = [];
|
|
456
490
|
const missing = [];
|
|
457
491
|
const includeRequired = (name, section) => {
|
|
@@ -495,10 +529,13 @@ export class ReferenceService {
|
|
|
495
529
|
throw new ServiceError('REFERENCE_RESPONSE_TOO_LARGE', 'Reference metadata exceeds the bounded reply');
|
|
496
530
|
return {
|
|
497
531
|
surface: input.surface,
|
|
498
|
-
sections: selected.map((section) =>
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
532
|
+
sections: selected.map((section) => {
|
|
533
|
+
const match = ranked.find((entry) => entry.section.id === section.id);
|
|
534
|
+
if (match === undefined || ranking === undefined)
|
|
535
|
+
return this.publicSection(section);
|
|
536
|
+
const { excerpt, matchLine } = ranking.describe(match);
|
|
537
|
+
return { ...this.publicSection(section), excerpt, matchLine };
|
|
538
|
+
}),
|
|
502
539
|
omitted: Math.max(0, prioritized.length - selected.length),
|
|
503
540
|
limitedByBytes: selected.length < requestedCount,
|
|
504
541
|
omittedSources: required
|
|
@@ -510,7 +547,6 @@ export class ReferenceService {
|
|
|
510
547
|
};
|
|
511
548
|
}
|
|
512
549
|
publicSection(section) {
|
|
513
|
-
|
|
514
|
-
return publicSection;
|
|
550
|
+
return publicReferenceSection(section);
|
|
515
551
|
}
|
|
516
552
|
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Source position lookup: map a file and line to the indexed definitions that contain it and
|
|
3
|
+
* to the exact chain of Clausewitz assignments enclosing it.
|
|
4
|
+
*
|
|
5
|
+
* The returned `keyPath` is relative to the innermost containing definition, so an agent can
|
|
6
|
+
* pass that definition's `id`, `kind`, and `keyPath` to `hoi4.source_lookup` navigation. The
|
|
7
|
+
* chain is derived from parser nodes, preserving repeated keys through zero-based occurrences.
|
|
8
|
+
*/
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
import { ServiceError } from '../core/result.js';
|
|
11
|
+
import { isWithin } from '../core/workspace.js';
|
|
12
|
+
import { positionAt } from '../core/source/lexer.js';
|
|
13
|
+
import { parseClausewitz } from '../core/source/parser.js';
|
|
14
|
+
const MAX_OWNERS = 8;
|
|
15
|
+
const MAX_CHAIN = 24;
|
|
16
|
+
const MAX_KEY_BYTES = 128;
|
|
17
|
+
const MAX_LINE_TEXT = 300;
|
|
18
|
+
function samePath(left, right) {
|
|
19
|
+
return process.platform === 'win32' ? left.toLowerCase() === right.toLowerCase() : left === right;
|
|
20
|
+
}
|
|
21
|
+
/** Resolve a display, relative, or absolute path to one scanned file, preferring the active layer. */
|
|
22
|
+
function selectFile(snapshot, requested) {
|
|
23
|
+
const normalized = requested.replaceAll('\\', '/').replace(/^\.\//u, '');
|
|
24
|
+
const absolute = path.isAbsolute(requested) ? path.resolve(requested) : undefined;
|
|
25
|
+
const candidates = [...snapshot.index.files.values()].filter((file) => file.displayPath === normalized ||
|
|
26
|
+
file.relativePath.replaceAll('\\', '/') === normalized ||
|
|
27
|
+
(absolute !== undefined && samePath(path.resolve(file.absolutePath), absolute)));
|
|
28
|
+
if (candidates.length === 0)
|
|
29
|
+
throw new ServiceError('SOURCE_FILE_NOT_INDEXED', 'No indexed source file matches the path; use a mod-relative path such as events/example.txt');
|
|
30
|
+
return candidates.find(({ shadowedBy }) => shadowedBy === undefined) ?? candidates[0];
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Scan only the requested file's folder family in every layer. Definitions and same-path
|
|
34
|
+
* overrides of a source position live beside it, so the whole workspace index is unnecessary.
|
|
35
|
+
*/
|
|
36
|
+
export function locatePatterns(engine, workspaceId, requested, principal) {
|
|
37
|
+
let relative = requested.replaceAll('\\', '/').replace(/^\.\//u, '');
|
|
38
|
+
if (path.isAbsolute(requested)) {
|
|
39
|
+
const absolute = path.resolve(requested);
|
|
40
|
+
const root = engine.resolver
|
|
41
|
+
.get(workspaceId, principal)
|
|
42
|
+
.roots.find((candidate) => isWithin(candidate.path, absolute));
|
|
43
|
+
if (root === undefined)
|
|
44
|
+
return undefined;
|
|
45
|
+
relative = path.relative(root.path, absolute).replaceAll('\\', '/');
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
const label = /^[A-Za-z0-9_.-]+:(?!\/)(.+)$/u.exec(relative);
|
|
49
|
+
if (label !== null)
|
|
50
|
+
relative = label[1];
|
|
51
|
+
}
|
|
52
|
+
const extension = path.posix.extname(relative);
|
|
53
|
+
const directory = path.posix.dirname(relative);
|
|
54
|
+
if (relative.length === 0 ||
|
|
55
|
+
extension.length === 0 ||
|
|
56
|
+
/[*?[\]{}!]/u.test(relative) ||
|
|
57
|
+
relative.split('/').includes('..'))
|
|
58
|
+
return undefined;
|
|
59
|
+
return [directory === '.' ? `*${extension}` : `${directory}/**/*${extension}`];
|
|
60
|
+
}
|
|
61
|
+
function entryValue(entry) {
|
|
62
|
+
return entry.type === 'assignment' ? entry.value : entry;
|
|
63
|
+
}
|
|
64
|
+
export function locateSource(snapshot, requestedPath, line, requestedColumn) {
|
|
65
|
+
const file = selectFile(snapshot, requestedPath);
|
|
66
|
+
const document = parseClausewitz(file.bytes, file.displayPath);
|
|
67
|
+
const starts = document.lineIndex.lineStarts;
|
|
68
|
+
if (line > starts.length)
|
|
69
|
+
throw new ServiceError('SOURCE_LINE_OUTSIDE_FILE', 'Requested line is beyond the end of the file', {
|
|
70
|
+
lines: starts.length,
|
|
71
|
+
});
|
|
72
|
+
const lineStart = starts[line - 1];
|
|
73
|
+
const lineEnd = line < starts.length ? starts[line] : document.text.length;
|
|
74
|
+
const lineText = document.text.slice(lineStart, lineEnd).replace(/\r?\n$/u, '');
|
|
75
|
+
// Without a column, land on the first non-blank character so an indented line selects its
|
|
76
|
+
// own assignment rather than the enclosing block.
|
|
77
|
+
const column = requestedColumn ?? Math.max(1, lineText.search(/\S/u) + 1);
|
|
78
|
+
if (column > lineText.length + 1)
|
|
79
|
+
throw new ServiceError('SOURCE_COLUMN_OUTSIDE_LINE', 'Requested column is beyond the end of the line');
|
|
80
|
+
const offset = lineStart + column - 1;
|
|
81
|
+
const owners = snapshot.index.symbols
|
|
82
|
+
.filter((symbol) => symbol.path === file.displayPath &&
|
|
83
|
+
symbol.location !== undefined &&
|
|
84
|
+
symbol.location.start.offset <= offset &&
|
|
85
|
+
offset < Math.max(symbol.location.end.offset, symbol.location.start.offset + 1))
|
|
86
|
+
.sort((left, right) => left.location.end.offset -
|
|
87
|
+
left.location.start.offset -
|
|
88
|
+
(right.location.end.offset - right.location.start.offset) ||
|
|
89
|
+
right.location.start.offset - left.location.start.offset);
|
|
90
|
+
const structureAvailable = !document.diagnostics.some(({ severity }) => severity === 'error' || severity === 'blocker');
|
|
91
|
+
const steps = [];
|
|
92
|
+
if (structureAvailable) {
|
|
93
|
+
let block = document.root;
|
|
94
|
+
while (block !== undefined) {
|
|
95
|
+
const index = block.entries.findIndex((entry) => entry.start <= offset && offset < Math.max(entry.end, entry.start + 1));
|
|
96
|
+
if (index < 0)
|
|
97
|
+
break;
|
|
98
|
+
const entry = block.entries[index];
|
|
99
|
+
const key = entry.type === 'assignment' ? entry.key.value : null;
|
|
100
|
+
const occurrence = key === null
|
|
101
|
+
? 0
|
|
102
|
+
: block.entries
|
|
103
|
+
.slice(0, index)
|
|
104
|
+
.filter((candidate) => candidate.type === 'assignment' && candidate.key.value === key)
|
|
105
|
+
.length;
|
|
106
|
+
steps.push({ entry, key, occurrence, index });
|
|
107
|
+
const value = entryValue(entry);
|
|
108
|
+
block = value.type === 'block' && value !== block ? value : undefined;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
const lineOf = (position) => positionAt(document.lineIndex, position).line;
|
|
112
|
+
const chain = steps.slice(0, MAX_CHAIN).map(({ entry, key, occurrence, index }) => ({
|
|
113
|
+
key: key !== null && Buffer.byteLength(key, 'utf8') <= MAX_KEY_BYTES ? key : null,
|
|
114
|
+
occurrence,
|
|
115
|
+
index,
|
|
116
|
+
startLine: lineOf(entry.start),
|
|
117
|
+
endLine: lineOf(Math.max(entry.start, entry.end - 1)),
|
|
118
|
+
}));
|
|
119
|
+
// The key path continues from the innermost definition that is itself a chain node.
|
|
120
|
+
let keyPath = null;
|
|
121
|
+
if (structureAvailable)
|
|
122
|
+
for (const owner of owners) {
|
|
123
|
+
const anchor = steps.findIndex(({ entry }) => entry.start === owner.location.start.offset && entry.end === owner.location.end.offset);
|
|
124
|
+
if (anchor < 0)
|
|
125
|
+
continue;
|
|
126
|
+
keyPath = steps
|
|
127
|
+
.slice(anchor + 1)
|
|
128
|
+
.map(({ key, occurrence, index }) => key !== null && Buffer.byteLength(key, 'utf8') <= MAX_KEY_BYTES
|
|
129
|
+
? { key, occurrence }
|
|
130
|
+
: { index });
|
|
131
|
+
break;
|
|
132
|
+
}
|
|
133
|
+
let text = lineText.slice(0, MAX_LINE_TEXT);
|
|
134
|
+
if (/[\uD800-\uDBFF]$/u.test(text))
|
|
135
|
+
text = text.slice(0, -1);
|
|
136
|
+
return {
|
|
137
|
+
path: file.displayPath,
|
|
138
|
+
rootKind: file.rootKind,
|
|
139
|
+
loadOrder: file.loadOrder,
|
|
140
|
+
shadowed: file.shadowedBy !== undefined,
|
|
141
|
+
line,
|
|
142
|
+
column,
|
|
143
|
+
text,
|
|
144
|
+
owners: owners.slice(0, MAX_OWNERS).map((owner) => ({
|
|
145
|
+
kind: owner.kind,
|
|
146
|
+
id: owner.id.slice(0, 1024),
|
|
147
|
+
startLine: owner.location.start.line,
|
|
148
|
+
endLine: owner.location.end.line,
|
|
149
|
+
overridden: owner.overridden,
|
|
150
|
+
})),
|
|
151
|
+
chain,
|
|
152
|
+
chainTruncated: steps.length > chain.length,
|
|
153
|
+
keyPath: keyPath === null || keyPath.length > MAX_CHAIN ? null : keyPath,
|
|
154
|
+
structureAvailable,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
@@ -4,6 +4,8 @@ import { readBoundedJsonLines } from './lines.js';
|
|
|
4
4
|
import { scanImpactSemanticReferences } from '../core/impact-semantic.js';
|
|
5
5
|
import { parseClausewitz } from '../core/source/parser.js';
|
|
6
6
|
import { navigateSource } from './source-navigation.js';
|
|
7
|
+
import { locatePatterns, locateSource } from './source-locate.js';
|
|
8
|
+
import { nearestNames } from './suggestions.js';
|
|
7
9
|
const semanticCache = new WeakMap();
|
|
8
10
|
/** Exact source definitions and usage over the existing shared Clausewitz index. */
|
|
9
11
|
export async function sourceLookup(engine, workspaceId, input, principal, signal) {
|
|
@@ -25,16 +27,38 @@ export async function sourceLookup(engine, workspaceId, input, principal, signal
|
|
|
25
27
|
'common/scripted_guis/**/*.txt',
|
|
26
28
|
'common/on_actions/**/*.txt',
|
|
27
29
|
]
|
|
28
|
-
: input.
|
|
29
|
-
?
|
|
30
|
-
:
|
|
30
|
+
: input.path !== undefined
|
|
31
|
+
? locatePatterns(engine, workspaceId, input.path, principal)
|
|
32
|
+
: input.includeReferences || input.kind === undefined
|
|
33
|
+
? undefined
|
|
34
|
+
: narrowPatterns[input.kind];
|
|
31
35
|
const snapshot = await engine.scan(workspaceId, patterns === undefined ? {} : { patterns }, principal, signal);
|
|
36
|
+
if (input.path !== undefined && input.line !== undefined)
|
|
37
|
+
return {
|
|
38
|
+
sourceScope: 'source_location',
|
|
39
|
+
revision: snapshot.revision,
|
|
40
|
+
complete: snapshot.complete,
|
|
41
|
+
skippedSourceCount: snapshot.skippedSourceCount,
|
|
42
|
+
definitionCount: 0,
|
|
43
|
+
definitionsTruncated: false,
|
|
44
|
+
referencesIncluded: false,
|
|
45
|
+
referenceCount: 0,
|
|
46
|
+
referencesTruncated: false,
|
|
47
|
+
referencesComplete: false,
|
|
48
|
+
unresolvedReferenceCount: 0,
|
|
49
|
+
location: locateSource(snapshot, input.path, input.line, input.column),
|
|
50
|
+
definitions: [],
|
|
51
|
+
references: [],
|
|
52
|
+
};
|
|
53
|
+
const symbol = input.symbol;
|
|
54
|
+
if (symbol === undefined)
|
|
55
|
+
throw new ServiceError('SOURCE_LOOKUP_TARGET_REQUIRED', 'Provide a symbol, or path and line');
|
|
32
56
|
if (input.expectedRevision !== undefined && input.expectedRevision !== snapshot.revision) {
|
|
33
57
|
throw new ServiceError('SOURCE_REVISION_STALE', 'Indexed source changed; look up the symbol again', { currentRevision: snapshot.revision });
|
|
34
58
|
}
|
|
35
59
|
const navigating = input.keyPath !== undefined || input.view === 'structure';
|
|
36
60
|
const matchedDefinitions = snapshot.index.symbols
|
|
37
|
-
.filter((entry) => entry.id ===
|
|
61
|
+
.filter((entry) => entry.id === symbol && (input.kind === undefined || entry.kind === input.kind))
|
|
38
62
|
.sort((left, right) => Number(left.overridden) - Number(right.overridden) || right.loadOrder - left.loadOrder);
|
|
39
63
|
if (navigating && new Set(matchedDefinitions.map(({ kind }) => kind)).size > 1)
|
|
40
64
|
throw new ServiceError('SOURCE_KIND_AMBIGUOUS', 'Nested navigation requires kind when the identifier has multiple symbol kinds');
|
|
@@ -108,7 +132,7 @@ export async function sourceLookup(engine, workspaceId, input, principal, signal
|
|
|
108
132
|
const referenceIdentities = new Set();
|
|
109
133
|
const matchedReferences = input.includeReferences
|
|
110
134
|
? [...snapshot.index.references, ...(semantic?.references ?? [])]
|
|
111
|
-
.filter((entry) => entry.to ===
|
|
135
|
+
.filter((entry) => entry.to === symbol && (input.kind === undefined || entry.toKind === input.kind))
|
|
112
136
|
.filter((entry) => {
|
|
113
137
|
const key = `${entry.path}:${entry.location?.start.offset ?? 0}:${entry.toKind}:${entry.to}:${entry.kind}`;
|
|
114
138
|
if (referenceIdentities.has(key))
|
|
@@ -135,8 +159,15 @@ export async function sourceLookup(engine, workspaceId, input, principal, signal
|
|
|
135
159
|
revision: snapshot.revision,
|
|
136
160
|
complete: snapshot.complete,
|
|
137
161
|
skippedSourceCount: snapshot.skippedSourceCount,
|
|
138
|
-
definitionCount: snapshot.index.symbols.filter((entry) => entry.id ===
|
|
162
|
+
definitionCount: snapshot.index.symbols.filter((entry) => entry.id === symbol && (input.kind === undefined || entry.kind === input.kind)).length,
|
|
139
163
|
definitionsTruncated: matchedDefinitions.length > definitions.length,
|
|
164
|
+
...(matchedDefinitions.length === 0
|
|
165
|
+
? {
|
|
166
|
+
suggestions: nearestNames(symbol, new Set(snapshot.index.symbols
|
|
167
|
+
.filter((entry) => input.kind === undefined || entry.kind === input.kind)
|
|
168
|
+
.map(({ id }) => id))),
|
|
169
|
+
}
|
|
170
|
+
: {}),
|
|
140
171
|
definitions,
|
|
141
172
|
references,
|
|
142
173
|
referencesIncluded: input.includeReferences,
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/** Near-miss identifier suggestions shared by source lookup and script checks. */
|
|
2
|
+
const MAX_DISTANCE = 3;
|
|
3
|
+
/** Levenshtein distance, cut off above three edits because larger distances are not suggested. */
|
|
4
|
+
export function editDistance(a, b) {
|
|
5
|
+
let row = Array.from({ length: b.length + 1 }, (_, index) => index);
|
|
6
|
+
for (let i = 0; i < a.length; i++) {
|
|
7
|
+
const next = [i + 1];
|
|
8
|
+
for (let j = 0; j < b.length; j++)
|
|
9
|
+
next.push(Math.min(next[j] + 1, row[j + 1] + 1, row[j] + Number(a[i] !== b[j])));
|
|
10
|
+
if (Math.min(...next) > MAX_DISTANCE)
|
|
11
|
+
return MAX_DISTANCE + 1;
|
|
12
|
+
row = next;
|
|
13
|
+
}
|
|
14
|
+
return row[b.length];
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The closest candidates within three edits, nearest first. `budget` bounds the number of
|
|
18
|
+
* distance computations, so a very large index cannot make a missed lookup expensive.
|
|
19
|
+
*/
|
|
20
|
+
export function nearestNames(target, candidates, options = {}) {
|
|
21
|
+
if (target.length > 128)
|
|
22
|
+
return [];
|
|
23
|
+
let budget = options.budget ?? 20_000;
|
|
24
|
+
const scored = [];
|
|
25
|
+
for (const candidate of candidates) {
|
|
26
|
+
if (candidate === target || Math.abs(candidate.length - target.length) > MAX_DISTANCE)
|
|
27
|
+
continue;
|
|
28
|
+
if (budget-- <= 0)
|
|
29
|
+
break;
|
|
30
|
+
const distance = editDistance(target, candidate);
|
|
31
|
+
if (distance <= MAX_DISTANCE)
|
|
32
|
+
scored.push({ candidate, distance });
|
|
33
|
+
}
|
|
34
|
+
return scored
|
|
35
|
+
.sort((a, b) => a.distance - b.distance || a.candidate.localeCompare(b.candidate, 'en'))
|
|
36
|
+
.slice(0, options.limit ?? 5)
|
|
37
|
+
.map(({ candidate }) => candidate);
|
|
38
|
+
}
|