@aeonsparx/pi-unreal 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -0
- package/LICENSE +21 -0
- package/README.md +199 -2
- package/docs/internals.md +245 -0
- package/extensions/unreal/index.ts +696 -0
- package/extensions/unreal/lib/cache.ts +190 -0
- package/extensions/unreal/lib/compiledb.ts +491 -0
- package/extensions/unreal/lib/descriptors.ts +631 -0
- package/extensions/unreal/lib/engine-resolver.ts +443 -0
- package/extensions/unreal/lib/pi-tools.ts +452 -0
- package/extensions/unreal/lib/plugin-resolver.ts +305 -0
- package/extensions/unreal/lib/query.ts +272 -0
- package/extensions/unreal/lib/registry.ts +208 -0
- package/extensions/unreal/lib/scan.ts +510 -0
- package/extensions/unreal/lib/store.ts +1490 -0
- package/extensions/unreal/lib/tools.ts +1074 -0
- package/extensions/unreal/lib/ubt.ts +380 -0
- package/extensions/unreal/lib/util.ts +210 -0
- package/package.json +60 -4
- package/prompts/unreal-context.md +36 -0
- package/skills/ue-engine-resolution/SKILL.md +129 -0
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Disk cache for plugin indexes.
|
|
3
|
+
*
|
|
4
|
+
* A cold scan on a real install is expensive (UE 5.8: ~890 `.uplugin` under `Engine/Plugins`,
|
|
5
|
+
* 10k+ engine headers, 2400+ `.Build.cs`), so indexes are persisted under
|
|
6
|
+
* `~/.pi/agent/unreal-cache/`. Entries are keyed by engine root + `Build.version` changelist +
|
|
7
|
+
* the mtime of every scanned directory, so a plugin add/remove (which moves a directory mtime)
|
|
8
|
+
* forces a rebuild. A pure content edit of an existing `.uplugin` does not move the parent
|
|
9
|
+
* directory mtime, which is why `MAX_AGE_MS` exists as the backstop.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
13
|
+
import { homedir } from "node:os";
|
|
14
|
+
import { joinPath, normalizePath, realFs, shortHash, type FsAdapter } from "./util.ts";
|
|
15
|
+
import type { PluginIndex, PluginSummary } from "./plugin-resolver.ts";
|
|
16
|
+
import { comparePluginSource } from "./plugin-resolver.ts";
|
|
17
|
+
|
|
18
|
+
const SCHEMA_VERSION = 2;
|
|
19
|
+
/** Backstop for content-only edits, which do not move a directory mtime. */
|
|
20
|
+
export const MAX_AGE_MS = 10 * 60 * 1000;
|
|
21
|
+
|
|
22
|
+
export interface CacheRootState {
|
|
23
|
+
dir: string;
|
|
24
|
+
mtimeMs: number;
|
|
25
|
+
exists: boolean;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface CachedIndexFile {
|
|
29
|
+
schema: number;
|
|
30
|
+
key: string;
|
|
31
|
+
builtAtMs: number;
|
|
32
|
+
engineRoot?: string;
|
|
33
|
+
projectDir?: string;
|
|
34
|
+
additionalDirs: string[];
|
|
35
|
+
scannedDirs: string[];
|
|
36
|
+
scannedFiles: number;
|
|
37
|
+
roots: CacheRootState[];
|
|
38
|
+
entries: PluginSummary[];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function cacheDir(fs: FsAdapter = realFs): string {
|
|
42
|
+
const base = process.env.PI_UNREAL_CACHE_DIR ?? joinPath(homedir(), ".pi/agent/unreal-cache");
|
|
43
|
+
const dir = normalizePath(base);
|
|
44
|
+
if (!fs.isDirectory(dir)) {
|
|
45
|
+
try {
|
|
46
|
+
mkdirSync(dir, { recursive: true });
|
|
47
|
+
} catch {
|
|
48
|
+
/* best-effort; a cache miss is an acceptable outcome */
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return dir;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function cacheFilePath(key: string, fs: FsAdapter = realFs): string {
|
|
55
|
+
return joinPath(cacheDir(fs), `${key}.json`);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function rootState(dir: string, fs: FsAdapter): CacheRootState {
|
|
59
|
+
try {
|
|
60
|
+
return { dir, mtimeMs: statSync(dir).mtimeMs, exists: true };
|
|
61
|
+
} catch {
|
|
62
|
+
return { dir, mtimeMs: 0, exists: false };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Key that changes when the index must be rebuilt: engine identity + engine version +
|
|
68
|
+
* mtime of each scan root.
|
|
69
|
+
*/
|
|
70
|
+
/** A scanned directory, given either bare or as the `{ dir, source }` record from `computeScanDirs`. */
|
|
71
|
+
export type CacheScanDir = string | { dir: string; source?: string };
|
|
72
|
+
|
|
73
|
+
export function computeCacheKey(input: {
|
|
74
|
+
engineRoot?: string;
|
|
75
|
+
projectDir?: string;
|
|
76
|
+
scannedDirs: CacheScanDir[];
|
|
77
|
+
versionChangelist?: number | string;
|
|
78
|
+
fs?: FsAdapter;
|
|
79
|
+
}): { key: string; roots: CacheRootState[] } {
|
|
80
|
+
const fs = input.fs ?? realFs;
|
|
81
|
+
const roots = input.scannedDirs.map((entry) => rootState(typeof entry === "string" ? entry : entry.dir, fs));
|
|
82
|
+
const parts = [
|
|
83
|
+
`engine=${input.engineRoot ?? "-"}`,
|
|
84
|
+
`project=${input.projectDir ?? "-"}`,
|
|
85
|
+
`changelist=${input.versionChangelist ?? "-"}`,
|
|
86
|
+
...roots.map((root) => `${root.dir}@${root.mtimeMs.toFixed(0)}`),
|
|
87
|
+
];
|
|
88
|
+
return { key: shortHash(parts.join("|")), roots };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export function loadCachedIndex(key: string, fs: FsAdapter = realFs): PluginIndex | undefined {
|
|
92
|
+
const path = cacheFilePath(key, fs);
|
|
93
|
+
const text = fs.readText(path);
|
|
94
|
+
if (!text) return undefined;
|
|
95
|
+
let parsed: CachedIndexFile;
|
|
96
|
+
try {
|
|
97
|
+
parsed = JSON.parse(text) as CachedIndexFile;
|
|
98
|
+
} catch {
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
if (parsed.schema !== SCHEMA_VERSION || parsed.key !== key) return undefined;
|
|
102
|
+
if (Date.now() - parsed.builtAtMs > MAX_AGE_MS) return undefined;
|
|
103
|
+
for (const root of parsed.roots) {
|
|
104
|
+
const current = rootState(root.dir, fs);
|
|
105
|
+
if (current.exists !== root.exists) return undefined;
|
|
106
|
+
if (current.exists && current.mtimeMs !== root.mtimeMs) return undefined;
|
|
107
|
+
}
|
|
108
|
+
return rebuildIndex(parsed);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function rebuildIndex(parsed: CachedIndexFile): PluginIndex {
|
|
112
|
+
const byName = new Map<string, PluginSummary[]>();
|
|
113
|
+
for (const entry of parsed.entries) {
|
|
114
|
+
const key = entry.name.toLowerCase();
|
|
115
|
+
const bucket = byName.get(key);
|
|
116
|
+
if (bucket) bucket.push(entry);
|
|
117
|
+
else byName.set(key, [entry]);
|
|
118
|
+
}
|
|
119
|
+
// A cached index must sort its buckets exactly like a freshly built one, otherwise the
|
|
120
|
+
// shadow winner changes between a cache miss and a cache hit.
|
|
121
|
+
for (const bucket of byName.values()) {
|
|
122
|
+
bucket.sort(comparePluginSource);
|
|
123
|
+
}
|
|
124
|
+
return {
|
|
125
|
+
engineRoot: parsed.engineRoot,
|
|
126
|
+
projectDir: parsed.projectDir,
|
|
127
|
+
additionalDirs: parsed.additionalDirs,
|
|
128
|
+
scannedDirs: parsed.scannedDirs,
|
|
129
|
+
scannedFiles: parsed.scannedFiles,
|
|
130
|
+
entries: parsed.entries,
|
|
131
|
+
byName,
|
|
132
|
+
builtAtMs: parsed.builtAtMs,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export function saveCachedIndex(index: PluginIndex, key: string, roots: CacheRootState[], fs: FsAdapter = realFs): boolean {
|
|
137
|
+
const payload: CachedIndexFile = {
|
|
138
|
+
schema: SCHEMA_VERSION,
|
|
139
|
+
key,
|
|
140
|
+
builtAtMs: index.builtAtMs,
|
|
141
|
+
engineRoot: index.engineRoot,
|
|
142
|
+
projectDir: index.projectDir,
|
|
143
|
+
additionalDirs: index.additionalDirs,
|
|
144
|
+
scannedDirs: index.scannedDirs,
|
|
145
|
+
scannedFiles: index.scannedFiles,
|
|
146
|
+
roots,
|
|
147
|
+
entries: index.entries,
|
|
148
|
+
};
|
|
149
|
+
try {
|
|
150
|
+
writeFileSync(cacheFilePath(key, fs), JSON.stringify(payload), "utf8");
|
|
151
|
+
return true;
|
|
152
|
+
} catch {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Removes every cached index. Exposed as `/unreal-cache clear`. */
|
|
158
|
+
export function clearCache(fs: FsAdapter = realFs): { dir: string; removed: number } {
|
|
159
|
+
const dir = cacheDir(fs);
|
|
160
|
+
let removed = 0;
|
|
161
|
+
for (const entry of fs.listDir(dir)) {
|
|
162
|
+
if (!entry.endsWith(".json")) continue;
|
|
163
|
+
try {
|
|
164
|
+
rmSync(joinPath(dir, entry), { force: true });
|
|
165
|
+
removed += 1;
|
|
166
|
+
} catch {
|
|
167
|
+
/* ignore */
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return { dir, removed };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function cacheStats(fs: FsAdapter = realFs): { dir: string; files: number; entries: number; bytes: number } {
|
|
174
|
+
const dir = cacheDir(fs);
|
|
175
|
+
const files = fs.listDir(dir).filter((entry) => entry.endsWith(".json"));
|
|
176
|
+
let entries = 0;
|
|
177
|
+
let bytes = 0;
|
|
178
|
+
for (const file of files) {
|
|
179
|
+
const text = fs.readText(joinPath(dir, file));
|
|
180
|
+
if (!text) continue;
|
|
181
|
+
bytes += text.length;
|
|
182
|
+
try {
|
|
183
|
+
const parsed = JSON.parse(text) as CachedIndexFile;
|
|
184
|
+
entries += parsed.entries?.length ?? 0;
|
|
185
|
+
} catch {
|
|
186
|
+
/* ignore */
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return { dir, files: files.length, entries, bytes };
|
|
190
|
+
}
|
|
@@ -0,0 +1,491 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* clangd support for MSVC-compiled Unreal projects.
|
|
3
|
+
*
|
|
4
|
+
* `UnrealBuildTool -Mode=GenerateClangDatabase` is the engine-sanctioned way to produce a compile
|
|
5
|
+
* database, and it works with MSVC installed (no clang required). It emits `cl.exe` command lines,
|
|
6
|
+
* which clangd cannot use as written:
|
|
7
|
+
*
|
|
8
|
+
* 1. Each entry points at a shared response file (`@<Target>.<n>.rsp`). The rsp tail carries
|
|
9
|
+
* `/YuFILE` (create/use a shared PCH), `/FmFILE.obj`, `/FInull`. `FILE` is a placeholder the
|
|
10
|
+
* build substitutes per translation unit (UnrealBuildTool's makefile `PrimarySourceFiles`
|
|
11
|
+
* template), so a database entry cannot rely on it. clangd's driver reports
|
|
12
|
+
* `option '/YuFILE': no source file to create a PCH with`.
|
|
13
|
+
* 2. `/link ...` tails are linker arguments, not compiler arguments.
|
|
14
|
+
*
|
|
15
|
+
* Epic's own VS Code integration does the same job in two pieces:
|
|
16
|
+
* `Engine/Source/Programs/UnrealBuildTool/GA/VSCodeProjectFileGenerator.cs` writes
|
|
17
|
+
* `compileCommands_<Target>.json` plus `compileCommands_<Target>/<Target>.0.rsp` (with
|
|
18
|
+
* `-FI <SharedPCH.h>`), and the `stripPostPch.py` script shipped with Epic's Unreal-clangd plugin
|
|
19
|
+
* removes `/Yu*`, `/Fm*`, `/Fe*`, `/FInull` and everything after `/link`. The project's `.clangd`
|
|
20
|
+
* finishes the job by removing those flags again (`CompileFlags.Remove` covers `/Yu* /Fm* /FI*
|
|
21
|
+
* /Fp*`) and forcing clang-cl as the compiler.
|
|
22
|
+
*
|
|
23
|
+
* This module is the portable TypeScript port of that pipeline. The database is written into the
|
|
24
|
+
* pi-unreal cache, never into the project tree: a generated database encodes this machine's
|
|
25
|
+
* absolute paths and toolchain version, so checking it in is a trap.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { spawn } from "node:child_process";
|
|
29
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
|
|
30
|
+
import { dirname, join } from "node:path";
|
|
31
|
+
import { cacheDir } from "./cache.ts";
|
|
32
|
+
import { buildQueryArgs, hostPlatform, locateUbt } from "./ubt.ts";
|
|
33
|
+
import { normalizePath } from "./util.ts";
|
|
34
|
+
|
|
35
|
+
export interface CompileEntry {
|
|
36
|
+
file: string;
|
|
37
|
+
directory: string;
|
|
38
|
+
command: string;
|
|
39
|
+
/** Kept when present: `arguments[]` is the token form some generators write. */
|
|
40
|
+
arguments?: string[];
|
|
41
|
+
output?: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface CompileDatabase {
|
|
45
|
+
path: string;
|
|
46
|
+
entries: CompileEntry[];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/* ─────────────────────────────────────────────────────────────────── tokens */
|
|
50
|
+
|
|
51
|
+
/** Split a command line on whitespace, keeping quoted runs together. */
|
|
52
|
+
export function tokenizeCommand(command: string): string[] {
|
|
53
|
+
const tokens: string[] = [];
|
|
54
|
+
let current = "";
|
|
55
|
+
let quote: string | null = null;
|
|
56
|
+
for (const char of command) {
|
|
57
|
+
if (quote) {
|
|
58
|
+
if (char === quote) quote = null;
|
|
59
|
+
else current += char;
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
if (char === '"' || char === "'") {
|
|
63
|
+
quote = char;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (char === " " || char === "\t" || char === "\n" || char === "\r") {
|
|
67
|
+
if (current) tokens.push(current);
|
|
68
|
+
current = "";
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
current += char;
|
|
72
|
+
}
|
|
73
|
+
if (current) tokens.push(current);
|
|
74
|
+
return tokens;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Rebuild a command line; tokens containing whitespace get quoted. */
|
|
78
|
+
export function joinTokens(tokens: string[]): string {
|
|
79
|
+
return tokens.map((token) => (/[ \t]/.test(token) ? `"${token}"` : token)).join(" ");
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const isSwitch = (token: string, ...prefixes: string[]) => prefixes.some((prefix) => token.startsWith(prefix));
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Drop the switches clangd cannot use. `/Yu` needs a PCH source, `/Fm`/`/Fe`/`/F*` write MSVC
|
|
86
|
+
* artifacts, `/FInull` is a placeholder, and `/link` starts the linker tail.
|
|
87
|
+
* Mirrors Epic's `stripPostPch.py`.
|
|
88
|
+
*/
|
|
89
|
+
export function stripPostPchTokens(tokens: string[]): { tokens: string[]; dropped: string[] } {
|
|
90
|
+
const kept: string[] = [];
|
|
91
|
+
const dropped: string[] = [];
|
|
92
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
93
|
+
const token = tokens[index] ?? "";
|
|
94
|
+
if (token.toLowerCase() === "/link") {
|
|
95
|
+
dropped.push("/link", ...tokens.slice(index + 1));
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
const lower = token.toLowerCase();
|
|
99
|
+
if (isSwitch(lower, "/yu", "/fm", "/fe", "/fp", "/fni", "/fina")) {
|
|
100
|
+
dropped.push(token);
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
kept.push(token);
|
|
104
|
+
}
|
|
105
|
+
return { tokens: kept, dropped };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Replace `@file` response indirection with the file's own tokens (depth-limited). */
|
|
109
|
+
export function expandResponseTokens(tokens: string[], opts: { baseDir: string; depth?: number; maxDepth?: number }): { tokens: string[]; expanded: string[]; missing: string[] } {
|
|
110
|
+
const maxDepth = opts.maxDepth ?? 3;
|
|
111
|
+
const depth = opts.depth ?? 0;
|
|
112
|
+
const out: string[] = [];
|
|
113
|
+
const expanded: string[] = [];
|
|
114
|
+
const missing: string[] = [];
|
|
115
|
+
for (const token of tokens) {
|
|
116
|
+
if (!token.startsWith("@") || depth >= maxDepth) {
|
|
117
|
+
out.push(token);
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
const path = token.slice(1).replace(/^"|"$/g, "");
|
|
121
|
+
const abs = path.includes("/") || path.includes("\\") ? path : join(opts.baseDir, path);
|
|
122
|
+
if (!existsSync(abs)) {
|
|
123
|
+
missing.push(abs);
|
|
124
|
+
out.push(token);
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
const text = readFileSync(abs, "utf8").replace(/^/, "");
|
|
128
|
+
const nested = expandResponseTokens(tokenizeCommand(text), { baseDir: dirname(abs), depth: depth + 1, maxDepth });
|
|
129
|
+
out.push(...nested.tokens);
|
|
130
|
+
expanded.push(abs);
|
|
131
|
+
missing.push(...nested.missing);
|
|
132
|
+
}
|
|
133
|
+
return { tokens: out, expanded, missing };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** UnrealBuildTool's `FILE` placeholder: the translation unit being compiled. */
|
|
137
|
+
function resolveFileToken(tokens: string[], file: string): string[] {
|
|
138
|
+
const quoted = `"${file}"`;
|
|
139
|
+
return tokens.map((token) => {
|
|
140
|
+
if (token === "FILE") return file;
|
|
141
|
+
if (token.endsWith("FILE")) return token.slice(0, -"FILE".length) + file;
|
|
142
|
+
if (token.includes('"FILE"')) return token.replace(/"FILE"/g, file);
|
|
143
|
+
if (token.includes("FILE")) return token.replace(/FILE/g, file);
|
|
144
|
+
return token;
|
|
145
|
+
}).map((token, index) => (index === 0 ? token : token === quoted ? file : token));
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/* ────────────────────────────────────────────────────────────────── entries */
|
|
149
|
+
|
|
150
|
+
export function readCompileCommands(path: string): CompileEntry[] {
|
|
151
|
+
const raw = JSON.parse(readFileSync(path, "utf8")) as CompileEntry[];
|
|
152
|
+
return Array.isArray(raw) ? raw : [];
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function writeCompileCommands(path: string, entries: CompileEntry[]): void {
|
|
156
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
157
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
158
|
+
writeFileSync(tmp, JSON.stringify(entries, null, 1));
|
|
159
|
+
renameSync(tmp, path);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Keep entries whose `file` starts with an include prefix and matches no exclude prefix. */
|
|
163
|
+
export function filterEntries(entries: CompileEntry[], opts: { include?: string[]; exclude?: string[] } = {}): CompileEntry[] {
|
|
164
|
+
const include = (opts.include ?? []).map((p) => normalizePath(p).toLowerCase());
|
|
165
|
+
const exclude = (opts.exclude ?? []).map((p) => normalizePath(p).toLowerCase());
|
|
166
|
+
return entries.filter((entry) => {
|
|
167
|
+
const file = normalizePath(entry.file).toLowerCase();
|
|
168
|
+
if (include.length && !include.some((prefix) => file.startsWith(prefix))) return false;
|
|
169
|
+
if (exclude.some((prefix) => file.startsWith(prefix))) return false;
|
|
170
|
+
return true;
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Header entries carry no independent information: a header's flags come from its includers. */
|
|
175
|
+
export function dropHeaderEntries(entries: CompileEntry[]): { entries: CompileEntry[]; dropped: number } {
|
|
176
|
+
const kept = entries.filter((entry) => !/\.(h|hpp|inl|cpp)\.h$/i.test(entry.file) && !/\.h$/i.test(entry.file));
|
|
177
|
+
return { entries: kept, dropped: entries.length - kept.length };
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Epic's shared-PCH translation units (`SharedPCH.*.h.cpp`) — clangd cannot build PCHs. */
|
|
181
|
+
function isPchEntry(file: string): boolean {
|
|
182
|
+
return /SharedPCH[^\/]*\.h\.cpp$/i.test(file) || /PCH\.[^\/]*\.h\.cpp$/i.test(file);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Make a database clangd can consume: expand response files, drop PCH/link switches, resolve the
|
|
187
|
+
* `FILE` placeholder, optionally keep only the requested source scope.
|
|
188
|
+
*/
|
|
189
|
+
export function normalizeDatabase(entries: CompileEntry[], opts: {
|
|
190
|
+
baseDir: string;
|
|
191
|
+
include?: string[];
|
|
192
|
+
exclude?: string[];
|
|
193
|
+
inlineResponseFiles?: boolean;
|
|
194
|
+
dropHeaders?: boolean;
|
|
195
|
+
target?: string;
|
|
196
|
+
} = { baseDir: "." }): { entries: CompileEntry[]; stats: { droppedPch: number; droppedHeaders: number; droppedSwitches: number; expandedRsp: number; missingRsp: string[]; kept: number } } {
|
|
197
|
+
const stats = { droppedPch: 0, droppedHeaders: 0, droppedSwitches: 0, expandedRsp: 0, missingRsp: [] as string[], kept: 0 };
|
|
198
|
+
let working = entries.filter((entry) => {
|
|
199
|
+
if (isPchEntry(entry.file)) {
|
|
200
|
+
stats.droppedPch++;
|
|
201
|
+
return false;
|
|
202
|
+
}
|
|
203
|
+
return true;
|
|
204
|
+
});
|
|
205
|
+
if (opts.dropHeaders) {
|
|
206
|
+
const result = dropHeaderEntries(working);
|
|
207
|
+
working = result.entries;
|
|
208
|
+
stats.droppedHeaders = result.dropped;
|
|
209
|
+
}
|
|
210
|
+
working = filterEntries(working, { include: opts.include, exclude: opts.exclude });
|
|
211
|
+
const out: CompileEntry[] = [];
|
|
212
|
+
for (const entry of working) {
|
|
213
|
+
const tokens = entry.arguments?.length ? [...entry.arguments] : tokenizeCommand(entry.command);
|
|
214
|
+
const expanded = expandResponseTokens(tokens, { baseDir: opts.baseDir });
|
|
215
|
+
stats.missingRsp.push(...expanded.missing);
|
|
216
|
+
const stripped = stripPostPchTokens(expanded.tokens);
|
|
217
|
+
stats.droppedSwitches += stripped.dropped.length;
|
|
218
|
+
const resolved = resolveFileToken(stripped.tokens, entry.file);
|
|
219
|
+
if (expanded.expanded.length) stats.expandedRsp += expanded.expanded.length;
|
|
220
|
+
out.push({ file: entry.file, directory: entry.directory, command: joinTokens(resolved) });
|
|
221
|
+
}
|
|
222
|
+
stats.kept = out.length;
|
|
223
|
+
return { entries: out, stats };
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/* ────────────────────────────────────────────────────────────────── running */
|
|
227
|
+
|
|
228
|
+
export interface GenerateOptions {
|
|
229
|
+
engineRoot: string;
|
|
230
|
+
projectPath?: string;
|
|
231
|
+
target: string;
|
|
232
|
+
platform?: string;
|
|
233
|
+
configuration?: string;
|
|
234
|
+
/** Where UnrealBuildTool writes compile_commands.json. Defaults to the pi-unreal cache. */
|
|
235
|
+
outputDir?: string;
|
|
236
|
+
/** Absolute prefixes kept in the final database (applied locally; UnrealBuildTool -Include= matches nothing useful). */
|
|
237
|
+
include?: string[];
|
|
238
|
+
exclude?: string[];
|
|
239
|
+
timeoutMs?: number;
|
|
240
|
+
onProgress?: (message: string) => void;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Where UnrealBuildTool writes the database for a target (it ignores `-OutputPath=` for this mode). */
|
|
244
|
+
export function ubtCompileDbPaths(opts: { engineRoot: string; projectDir?: string; platform?: string }): string[] {
|
|
245
|
+
const platform = opts.platform ?? hostPlatform();
|
|
246
|
+
const paths = [
|
|
247
|
+
opts.projectDir ? join(opts.projectDir, "Intermediate", "Build", "Windows", "UnrealBuildTool", "compile_commands.json") : "",
|
|
248
|
+
opts.projectDir ? join(opts.projectDir, "Intermediate", "Build", platform, "UnrealBuildTool", "compile_commands.json") : "",
|
|
249
|
+
join(opts.engineRoot, "Engine", "Intermediate", "Build", platform, "UnrealBuildTool", "compile_commands.json"),
|
|
250
|
+
].filter(Boolean);
|
|
251
|
+
return paths.map(normalizePath);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** Run `-Mode=GenerateClangDatabase`. Costs minutes: UBT regenerates the makefile and UHT code. */
|
|
255
|
+
export async function runGenerateClangDatabase(opts: GenerateOptions): Promise<{ rawPath: string; log: string; seconds: number; warnings: string[] }> {
|
|
256
|
+
const warnings: string[] = [];
|
|
257
|
+
const ubt = locateUbt(opts.engineRoot);
|
|
258
|
+
if (!ubt) throw new Error(`UnrealBuildTool not found under ${opts.engineRoot} — is the engine installed?`);
|
|
259
|
+
const command = ubt.kind === "exe" ? ubt.path : undefined;
|
|
260
|
+
if (!command) throw new Error(`only the ${ubt.kind} launcher at ${ubt.path} was found; run UnrealBuildTool from a shell that can execute it`);
|
|
261
|
+
const platform = opts.platform ?? hostPlatform();
|
|
262
|
+
const configuration = opts.configuration ?? "Development";
|
|
263
|
+
// UnrealBuildTool does not create -OutputDir, and dies with DirectoryNotFoundException if it is missing.
|
|
264
|
+
const outputDir = opts.outputDir ?? normalizePath(join(cacheDir(), "compiledb", "raw", `${opts.target}.${platform}.${configuration}`));
|
|
265
|
+
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
|
|
266
|
+
const args = [
|
|
267
|
+
// TargetDescriptor.ParseSingleCommandLine only inspects arguments that do NOT start with '-', and
|
|
268
|
+
// its '+' split needs every token to be the same kind (all platforms, all configs, all names):
|
|
269
|
+
// `Win64+Development+LpEditor` dies with `The platform name Development is not a valid platform
|
|
270
|
+
// name`, and a bare `-Target=LpEditor` dies with `No platforms specified for target`. Separate
|
|
271
|
+
// positional tokens are the form that parses: platform, configuration, target name.
|
|
272
|
+
platform,
|
|
273
|
+
configuration,
|
|
274
|
+
opts.target,
|
|
275
|
+
// `-Platform=` / `-Configuration=` are deliberately absent: they set BuildConfiguration, and the
|
|
276
|
+
// positional token `Win64` is already consumed as a platform — passing a second one via
|
|
277
|
+
// `-Platform=Win64` makes TargetDescriptor reject it as a duplicate platform.
|
|
278
|
+
// Quotes belong to the shell, not to the argument: UnrealBuildTool reads the value verbatim.
|
|
279
|
+
...(opts.projectPath ? [`-Project=${opts.projectPath}`] : []),
|
|
280
|
+
"-Mode=GenerateClangDatabase",
|
|
281
|
+
// -Include= rules are token-path patterns and match nothing useful here (measured: -Include=Source/Lp
|
|
282
|
+
// and -Include=*/Source/Lp/* both yield an empty database), so scoping happens in normalizeDatabase.
|
|
283
|
+
"-NoExecCodeGenActions",
|
|
284
|
+
...(outputDir ? [`-OutputDir=${outputDir}`] : []),
|
|
285
|
+
"-Progress",
|
|
286
|
+
];
|
|
287
|
+
const started = Date.now();
|
|
288
|
+
opts.onProgress?.(`running UnrealBuildTool -Mode=GenerateClangDatabase ${platform}+${configuration}+${opts.target}`);
|
|
289
|
+
const log = await new Promise<string>((resolve, reject) => {
|
|
290
|
+
const child = spawn(command, args, { cwd: opts.engineRoot, windowsHide: true, shell: false });
|
|
291
|
+
let text = "";
|
|
292
|
+
child.stdout.on("data", (chunk: Buffer) => {
|
|
293
|
+
text += chunk.toString("utf8");
|
|
294
|
+
const line = text.split("\n").filter(Boolean).pop();
|
|
295
|
+
if (line) opts.onProgress?.(line.trim());
|
|
296
|
+
});
|
|
297
|
+
child.stderr.on("data", (chunk: Buffer) => {
|
|
298
|
+
text += chunk.toString("utf8");
|
|
299
|
+
});
|
|
300
|
+
const timer = opts.timeoutMs ? setTimeout(() => {
|
|
301
|
+
child.kill("SIGKILL");
|
|
302
|
+
reject(new Error(`UnrealBuildTool timed out after ${opts.timeoutMs} ms; log tail:\n${text.slice(-2000)}`));
|
|
303
|
+
}, opts.timeoutMs) : undefined;
|
|
304
|
+
child.on("error", (error: Error) => {
|
|
305
|
+
if (timer) clearTimeout(timer);
|
|
306
|
+
reject(error);
|
|
307
|
+
});
|
|
308
|
+
child.on("close", (code: number | null) => {
|
|
309
|
+
if (timer) clearTimeout(timer);
|
|
310
|
+
if (code === 0) resolve(text);
|
|
311
|
+
else reject(new Error(`UnrealBuildTool exited with ${code}; log tail:\n${text.slice(-2000)}`));
|
|
312
|
+
});
|
|
313
|
+
});
|
|
314
|
+
const seconds = (Date.now() - started) / 1000;
|
|
315
|
+
const written = writtenPathFrom(log);
|
|
316
|
+
const candidates = [outputDir ? normalizePath(join(outputDir, "compile_commands.json")) : "", written, ...ubtCompileDbPaths({ engineRoot: opts.engineRoot, projectDir: opts.projectPath ? dirname(opts.projectPath) : undefined, platform })].filter(Boolean);
|
|
317
|
+
const rawPath = candidates.find((path) => path && existsSync(path));
|
|
318
|
+
if (!rawPath) throw new Error(`UnrealBuildTool reported success but no compile_commands.json was found. Looked at:\n${candidates.map((path) => `- ${path}`).join("\n")}`);
|
|
319
|
+
const size = statSync(rawPath).size;
|
|
320
|
+
if (size < 1024) warnings.push(`${rawPath} is only ${size} bytes — the target generated almost no source entries`);
|
|
321
|
+
return { rawPath, log, seconds, warnings };
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function writtenPathFrom(log: string): string | undefined {
|
|
325
|
+
// UnrealBuildTool's line is `ClangDatabase written to <path>`; older builds log `Writing <path>`.
|
|
326
|
+
const match = /(?:ClangDatabase written to|Writing)\s+"?([^"\r\n]+\.json)"?/i.exec(
|
|
327
|
+
log.split(/\r?\n/).filter((line) => /compile_commands\.json|ClangDatabase written/i.test(line)).slice(-1)[0] ?? "",
|
|
328
|
+
);
|
|
329
|
+
return match?.[1] ? normalizePath(match[1].replace(/\\/g, "/")) : undefined;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
export interface EnsureOptions extends GenerateOptions {
|
|
333
|
+
/** Cache key suffix. Defaults to `<target>.<platform>.<configuration>`. */
|
|
334
|
+
key?: string;
|
|
335
|
+
/** Ignore a cached database and regenerate. */
|
|
336
|
+
refresh?: boolean;
|
|
337
|
+
/** Drop `.h`/`.hpp`/`.inl` entries (clangd infers header flags from includers). */
|
|
338
|
+
dropHeaders?: boolean;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
export interface EnsureResult {
|
|
342
|
+
path: string;
|
|
343
|
+
rawPath?: string;
|
|
344
|
+
entries: number;
|
|
345
|
+
seconds: number;
|
|
346
|
+
reused: boolean;
|
|
347
|
+
stats: { droppedPch: number; droppedHeaders: number; droppedSwitches: number; expandedRsp: number; kept: number };
|
|
348
|
+
warnings: string[];
|
|
349
|
+
scope: string[];
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/** Generate (or reuse) a clangd-ready database in the cache. */
|
|
353
|
+
export async function ensureCompileDb(opts: EnsureOptions): Promise<EnsureResult> {
|
|
354
|
+
const key = opts.key ?? `${opts.target}.${opts.platform ?? hostPlatform()}.${opts.configuration ?? "Development"}`;
|
|
355
|
+
const dir = normalizePath(join(cacheDir(), "compiledb", key));
|
|
356
|
+
const path = normalizePath(join(dir, "compile_commands.json"));
|
|
357
|
+
const warnings: string[] = [];
|
|
358
|
+
if (!opts.refresh && existsSync(path)) {
|
|
359
|
+
const entries = readCompileCommands(path);
|
|
360
|
+
if (entries.length) {
|
|
361
|
+
return {
|
|
362
|
+
path,
|
|
363
|
+
entries: entries.length,
|
|
364
|
+
seconds: 0,
|
|
365
|
+
reused: true,
|
|
366
|
+
stats: { droppedPch: 0, droppedHeaders: 0, droppedSwitches: 0, expandedRsp: 0, kept: entries.length },
|
|
367
|
+
warnings: [`reusing cached database (${entries.length} entries, ${Math.round(statSync(path).size / 1024)} KB, ${Math.round((Date.now() - statSync(path).mtimeMs) / 1000)}s old) — pass refresh to regenerate`],
|
|
368
|
+
scope: opts.include ?? [],
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
warnings.push(`cached database ${path} is empty, regenerating`);
|
|
372
|
+
}
|
|
373
|
+
const generated = await runGenerateClangDatabase(opts);
|
|
374
|
+
const rawEntries = readCompileCommands(generated.rawPath);
|
|
375
|
+
const normalized = normalizeDatabase(rawEntries, {
|
|
376
|
+
baseDir: dirname(generated.rawPath),
|
|
377
|
+
include: opts.include,
|
|
378
|
+
exclude: opts.exclude,
|
|
379
|
+
dropHeaders: opts.dropHeaders,
|
|
380
|
+
target: opts.target,
|
|
381
|
+
});
|
|
382
|
+
warnings.push(...generated.warnings);
|
|
383
|
+
if (normalized.stats.missingRsp.length) warnings.push(`response files not readable: ${normalized.stats.missingRsp.slice(0, 3).join(", ")} (${normalized.stats.missingRsp.length} total)`);
|
|
384
|
+
if (!normalized.entries.length) warnings.push(`no entries survived post-processing of ${rawEntries.length} generated entries — check the include/exclude scope`);
|
|
385
|
+
mkdirSync(dir, { recursive: true });
|
|
386
|
+
writeCompileCommands(path, normalized.entries);
|
|
387
|
+
return {
|
|
388
|
+
path,
|
|
389
|
+
rawPath: generated.rawPath,
|
|
390
|
+
entries: normalized.entries.length,
|
|
391
|
+
seconds: generated.seconds,
|
|
392
|
+
reused: false,
|
|
393
|
+
stats: { droppedPch: normalized.stats.droppedPch, droppedHeaders: normalized.stats.droppedHeaders, droppedSwitches: normalized.stats.droppedSwitches, expandedRsp: normalized.stats.expandedRsp, kept: normalized.stats.kept },
|
|
394
|
+
warnings,
|
|
395
|
+
scope: opts.include ?? [],
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/* ─────────────────────────────────────────────────────────────────── clangd */
|
|
400
|
+
|
|
401
|
+
const CLANGD_LAYOUTS = [
|
|
402
|
+
"bin/clangd.exe",
|
|
403
|
+
"clangd.exe",
|
|
404
|
+
"clangd_*/bin/clangd.exe",
|
|
405
|
+
"bin/clangd",
|
|
406
|
+
];
|
|
407
|
+
|
|
408
|
+
/** Where a clangd binary might live on this machine (MSVC boxes ship one with the VS LLVM toolset). */
|
|
409
|
+
export function discoverClangd(fs: { exists: (p: string) => boolean } = { exists: existsSync }): string[] {
|
|
410
|
+
const found: string[] = [];
|
|
411
|
+
const roots = [
|
|
412
|
+
process.env.CLANGD_PATH,
|
|
413
|
+
"C:/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/Llvm/x64/bin",
|
|
414
|
+
"C:/Program Files/Microsoft Visual Studio/2022/Professional/VC/Tools/Llvm/x64/bin",
|
|
415
|
+
"C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/Llvm/x64/bin",
|
|
416
|
+
"C:/Program Files/Microsoft Visual Studio/18/Community/VC/Tools/Llvm/x64/bin",
|
|
417
|
+
"C:/Program Files/Microsoft Visual Studio/18/Professional/VC/Tools/Llvm/x64/bin",
|
|
418
|
+
"C:/Program Files/LLVM/bin",
|
|
419
|
+
process.env.LOCALAPPDATA ? join(process.env.LOCALAPPDATA, "Microsoft/WinGet/Links") : undefined,
|
|
420
|
+
process.env.LOCALAPPDATA ? join(process.env.LOCALAPPDATA, "Microsoft/WinGet/Packages") : undefined,
|
|
421
|
+
].filter(Boolean) as string[];
|
|
422
|
+
for (const root of roots) {
|
|
423
|
+
for (const layout of CLANGD_LAYOUTS) {
|
|
424
|
+
const candidate = normalizePath(join(root, layout));
|
|
425
|
+
if (!candidate.includes("*") && fs.exists(candidate)) found.push(candidate);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
if (process.env.LOCALAPPDATA) {
|
|
429
|
+
const packages = normalizePath(join(process.env.LOCALAPPDATA, "Microsoft/WinGet/Packages"));
|
|
430
|
+
if (existsSync(packages)) {
|
|
431
|
+
for (const dir of ["LLVM_Microsoft.Winget.Source_8wekyb3d8bbwe", "LLVM.clangd_Microsoft.Winget.Source_8wekyb3d8bbwe"]) {
|
|
432
|
+
const candidate = normalizePath(join(packages, dir, "bin/clangd.exe"));
|
|
433
|
+
if (existsSync(candidate)) found.push(candidate);
|
|
434
|
+
for (const sub of listSubdirs(packages)) {
|
|
435
|
+
const nested = normalizePath(join(packages, sub, "clangd/bin/clangd.exe"));
|
|
436
|
+
if (existsSync(nested)) found.push(nested);
|
|
437
|
+
const versioned = normalizePath(join(packages, sub, "clangd_21.1.8/bin/clangd.exe"));
|
|
438
|
+
if (existsSync(versioned)) found.push(versioned);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
return [...new Set(found)];
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function listSubdirs(dir: string): string[] {
|
|
447
|
+
try {
|
|
448
|
+
return readdirSync(dir, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name);
|
|
449
|
+
} catch {
|
|
450
|
+
return [];
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Ask clangd to compile one file with the generated database. This is the acceptance test for a
|
|
456
|
+
* database: a clean run builds a preamble and indexes the headers of the translation unit.
|
|
457
|
+
*/
|
|
458
|
+
export async function clangdCheck(opts: { file: string; dbDir: string; clangdPath?: string; timeoutMs?: number }): Promise<{ ok: boolean; clangd?: string; summary: string[]; notes: string[]; raw: string }> {
|
|
459
|
+
const clangd = opts.clangdPath ?? discoverClangd()[0];
|
|
460
|
+
if (!clangd) return { ok: false, summary: [], notes: ["no clangd binary found; look at CLANGD_PATH and the VS LLVM toolset"], raw: "" };
|
|
461
|
+
if (!existsSync(opts.file)) return { ok: false, clangd, summary: [], notes: [`file not found: ${opts.file}`], raw: "" };
|
|
462
|
+
// clangd 22 takes a number here: `--check-locations=lines` is rejected at option parsing
|
|
463
|
+
// (`'lines' is invalid value for boolean argument`), which aborts the check before any compile.
|
|
464
|
+
const args = [`--check=${opts.file}`, `--compile-commands-dir=${opts.dbDir}`, "--background-index=false", "--check-locations=1", "--limit-references=0", "--limit-results=0"];
|
|
465
|
+
const raw = await new Promise<string>((resolve, reject) => {
|
|
466
|
+
const child = spawn(clangd, args, { windowsHide: true });
|
|
467
|
+
let text = "";
|
|
468
|
+
child.stdout.on("data", (chunk: Buffer) => {
|
|
469
|
+
text += chunk.toString("utf8");
|
|
470
|
+
});
|
|
471
|
+
child.stderr.on("data", (chunk: Buffer) => {
|
|
472
|
+
text += chunk.toString("utf8");
|
|
473
|
+
});
|
|
474
|
+
const timer = setTimeout(() => {
|
|
475
|
+
child.kill("SIGKILL");
|
|
476
|
+
reject(new Error(`clangd timed out after ${opts.timeoutMs ?? 120000} ms`));
|
|
477
|
+
}, opts.timeoutMs ?? 120000);
|
|
478
|
+
child.on("error", (error) => {
|
|
479
|
+
clearTimeout(timer);
|
|
480
|
+
reject(error);
|
|
481
|
+
});
|
|
482
|
+
child.on("close", () => {
|
|
483
|
+
clearTimeout(timer);
|
|
484
|
+
resolve(text);
|
|
485
|
+
});
|
|
486
|
+
});
|
|
487
|
+
const lines = raw.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
488
|
+
const summary = lines.filter((line) => /All checks completed|Built preamble|Indexing headers|Building AST|Loaded compilation|Invalid argument|no source file to create a PCH|^.*error:|fatal|option '/i.test(line)).slice(0, 24);
|
|
489
|
+
const ok = /All checks completed, 0 errors/.test(raw) && !/no source file to create a PCH|Loaded compilation\(s\) failed|invalid value/.test(raw);
|
|
490
|
+
return { ok, clangd, summary, notes: ok ? ["clangd compiled the file with the generated database"] : ["clangd rejected the generated database — see summary"], raw };
|
|
491
|
+
}
|