@geml/geml 1.7.8 → 1.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +288 -285
- package/codemap/adapters/crg.mjs +120 -120
- package/codemap/adapters/joern.mjs +131 -131
- package/codemap/adapters/scip.mjs +658 -658
- package/codemap/browser-stub.mjs +34 -34
- package/codemap/build.mjs +629 -629
- package/codemap/cross-stack.mjs +303 -303
- package/codemap/detect.mjs +399 -399
- package/codemap/emit.mjs +510 -510
- package/codemap/entries.mjs +129 -129
- package/codemap/exclude.mjs +56 -56
- package/codemap/find.mjs +49 -49
- package/codemap/foldings.mjs +110 -110
- package/codemap/joern-export.sc +83 -83
- package/codemap/mcp-server.mjs +434 -431
- package/codemap/normalize.mjs +275 -275
- package/codemap/recipe-trust.mjs +103 -103
- package/codemap/refresh.mjs +313 -313
- package/codemap/render-all.mjs +90 -90
- package/codemap/serve.mjs +585 -585
- package/codemap/sfc-virtualize.mjs +367 -367
- package/codemap/verify.mjs +158 -158
- package/dist/attrs.d.ts +8 -0
- package/dist/attrs.js +24 -0
- package/dist/cli.js +190 -139
- package/dist/diagnostics.d.ts +1 -1
- package/dist/diagnostics.js +5 -1
- package/dist/geml.js +63 -25
- package/dist/inline.d.ts +7 -1
- package/dist/inline.js +203 -137
- package/dist/mcp.js +68 -26
- package/dist/render-html.js +35 -35
- package/dist/render.js +157 -157
- package/package.json +1 -1
- package/skill/SKILL.md +167 -167
- package/skill/references/authoring.geml +369 -365
package/codemap/build.mjs
CHANGED
|
@@ -1,629 +1,629 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
// geml-code-graph build — one shot: adapter → exchange format (build/) → GEML tree (graph/).
|
|
3
|
-
//
|
|
4
|
-
// geml codemap build --root <repo-root> # AUTO: detect languages,
|
|
5
|
-
// # run the right indexer(s), merge into one map
|
|
6
|
-
// geml codemap build --db <graph.db> --root <repo-root> # crg
|
|
7
|
-
// geml codemap build --adapter joern --raw <dir> \
|
|
8
|
-
// --adapter scip --raw <index.scip> --root <repo> # merged multi-language
|
|
9
|
-
// geml codemap build --adapter joern --raw <dir> --root <repo-root> # joern
|
|
10
|
-
// [--out .geml-code-graph] [--build .geml-code-graph/_build]
|
|
11
|
-
// [--container module|dir|file] container granularity (default dir)
|
|
12
|
-
// [--lang <JAVASRC|NEWC|…>] force the Joern frontend (auto mode,
|
|
13
|
-
// mixed-majority repos)
|
|
14
|
-
// [--joern <path>] Joern install (auto mode): the launcher OR the
|
|
15
|
-
// unzipped joern-cli dir; else GEML_JOERN, else PATH
|
|
16
|
-
// [--history [-m "msg"]] snapshot changed docs into .gemlhistory
|
|
17
|
-
// sidecars (per-node history + revert)
|
|
18
|
-
//
|
|
19
|
-
// Auto mode (no --adapter and no --db, just --root): detect.mjs picks the
|
|
20
|
-
// indexer per language from manifests + source extensions, we run scip
|
|
21
|
-
// (npx @sourcegraph/scip-typescript for TS/JS, rust-analyzer scip for Rust)
|
|
22
|
-
// and/or Joern (joern-export.sc) into <out>/_build/, then feed the results
|
|
23
|
-
// into the SAME merge as the explicit --adapter path, and record the replay
|
|
24
|
-
// recipe into _index/refresh.json.
|
|
25
|
-
//
|
|
26
|
-
// Output shape: docs/codemap-profile.md — one document per container (single
|
|
27
|
-
// meta with module/src/entry, empty-body code blocks with src=/anchor=, and
|
|
28
|
-
// the #calls / #called-by / #unresolved CSV edge tables). Verify with
|
|
29
|
-
// geml codemap verify (geml check + profile reference checks).
|
|
30
|
-
//
|
|
31
|
-
// Adapters (docs/DESIGN-geml-code-graph.md §3):
|
|
32
|
-
// crg code-review-graph SQLite graph.db (tree-sitter level; everything
|
|
33
|
-
// honestly labelled resolution:"heuristic") [P0, default]
|
|
34
|
-
// joern Joern CPG export: run geml-parser/codemap/joern-export.sc inside
|
|
35
|
-
// joern first; --raw points at its outDir [P1]
|
|
36
|
-
//
|
|
37
|
-
// After building, run: geml codemap verify <out-dir>
|
|
38
|
-
import { writeFileSync, mkdirSync, existsSync, readFileSync, statSync } from "node:fs";
|
|
39
|
-
import { join, resolve, basename, dirname, relative, sep } from "node:path";
|
|
40
|
-
import { fileURLToPath } from "node:url";
|
|
41
|
-
import { execFileSync, spawnSync } from "node:child_process";
|
|
42
|
-
import { emit } from "./emit.mjs";
|
|
43
|
-
import { makeExcluder } from "./exclude.mjs";
|
|
44
|
-
import { detectLanguages, indexerCommand, collectSourceFiles } from "./detect.mjs";
|
|
45
|
-
import { loadOrSeedFoldings } from "./foldings.mjs";
|
|
46
|
-
import { detectEntries } from "./entries.mjs";
|
|
47
|
-
import { discoverModuleRoots } from "./normalize.mjs";
|
|
48
|
-
import { recipeFingerprint, trustRecipe, RECIPE_VERSION } from "./recipe-trust.mjs";
|
|
49
|
-
import { buildCrossStackOverlay } from "./cross-stack.mjs";
|
|
50
|
-
|
|
51
|
-
const args = process.argv.slice(2);
|
|
52
|
-
const flag = (name, dflt) => {
|
|
53
|
-
const i = args.indexOf(name);
|
|
54
|
-
return i >= 0 ? args[i + 1] : dflt;
|
|
55
|
-
};
|
|
56
|
-
|
|
57
|
-
const USAGE = [
|
|
58
|
-
"usage: geml codemap build [--root <repo-root>] # auto-detect languages, index, and merge (--root defaults to the current directory)",
|
|
59
|
-
" or: geml codemap build (--db <graph.db> | --adapter joern|scip --raw <dir|index.scip> [--remap <virtual-dir>])+ [--root <repo-root>] [--out .geml-code-graph] [--build .geml-code-graph/_build] [--container module|dir|file] [--lang <LANG>] [--joern <path>] [--exclude <glob>]... [--no-gitignore] [--history [-m msg]]",
|
|
60
|
-
].join("\n");
|
|
61
|
-
if (args.includes("--help") || args.includes("-h")) { console.log(USAGE); process.exit(0); }
|
|
62
|
-
|
|
63
|
-
// --root defaults to the current directory, so `geml codemap build` with no
|
|
64
|
-
// arguments indexes the repo you're standing in.
|
|
65
|
-
const root = flag("--root", ".");
|
|
66
|
-
const outDir = resolve(flag("--out", ".geml-code-graph"));
|
|
67
|
-
// Intermediates live INSIDE the codemap dir (alongside _index) so a build
|
|
68
|
-
// leaves nothing scattered at the repo root — `.geml-code-graph/_build/`.
|
|
69
|
-
const buildDir = resolve(flag("--build", join(outDir, "_build")));
|
|
70
|
-
|
|
71
|
-
// Adapter inputs are REPEATABLE — one codemap can merge several extractions
|
|
72
|
-
// (e.g. Joern for the Java modules + SCIP for the TypeScript ones). Each
|
|
73
|
-
// `--adapter X` opens a group; the following `--db`/`--raw` belongs to it.
|
|
74
|
-
// A bare `--db` without `--adapter` keeps the historical crg default.
|
|
75
|
-
const inputs = [];
|
|
76
|
-
{
|
|
77
|
-
let cur = null;
|
|
78
|
-
for (let i = 0; i < args.length; i++) {
|
|
79
|
-
if (args[i] === "--adapter") { cur = { adapter: args[++i] }; inputs.push(cur); }
|
|
80
|
-
else if (args[i] === "--db") { if (!cur) { cur = { adapter: "crg" }; inputs.push(cur); } cur.db = args[++i]; cur = null; }
|
|
81
|
-
else if (args[i] === "--raw") { if (!cur) { console.error("--raw needs a preceding --adapter"); process.exit(2); } cur.raw = args[++i]; cur = null; }
|
|
82
|
-
// --remap <virtual dir>: the preceding scip input was produced over an
|
|
83
|
-
// SFC virtual dir (sfc-virtualize.mjs) — the adapter maps shadow paths
|
|
84
|
-
// back to the original .vue/.svelte sources. Recorded into refresh.json
|
|
85
|
-
// so replays keep the remapping.
|
|
86
|
-
else if (args[i] === "--remap") { if (!inputs.length) { console.error("--remap needs a preceding --adapter/--raw group"); process.exit(2); } inputs[inputs.length - 1].remap = args[++i]; }
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
// ---- auto-detect mode -------------------------------------------------------
|
|
90
|
-
// A --root with NO --adapter and NO --db (inputs still empty): detect the
|
|
91
|
-
// languages ourselves, run the right indexer(s) into <out>/_build/, then push
|
|
92
|
-
// the results into `inputs` so the merge below runs UNCHANGED. This is the
|
|
93
|
-
// one-command onboarding path; the explicit --adapter/--db paths are untouched.
|
|
94
|
-
let recordRecipe = null; // { rootAbs, steps } — written to refresh.json after emit
|
|
95
|
-
let detectedLanguages = []; // languages seen in auto-detect; seeds foldings' language conventions
|
|
96
|
-
let entryHints = []; // app-entry hints (entries.mjs), matched to symbols in emit
|
|
97
|
-
if (root && !inputs.length) {
|
|
98
|
-
const rootAbs = resolve(root);
|
|
99
|
-
const excludeGlobs0 = args.flatMap((v, i) => (args[i - 1] === "--exclude" ? [v] : []));
|
|
100
|
-
const { files, manifests, pkgs } = collectSourceFiles(rootAbs);
|
|
101
|
-
const excluder = makeExcluder({
|
|
102
|
-
root: rootAbs, globs: excludeGlobs0, gitignore: !args.includes("--no-gitignore"),
|
|
103
|
-
files: [...files, ...manifests, ...pkgs], run: execFileSync,
|
|
104
|
-
});
|
|
105
|
-
const jobs = detectLanguages(rootAbs, { files, manifests, pkgs, excluder });
|
|
106
|
-
detectedLanguages = [...new Set(jobs.map((j) => j.language))];
|
|
107
|
-
// App-entry hints from manifests/layout/source markers — pure detection now,
|
|
108
|
-
// matched to extracted symbols (or noted file-level) inside emit.
|
|
109
|
-
entryHints = detectEntries(rootAbs, {
|
|
110
|
-
files: files.filter((f) => !excluder(f)),
|
|
111
|
-
manifests: manifests.filter((m) => !excluder(m)),
|
|
112
|
-
pkgs: (pkgs ?? []).filter((p) => !excluder(p)),
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
// --lang forces the Joern frontend (GEML_LANG) — the escape hatch for a
|
|
116
|
-
// mixed repo whose majority language isn't the one you want. Joern jobs only.
|
|
117
|
-
const langOverride = flag("--lang");
|
|
118
|
-
if (langOverride) {
|
|
119
|
-
const L = langOverride.toUpperCase();
|
|
120
|
-
const touched = jobs.filter((j) => j.indexer === "joern");
|
|
121
|
-
for (const j of touched) { j.gemlLang = L; j.signal += ` --lang ${L}`; }
|
|
122
|
-
if (!touched.length) console.error(`--lang ${langOverride}: no Joern language detected to override (ignored)`);
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
if (!jobs.length) {
|
|
126
|
-
console.error(`could not auto-detect a supported language under ${rootAbs}.`);
|
|
127
|
-
console.error("supported: TypeScript/JS, Rust (scip); Java, C, Python, Go, Kotlin (joern).");
|
|
128
|
-
console.error("pass an explicit --adapter scip|joern --raw <in> or --db <graph.db> instead (geml codemap build --help).");
|
|
129
|
-
process.exit(1);
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
// Space-aware quote: wrap a token in double quotes only when it contains
|
|
133
|
-
// whitespace or a quote. Used for (a) the PROGRAM token of a spawned command
|
|
134
|
-
// and (b) recording human-readable recipe steps into refresh.json. The
|
|
135
|
-
// program token must NOT be blanket-quoted: a bare launcher name resolved via
|
|
136
|
-
// PATH (npx / joern) whose .cmd/.bat shim uses %~dp0 breaks if the name is
|
|
137
|
-
// quoted — cmd then resolves %~dp0 against the cwd, not the shim's dir. A
|
|
138
|
-
// spaced launcher PATH is a full path, so quoting it keeps %~dp0 correct.
|
|
139
|
-
// WHEN it does quote, it defers to shq below: escaping only `"` left a
|
|
140
|
-
// trailing backslash (a path ending `...\`) escaping our own closing quote,
|
|
141
|
-
// which merges the next token into this one. Same CRT rules, one source.
|
|
142
|
-
const q = (s) => (/[\s"]/.test(String(s)) ? shq(s) : String(s));
|
|
143
|
-
// Hardened quote for command ARGUMENTS on win32. Node does NOT escape args
|
|
144
|
-
// under shell:true — it only concatenates them (Node DEP0190) — so an
|
|
145
|
-
// unquoted argument such as a source directory named `a&calc` reaching the
|
|
146
|
-
// --output path would break out of the command and run `calc`. ALWAYS wrap in
|
|
147
|
-
// double quotes: inside quotes cmd.exe treats & | < > ( ) ^ and whitespace as
|
|
148
|
-
// literal, neutralizing injection while keeping spaced paths intact. Embedded
|
|
149
|
-
// quotes / trailing backslash runs follow the CRT rules so the child's
|
|
150
|
-
// CommandLineToArgvW recovers the exact token.
|
|
151
|
-
const shq = (s) => `"${String(s).replace(/(\\*)"/g, '$1$1\\"').replace(/(\\+)$/, '$1$1')}"`;
|
|
152
|
-
// Run a command PORTABLY. On Windows we MUST go through cmd.exe (shell:true):
|
|
153
|
-
// npx.cmd / joern.bat / rust-analyzer.bat are .cmd/.bat launchers and modern
|
|
154
|
-
// Node refuses to spawn those with shell:false (EINVAL). Build ONE pre-escaped
|
|
155
|
-
// command string ourselves (never an args array — that is the unescaped
|
|
156
|
-
// DEP0190 path): the program via q (bare names stay bare so a shim's %~dp0
|
|
157
|
-
// resolves), every argument via shq (always quoted, so no argument can
|
|
158
|
-
// inject). On unix we exec the binary directly (no shell, no injection).
|
|
159
|
-
const runCmd = (argv, opts = {}) =>
|
|
160
|
-
(process.platform === "win32"
|
|
161
|
-
? spawnSync([q(argv[0]), ...argv.slice(1).map(shq)].join(" "), { shell: true, ...opts })
|
|
162
|
-
: spawnSync(argv[0], argv.slice(1), opts));
|
|
163
|
-
|
|
164
|
-
// Resolve the Joern launcher, honoring an explicit install location so users
|
|
165
|
-
// (Windows especially) need not put joern on PATH. A --joern / GEML_JOERN
|
|
166
|
-
// value may be the launcher itself OR the unzipped joern-cli DIRECTORY holding
|
|
167
|
-
// it (joern.bat on Windows, joern on unix). Tried in order — --joern flag,
|
|
168
|
-
// then GEML_JOERN, then `joern` on PATH — and the first that answers
|
|
169
|
-
// `--version` wins. GEML_SRC/GEML_OUT/GEML_LANG always pass as ENV VARS, never
|
|
170
|
-
// --param (the Windows joern.bat -> repl-bridge hop mangles --param).
|
|
171
|
-
const launcherName = process.platform === "win32" ? "joern.bat" : "joern";
|
|
172
|
-
const asLauncher = (v) => {
|
|
173
|
-
try { if (statSync(v).isDirectory()) return join(v, launcherName); } catch { /* not a dir: a launcher path or bare command */ }
|
|
174
|
-
return v;
|
|
175
|
-
};
|
|
176
|
-
let joernBin = null;
|
|
177
|
-
const joernJobs = jobs.filter((j) => j.indexer === "joern");
|
|
178
|
-
if (joernJobs.length) {
|
|
179
|
-
// Joern creates a `workspace/` CPG cache in its CWD on startup — even for
|
|
180
|
-
// `--version`. Probe (and, below, run) it FROM the build dir so that cache
|
|
181
|
-
// lands in _build/, never scattered at the repo root. buildDir must exist
|
|
182
|
-
// first (it is also (re)created before emit); mkdir is idempotent.
|
|
183
|
-
mkdirSync(buildDir, { recursive: true });
|
|
184
|
-
for (const cand of [flag("--joern"), process.env.GEML_JOERN, "joern"].filter((v) => v)) {
|
|
185
|
-
const bin = asLauncher(cand);
|
|
186
|
-
const r = runCmd([bin, "--version"], { stdio: "ignore", cwd: buildDir });
|
|
187
|
-
if (!r.error && r.status === 0) { joernBin = bin; break; }
|
|
188
|
-
}
|
|
189
|
-
if (!joernBin) {
|
|
190
|
-
const langs = [...new Set(joernJobs.map((j) => j.language))].join(", ");
|
|
191
|
-
console.error(
|
|
192
|
-
`Joern is required for ${langs} but was not found (looked at --joern, GEML_JOERN, then PATH).\n`
|
|
193
|
-
+ "Install one and retry:\n"
|
|
194
|
-
+ " macOS/Linux:\n"
|
|
195
|
-
+ " mkdir joern && cd joern\n"
|
|
196
|
-
+ ' curl -L "https://github.com/joernio/joern/releases/latest/download/joern-install.sh" -o joern-install.sh\n'
|
|
197
|
-
+ " chmod +x joern-install.sh && ./joern-install.sh\n"
|
|
198
|
-
+ " Windows:\n"
|
|
199
|
-
+ " download joern-cli.zip from https://github.com/joernio/joern/releases , unzip it, then\n"
|
|
200
|
-
+ " add that folder to PATH or pass --joern <unzipped-folder>\n"
|
|
201
|
-
+ "Docs: https://docs.joern.io/installation",
|
|
202
|
-
);
|
|
203
|
-
process.exit(1);
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
// Same courtesy for Rust: rust-analyzer produces the SCIP index, so probe it
|
|
208
|
-
// BEFORE any slow work and fail with install instructions instead of a
|
|
209
|
-
// mid-build spawn error. (A rustup shim without the component installed also
|
|
210
|
-
// answers `--version` non-zero, so it lands here too.)
|
|
211
|
-
if (jobs.some((j) => j.language === "Rust")) {
|
|
212
|
-
const r = runCmd(["rust-analyzer", "--version"], { stdio: "ignore" });
|
|
213
|
-
if (r.error || r.status !== 0) {
|
|
214
|
-
console.error(
|
|
215
|
-
"rust-analyzer is required for Rust but was not found on PATH (or is not runnable).\n"
|
|
216
|
-
+ "Install it and retry:\n"
|
|
217
|
-
+ " rustup component add rust-analyzer # rustup-managed toolchains\n"
|
|
218
|
-
+ " or download a release binary: https://github.com/rust-lang/rust-analyzer/releases\n"
|
|
219
|
-
+ "and make sure `rust-analyzer --version` works in this shell.",
|
|
220
|
-
);
|
|
221
|
-
process.exit(1);
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
// Transparent plan before doing any slow work.
|
|
226
|
-
// A monorepo with vendored trees (next.js's src/compiled: 140 package.json
|
|
227
|
-
// bundles) turns the full job list into a wall — summarize past 10.
|
|
228
|
-
if (jobs.length > 10) {
|
|
229
|
-
const byLang = new Map();
|
|
230
|
-
for (const j of jobs) byLang.set(j.language, (byLang.get(j.language) ?? 0) + 1);
|
|
231
|
-
const langs = [...byLang].map(([l, n]) => (n > 1 ? `${l}×${n}` : l)).join(", ");
|
|
232
|
-
const sample = jobs.slice(0, 5).map((j) => j.subroot ?? j.language).join("; ");
|
|
233
|
-
console.error(`detected: ${jobs.length} jobs (${langs}) — e.g. ${sample}; … (vendored trees inflating this? --exclude "path/**" trims them)`);
|
|
234
|
-
} else {
|
|
235
|
-
console.error(`detected: ${jobs.map((j) => `${j.language}${j.subroot ? `[${j.subroot}]` : ""} (${j.signal}) -> ${j.indexer}${j.gemlLang ? `[${j.gemlLang}]` : ""}`).join("; ")}`);
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
const scriptPath = resolve(dirname(fileURLToPath(import.meta.url)), "joern-export.sc");
|
|
239
|
-
const scriptPosix = scriptPath.replace(/\\/g, "/");
|
|
240
|
-
const sfcScript = resolve(dirname(fileURLToPath(import.meta.url)), "sfc-virtualize.mjs");
|
|
241
|
-
const sfcScriptPosix = sfcScript.replace(/\\/g, "/");
|
|
242
|
-
const relToRoot = (p) => (relative(rootAbs, p).replace(/\\/g, "/") || ".");
|
|
243
|
-
mkdirSync(buildDir, { recursive: true });
|
|
244
|
-
// Structured recipe steps { cwd?, env?, argv:[...] } (security fix R2-1).
|
|
245
|
-
// Attacker-controllable sub-project dir names appear here ONLY as DISCRETE
|
|
246
|
-
// structured values (a step's cwd, or an argv element) — NEVER concatenated
|
|
247
|
-
// into a shell string at rest. refresh executes each step without building an
|
|
248
|
-
// attacker-influenced command line (see codemap/refresh.mjs).
|
|
249
|
-
const indexSteps = [];
|
|
250
|
-
// Build a recorded step's env map, dropping undefined values so the step's
|
|
251
|
-
// fingerprint stays stable (GEML_LANG is unset for scip jobs).
|
|
252
|
-
const envOf = (obj) => {
|
|
253
|
-
const env = {};
|
|
254
|
-
for (const [k, v] of Object.entries(obj)) if (v != null) env[k] = String(v);
|
|
255
|
-
return env;
|
|
256
|
-
};
|
|
257
|
-
|
|
258
|
-
console.error("indexing...");
|
|
259
|
-
const failedLangs = [];
|
|
260
|
-
for (const job of jobs) {
|
|
261
|
-
let cmd = indexerCommand(job, { root: rootAbs, buildDir, scriptPath, sfcScript });
|
|
262
|
-
let preStep = null;
|
|
263
|
-
if (cmd.pre) {
|
|
264
|
-
// SFC job: run the virtualizer first. If it fails (offline npx, exotic
|
|
265
|
-
// SFC syntax), the project must not lose its plain TS coverage — fall
|
|
266
|
-
// back to the sfc-less job and say the gap out loud.
|
|
267
|
-
const pr = runCmd(cmd.pre.argv, {
|
|
268
|
-
cwd: cmd.pre.cwd, stdio: "inherit",
|
|
269
|
-
env: { ...process.env, ...cmd.pre.env },
|
|
270
|
-
});
|
|
271
|
-
if (pr.error || pr.status !== 0) {
|
|
272
|
-
console.error(
|
|
273
|
-
`sfc virtualizer failed for ${job.language}${job.subroot ? `[${job.subroot}]` : ""} `
|
|
274
|
-
+ `(${pr.error ? pr.error.message : `exit ${pr.status}`}) — falling back to plain TS indexing; `
|
|
275
|
-
+ ".vue/.svelte files stay invisible until this is fixed and build re-runs.",
|
|
276
|
-
);
|
|
277
|
-
cmd = indexerCommand({ ...job, sfc: undefined }, { root: rootAbs, buildDir, scriptPath, sfcScript });
|
|
278
|
-
} else {
|
|
279
|
-
// Runs at root (cmd.pre.cwd === root), so no cwd; the virtualizer reads
|
|
280
|
-
// GEML_SRC/GEML_OUT (relative to root) from env. argv[-1] is the script
|
|
281
|
-
// path — record the forward-slash form.
|
|
282
|
-
preStep = {
|
|
283
|
-
env: envOf({ GEML_SRC: relToRoot(cmd.pre.env.GEML_SRC), GEML_OUT: relToRoot(cmd.pre.env.GEML_OUT) }),
|
|
284
|
-
argv: [...cmd.pre.argv.slice(0, -1), sfcScriptPosix],
|
|
285
|
-
};
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
// scip runs the npx launcher; joern runs the resolved launcher. Env
|
|
289
|
-
// (GEML_SRC/OUT/LANG for joern) rides through the spawn options.
|
|
290
|
-
const argv = [job.indexer === "joern" ? joernBin : cmd.argv[0], ...cmd.argv.slice(1)];
|
|
291
|
-
const r = runCmd(argv, {
|
|
292
|
-
cwd: cmd.cwd, stdio: "inherit",
|
|
293
|
-
env: cmd.env ? { ...process.env, ...cmd.env } : process.env,
|
|
294
|
-
});
|
|
295
|
-
if (r.error || r.status !== 0) {
|
|
296
|
-
// One language failing must not sink the others' finished work — keep
|
|
297
|
-
// going, build what succeeded, and say the gap out loud below. Name the
|
|
298
|
-
// subroot so a monorepo says WHICH project's indexer died, not just the
|
|
299
|
-
// language (several TS projects can each have their own scip job).
|
|
300
|
-
const where = job.subroot ? `${job.language} at ${job.subroot}` : job.language;
|
|
301
|
-
console.error(`indexer failed for ${where} (${job.indexer}): ${r.error ? r.error.message : `exit ${r.status}`}`);
|
|
302
|
-
failedLangs.push(job.language);
|
|
303
|
-
continue;
|
|
304
|
-
}
|
|
305
|
-
inputs.push({ adapter: cmd.adapter, raw: cmd.raw, remap: cmd.remapDir });
|
|
306
|
-
// Recipe step (paths relative to <root>, the cwd refresh replays in) —
|
|
307
|
-
// successful steps only, so `refresh` replays a recipe that works.
|
|
308
|
-
if (preStep) indexSteps.push(preStep);
|
|
309
|
-
if (job.indexer === "scip") {
|
|
310
|
-
// Subrooted jobs replay from their own dir (SFC jobs from the virtual
|
|
311
|
-
// dir, standalone crates from the crate dir); the output path is written
|
|
312
|
-
// relative to THAT cwd, recorded as step.cwd (omitted when it is root).
|
|
313
|
-
const relRaw = relative(cmd.cwd, cmd.raw).replace(/\\/g, "/");
|
|
314
|
-
const cwdRel = relToRoot(cmd.cwd);
|
|
315
|
-
const step = {};
|
|
316
|
-
if (cwdRel !== ".") step.cwd = cwdRel;
|
|
317
|
-
step.argv = cmd.argv[0] === "npx"
|
|
318
|
-
? [...cmd.argv.slice(0, -1), relRaw]
|
|
319
|
-
: ["rust-analyzer", "scip", ".", "--output", relRaw];
|
|
320
|
-
indexSteps.push(step);
|
|
321
|
-
} else {
|
|
322
|
-
// Joern replays IN the build dir (cmd.cwd), so its workspace cache lands
|
|
323
|
-
// under _build/ on refresh too — not at the repo root. Re-base the env
|
|
324
|
-
// paths on that cwd: GEML_SRC climbs back to the root, GEML_OUT is the raw
|
|
325
|
-
// dir's name (a sibling under _build). Mirrors the subrooted-scip step.
|
|
326
|
-
const relRaw = relative(cmd.cwd, cmd.raw).replace(/\\/g, "/");
|
|
327
|
-
const srcRel = relative(cmd.cwd, rootAbs).replace(/\\/g, "/") || ".";
|
|
328
|
-
const cwdRel = relToRoot(cmd.cwd);
|
|
329
|
-
const step = {
|
|
330
|
-
env: envOf({ GEML_SRC: srcRel, GEML_OUT: relRaw, GEML_LANG: job.gemlLang }),
|
|
331
|
-
argv: ["joern", "--script", scriptPosix],
|
|
332
|
-
};
|
|
333
|
-
if (cwdRel !== ".") step.cwd = cwdRel;
|
|
334
|
-
indexSteps.push(step);
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
if (!inputs.length) {
|
|
338
|
-
console.error("every indexer failed — nothing to build.");
|
|
339
|
-
process.exit(1);
|
|
340
|
-
}
|
|
341
|
-
if (failedLangs.length) {
|
|
342
|
-
console.error(`WARNING: continuing WITHOUT ${failedLangs.join(", ")} — the codemap covers the remaining language(s) only. Fix that indexer and re-run build to fill the gap.`);
|
|
343
|
-
}
|
|
344
|
-
console.error("merging...");
|
|
345
|
-
recordRecipe = { rootAbs, indexSteps };
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
const bad = inputs.find((s) => !["crg", "joern", "scip"].includes(s.adapter) || (s.adapter === "crg" ? !s.db : !s.raw));
|
|
349
|
-
if (!inputs.length || bad) {
|
|
350
|
-
console.error(USAGE);
|
|
351
|
-
process.exit(2);
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
// Extract every input and concatenate — anchors are namespaced by language and
|
|
355
|
-
// path, so inputs don't collide; identical anchors across inputs are dropped
|
|
356
|
-
// with a warning (first input wins).
|
|
357
|
-
const symbols = [];
|
|
358
|
-
const edges = [];
|
|
359
|
-
const seenAnchors = new Set();
|
|
360
|
-
for (const spec of inputs) {
|
|
361
|
-
const { extract } = await import(`./adapters/${spec.adapter}.mjs`);
|
|
362
|
-
const r = extract(spec.adapter === "crg" ? { db: spec.db, root } : { raw: spec.raw, root, remapDir: spec.remap });
|
|
363
|
-
let dropped = 0;
|
|
364
|
-
for (const s of r.symbols) {
|
|
365
|
-
if (seenAnchors.has(s.anchor)) { dropped++; continue; }
|
|
366
|
-
seenAnchors.add(s.anchor);
|
|
367
|
-
symbols.push(s);
|
|
368
|
-
}
|
|
369
|
-
for (const e of r.edges) edges.push(e);
|
|
370
|
-
console.error(`input ${spec.adapter}: ${r.symbols.length} symbols, ${r.edges.length} edges${dropped ? ` (${dropped} duplicate anchors dropped)` : ""}`);
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
// Exclusion: drop symbols whose source file is git-ignored (default) or
|
|
374
|
-
// matches an explicit --exclude glob. Vendored copies and third-party dumps
|
|
375
|
-
// belong out of the graph; the edge tables key on surviving anchors, so
|
|
376
|
-
// dangling references simply vanish (emit skips edges whose endpoints are
|
|
377
|
-
// gone). --no-gitignore turns off the git-driven half.
|
|
378
|
-
const excludeGlobs = args.flatMap((v, i) => (args[i - 1] === "--exclude" ? [v] : []));
|
|
379
|
-
const excluder = makeExcluder({
|
|
380
|
-
root: resolve(root),
|
|
381
|
-
globs: excludeGlobs,
|
|
382
|
-
gitignore: !args.includes("--no-gitignore"),
|
|
383
|
-
files: [...new Set(symbols.map((s) => s.file))],
|
|
384
|
-
run: execFileSync,
|
|
385
|
-
});
|
|
386
|
-
const kept = symbols.filter((s) => !excluder(s.file));
|
|
387
|
-
const excludedCount = symbols.length - kept.length;
|
|
388
|
-
|
|
389
|
-
// App-entry hints for the EXPLICIT-adapter path too (auto mode computed them
|
|
390
|
-
// alongside language detection): the entry signals live in the repo's
|
|
391
|
-
// manifests and sources, not in how the indexes were produced.
|
|
392
|
-
if (root && !recordRecipe && !entryHints.length) {
|
|
393
|
-
const rootAbs = resolve(root);
|
|
394
|
-
const c = collectSourceFiles(rootAbs);
|
|
395
|
-
const excl = makeExcluder({
|
|
396
|
-
root: rootAbs, globs: excludeGlobs, gitignore: !args.includes("--no-gitignore"),
|
|
397
|
-
files: [...c.files, ...c.manifests, ...c.pkgs], run: execFileSync,
|
|
398
|
-
});
|
|
399
|
-
entryHints = detectEntries(rootAbs, {
|
|
400
|
-
files: c.files.filter((f) => !excl(f)),
|
|
401
|
-
manifests: c.manifests.filter((m) => !excl(m)),
|
|
402
|
-
pkgs: c.pkgs.filter((p) => !excl(p)),
|
|
403
|
-
});
|
|
404
|
-
}
|
|
405
|
-
if (excludedCount) {
|
|
406
|
-
symbols.length = 0;
|
|
407
|
-
for (const s of kept) symbols.push(s);
|
|
408
|
-
console.error(
|
|
409
|
-
`excluded ${excludedCount} symbol(s) via ${!args.includes("--no-gitignore") ? ".gitignore" : "(gitignore off)"}`
|
|
410
|
-
+ `${excludeGlobs.length ? ` + ${excludeGlobs.length} --exclude glob(s)` : ""}`,
|
|
411
|
-
);
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
// Cross-stack API links: detect frontend HTTP call sites + backend route
|
|
415
|
-
// declarations across the merged graph and append `http` edges wiring each
|
|
416
|
-
// caller's enclosing symbol to the handler's — the seam that joins a full
|
|
417
|
-
// stack's two otherwise-disjoint trees. Kept OUT of the verified `calls`
|
|
418
|
-
// relation (own edge kind + confidence). Best-effort: a detector failure must
|
|
419
|
-
// never sink an otherwise-good build.
|
|
420
|
-
try {
|
|
421
|
-
const rootAbs = resolve(root);
|
|
422
|
-
const scanFiles = [...new Set(symbols.map((s) => s.file))];
|
|
423
|
-
const { edges: httpEdges, audit } = buildCrossStackOverlay({
|
|
424
|
-
symbols,
|
|
425
|
-
files: scanFiles,
|
|
426
|
-
// `rel` is indexer OUTPUT, not a path this build authored, so a `..`
|
|
427
|
-
// segment must not turn a source read into an escape from the scanned
|
|
428
|
-
// root. Same gate the document resolver uses: resolve first, then require
|
|
429
|
-
// the result to BE the root or sit under it — compared with the separator,
|
|
430
|
-
// so a sibling named `<root>-evil` is not mistaken for a child.
|
|
431
|
-
readText: (rel) => {
|
|
432
|
-
try {
|
|
433
|
-
const p = resolve(rootAbs, ...String(rel).split("/"));
|
|
434
|
-
if (p !== rootAbs && !p.startsWith(rootAbs + sep)) return null;
|
|
435
|
-
return readFileSync(p, "utf8");
|
|
436
|
-
} catch { return null; }
|
|
437
|
-
},
|
|
438
|
-
});
|
|
439
|
-
for (const e of httpEdges) edges.push(e);
|
|
440
|
-
if (httpEdges.length) {
|
|
441
|
-
console.error(
|
|
442
|
-
`cross-stack: ${httpEdges.length} api link(s) across ${audit.endpoints} endpoint(s)`
|
|
443
|
-
+ (audit.divergent.length ? `, ${audit.divergent.length} method-divergent` : "")
|
|
444
|
-
+ (audit.deadRoutes.length ? `, ${audit.deadRoutes.length} uncalled route(s)` : ""));
|
|
445
|
-
mkdirSync(join(outDir, "_index"), { recursive: true });
|
|
446
|
-
const auditPath = join(outDir, "_index", "cross-stack.json");
|
|
447
|
-
const auditContent = JSON.stringify(audit, null, 2) + "\n";
|
|
448
|
-
if (!existsSync(auditPath) || readFileSync(auditPath, "utf8") !== auditContent) writeFileSync(auditPath, auditContent);
|
|
449
|
-
}
|
|
450
|
-
} catch (e) {
|
|
451
|
-
console.error(`cross-stack overlay skipped: ${e.message}`);
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
// Exchange format on disk — the layer contract (§3). Deterministic order so
|
|
455
|
-
// the jsonl files diff cleanly across builds.
|
|
456
|
-
symbols.sort((a, b) => a.anchor.localeCompare(b.anchor));
|
|
457
|
-
edges.sort((a, b) =>
|
|
458
|
-
String(a.from ?? a.from_text ?? "").localeCompare(String(b.from ?? b.from_text ?? "")) || a.kind.localeCompare(b.kind)
|
|
459
|
-
|| String(a.to ?? a.to_text).localeCompare(String(b.to ?? b.to_text)));
|
|
460
|
-
mkdirSync(buildDir, { recursive: true });
|
|
461
|
-
const jsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
462
|
-
const writeIfChanged = (p, content) => {
|
|
463
|
-
if (existsSync(p) && readFileSync(p, "utf8") === content) return;
|
|
464
|
-
writeFileSync(p, content);
|
|
465
|
-
};
|
|
466
|
-
writeIfChanged(join(buildDir, "symbols.jsonl"), jsonl(symbols));
|
|
467
|
-
writeIfChanged(join(buildDir, "edges.jsonl"), jsonl(edges));
|
|
468
|
-
|
|
469
|
-
// Container granularity (codemap profile): module = first path segment,
|
|
470
|
-
// dir = containing directory (default), file = one document per source file.
|
|
471
|
-
const containerGranularity = flag("--container", "dir");
|
|
472
|
-
if (!["module", "dir", "file"].includes(containerGranularity)) {
|
|
473
|
-
console.error(`--container must be module|dir|file (got '${containerGranularity}')`);
|
|
474
|
-
process.exit(2);
|
|
475
|
-
}
|
|
476
|
-
// Best-effort commit stamp for index.geml meta.
|
|
477
|
-
let commit;
|
|
478
|
-
try {
|
|
479
|
-
commit = execFileSync("git", ["-C", resolve(root), "rev-parse", "--short", "HEAD"], { encoding: "utf8" }).trim();
|
|
480
|
-
} catch { /* not a git repo */ }
|
|
481
|
-
|
|
482
|
-
// Ceremony-folding config: read _index/foldings.geml, or seed it on this first
|
|
483
|
-
// build from the discovered module roots + detected languages. Human-owned
|
|
484
|
-
// once seeded (never rewritten); threaded into emit for display normalisation.
|
|
485
|
-
const { config: foldings, seeded: foldingsSeeded } = loadOrSeedFoldings({
|
|
486
|
-
outDir,
|
|
487
|
-
moduleRoots: discoverModuleRoots(resolve(root)),
|
|
488
|
-
languages: detectedLanguages,
|
|
489
|
-
});
|
|
490
|
-
if (foldingsSeeded) console.error("seeded _index/foldings.geml — edit to tune module folding");
|
|
491
|
-
|
|
492
|
-
const stats = emit({
|
|
493
|
-
symbols, edges, outDir, buildDir,
|
|
494
|
-
repoName: basename(resolve(root)),
|
|
495
|
-
container: containerGranularity,
|
|
496
|
-
commit,
|
|
497
|
-
root: resolve(root),
|
|
498
|
-
foldings,
|
|
499
|
-
entryHints,
|
|
500
|
-
});
|
|
501
|
-
|
|
502
|
-
// Keep the transient build dir out of version control while the `.geml` graph
|
|
503
|
-
// and `_index/` stay committable (the graph is meant to be committed & shared).
|
|
504
|
-
// `_build/` holds only regenerable intermediates — the *.jsonl exchange files
|
|
505
|
-
// and Joern's `workspace/` CPG cache — so one ignore rule covers them all.
|
|
506
|
-
// Written once; a user's later edits to this file are preserved, never clobbered.
|
|
507
|
-
const ignoreFile = join(outDir, ".gitignore");
|
|
508
|
-
if (!existsSync(ignoreFile)) writeFileSync(ignoreFile, "_build/\n");
|
|
509
|
-
|
|
510
|
-
console.error(
|
|
511
|
-
`geml-code-graph: ${stats.methods} methods (${stats.symbols} symbols), ${stats.edges} edges `
|
|
512
|
-
+ `(${stats.resolved} resolved), ${stats.leaves} leaves, ${stats.entries} app entries -> `
|
|
513
|
-
+ `${stats.containers} containers (${stats.written} of ${stats.docs} files written), `
|
|
514
|
-
+ `${(stats.bytes / 1048576).toFixed(2)} MB -> ${outDir}`,
|
|
515
|
-
);
|
|
516
|
-
// A build that deletes files says which ones. Silence here is how the orphans
|
|
517
|
-
// accumulated in the first place — name them, so a rename that drops a whole
|
|
518
|
-
// naming scheme is visible in the log rather than three renamings later.
|
|
519
|
-
if (stats.pruned?.length) {
|
|
520
|
-
console.error(` pruned ${stats.pruned.length} document(s) no longer produced: ${stats.pruned.join(", ")}`);
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
// --history: snapshot every changed document into its .gemlhistory sidecar —
|
|
524
|
-
// the graph's own architectural history (geml history get / revert per node).
|
|
525
|
-
// Targets = documents rewritten this build, plus any document that has no
|
|
526
|
-
// sidecar yet (first run, or --history adopted later).
|
|
527
|
-
if (args.includes("--history")) {
|
|
528
|
-
const histMod = resolve(dirname(fileURLToPath(import.meta.url)), "../dist/history.js");
|
|
529
|
-
if (!existsSync(histMod)) {
|
|
530
|
-
console.error("--history needs the built parser (cd geml-parser && npm install && npm run build)");
|
|
531
|
-
process.exit(1);
|
|
532
|
-
}
|
|
533
|
-
const { save, isCurrent } = await import(`file://${histMod.replace(/\\/g, "/")}`);
|
|
534
|
-
const message = flag("-m", flag("--message", "graph build"));
|
|
535
|
-
const targets = new Set(stats.writtenDocs);
|
|
536
|
-
for (const d of stats.allDocs) {
|
|
537
|
-
if (targets.has(d)) continue;
|
|
538
|
-
const gemlPath = join(outDir, d);
|
|
539
|
-
const sidecar = gemlPath.replace(/\.geml$/, ".gemlhistory");
|
|
540
|
-
// No sidecar yet, or the sidecar tip drifted from the file (a previous
|
|
541
|
-
// save was refused): both need a snapshot even though this build
|
|
542
|
-
// did not rewrite the document.
|
|
543
|
-
if (!existsSync(sidecar) || !isCurrent(sidecar, gemlPath)) targets.add(d);
|
|
544
|
-
}
|
|
545
|
-
let committed = 0;
|
|
546
|
-
const histFailed = [];
|
|
547
|
-
for (const d of [...targets].sort()) {
|
|
548
|
-
const gemlPath = join(outDir, d);
|
|
549
|
-
try {
|
|
550
|
-
save({ gemlPath, historyPath: gemlPath.replace(/\.geml$/, ".gemlhistory"), summary: message });
|
|
551
|
-
committed++;
|
|
552
|
-
} catch (e) {
|
|
553
|
-
// One document's history refusing a save (e.g. the round-trip gate)
|
|
554
|
-
// must not abort the build or the other documents' snapshots. The
|
|
555
|
-
// failing document's previous revision stays intact.
|
|
556
|
-
histFailed.push(d);
|
|
557
|
-
console.error(`history: ${d}: ${e.message}`);
|
|
558
|
-
}
|
|
559
|
-
}
|
|
560
|
-
console.error(
|
|
561
|
-
`history: committed ${committed} document(s) (${stats.allDocs.length - committed - histFailed.length} unchanged, skipped)`
|
|
562
|
-
+ (histFailed.length ? `; FAILED: ${histFailed.join(", ")}` : ""),
|
|
563
|
-
);
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
// Auto mode records the exact replay recipe (index → explicit build → verify)
|
|
567
|
-
// into _index/refresh.json on the FIRST build, so `geml codemap refresh` (and
|
|
568
|
-
// the commit hook) can reproduce it. An existing recipe is left untouched —
|
|
569
|
-
// EXCEPT one whose on-disk schema `version` does not match RECIPE_VERSION:
|
|
570
|
-
// refresh refuses an out-of-date recipe, so a rebuild re-records it in the
|
|
571
|
-
// current format. Judging by a standalone schema version (not the parser
|
|
572
|
-
// version, which bumps every patch) means a FUTURE format change is cleanly
|
|
573
|
-
// detected without a parser bump forcing a needless re-index. This is the
|
|
574
|
-
// "re-run build" upgrade path refresh points users to; a recipe already at the
|
|
575
|
-
// current version stays write-once (not clobbered). Paths are relative to
|
|
576
|
-
// <root>, which is the cwd refresh runs each step in.
|
|
577
|
-
if (recordRecipe) {
|
|
578
|
-
const cfgPath = join(outDir, "_index", "refresh.json");
|
|
579
|
-
let needsRerecord = false;
|
|
580
|
-
if (existsSync(cfgPath)) {
|
|
581
|
-
try { needsRerecord = JSON.parse(readFileSync(cfgPath, "utf8")).version !== RECIPE_VERSION; }
|
|
582
|
-
catch { needsRerecord = true; } // unparseable → re-record clean
|
|
583
|
-
}
|
|
584
|
-
if (!existsSync(cfgPath) || needsRerecord) {
|
|
585
|
-
const rel = (p) => (relative(recordRecipe.rootAbs, p).replace(/\\/g, "/") || ".");
|
|
586
|
-
const relOut = rel(outDir);
|
|
587
|
-
// Structured build + verify steps (security fix R2-1): argv arrays, never a
|
|
588
|
-
// shell string. Each `--adapter/--raw[/--remap]` group is discrete tokens.
|
|
589
|
-
const buildArgv = ["geml", "codemap", "build",
|
|
590
|
-
...inputs.flatMap((s) => ["--adapter", s.adapter, "--raw", rel(s.raw), ...(s.remap ? ["--remap", rel(s.remap)] : [])]),
|
|
591
|
-
"--root", ".", "--out", relOut,
|
|
592
|
-
...(containerGranularity !== "dir" ? ["--container", containerGranularity] : []),
|
|
593
|
-
...(args.includes("--history") ? ["--history"] : []),
|
|
594
|
-
];
|
|
595
|
-
// Parser version — recorded as `generator` PROVENANCE only, never as part of
|
|
596
|
-
// the compatibility check or the fingerprint (it bumps every patch release;
|
|
597
|
-
// judging by it would force a full re-index of every project each release).
|
|
598
|
-
const pkgVersion = (() => {
|
|
599
|
-
try { return JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version; }
|
|
600
|
-
catch { return "?"; }
|
|
601
|
-
})();
|
|
602
|
-
const cfg = {
|
|
603
|
-
version: RECIPE_VERSION,
|
|
604
|
-
generator: `geml ${pkgVersion}`,
|
|
605
|
-
// Project root relative to the codemap dir (refresh runs each step under
|
|
606
|
-
// <root>). Normally outDir is a subdir of root so relative() yields ".."
|
|
607
|
-
// etc.; when --out == --root it yields "" and the project root IS the
|
|
608
|
-
// codemap dir, so record "." — recording ".." would send refresh into
|
|
609
|
-
// the PARENT of the real root.
|
|
610
|
-
root: relative(outDir, recordRecipe.rootAbs).replace(/\\/g, "/") || ".",
|
|
611
|
-
steps: [...recordRecipe.indexSteps, { argv: buildArgv }, { argv: ["geml", "codemap", "verify", relOut] }],
|
|
612
|
-
};
|
|
613
|
-
mkdirSync(join(outDir, "_index"), { recursive: true });
|
|
614
|
-
writeFileSync(cfgPath, JSON.stringify(cfg, null, 2) + "\n");
|
|
615
|
-
console.error(`recorded build recipe -> ${cfgPath}`);
|
|
616
|
-
// Auto-trust the recipe we just authored (security fix C2). The user ran
|
|
617
|
-
// build locally, so their own recipe is trusted by construction and the
|
|
618
|
-
// normal build -> refresh flow needs no prompt. Uses the SAME fingerprint
|
|
619
|
-
// fn as refresh, so the two agree exactly. Best-effort: a trust-store write
|
|
620
|
-
// failure must not fail an otherwise-successful build — the user can still
|
|
621
|
-
// approve later with `geml codemap refresh --trust`.
|
|
622
|
-
try {
|
|
623
|
-
trustRecipe(recipeFingerprint(cfg), outDir);
|
|
624
|
-
} catch (e) {
|
|
625
|
-
console.error(`warning: could not record the codemap recipe as trusted (${e.message}); run \`geml codemap refresh --trust\` after reviewing _index/refresh.json`);
|
|
626
|
-
}
|
|
627
|
-
}
|
|
628
|
-
}
|
|
629
|
-
console.error(`next: geml codemap verify ${outDir}`);
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// geml-code-graph build — one shot: adapter → exchange format (build/) → GEML tree (graph/).
|
|
3
|
+
//
|
|
4
|
+
// geml codemap build --root <repo-root> # AUTO: detect languages,
|
|
5
|
+
// # run the right indexer(s), merge into one map
|
|
6
|
+
// geml codemap build --db <graph.db> --root <repo-root> # crg
|
|
7
|
+
// geml codemap build --adapter joern --raw <dir> \
|
|
8
|
+
// --adapter scip --raw <index.scip> --root <repo> # merged multi-language
|
|
9
|
+
// geml codemap build --adapter joern --raw <dir> --root <repo-root> # joern
|
|
10
|
+
// [--out .geml-code-graph] [--build .geml-code-graph/_build]
|
|
11
|
+
// [--container module|dir|file] container granularity (default dir)
|
|
12
|
+
// [--lang <JAVASRC|NEWC|…>] force the Joern frontend (auto mode,
|
|
13
|
+
// mixed-majority repos)
|
|
14
|
+
// [--joern <path>] Joern install (auto mode): the launcher OR the
|
|
15
|
+
// unzipped joern-cli dir; else GEML_JOERN, else PATH
|
|
16
|
+
// [--history [-m "msg"]] snapshot changed docs into .gemlhistory
|
|
17
|
+
// sidecars (per-node history + revert)
|
|
18
|
+
//
|
|
19
|
+
// Auto mode (no --adapter and no --db, just --root): detect.mjs picks the
|
|
20
|
+
// indexer per language from manifests + source extensions, we run scip
|
|
21
|
+
// (npx @sourcegraph/scip-typescript for TS/JS, rust-analyzer scip for Rust)
|
|
22
|
+
// and/or Joern (joern-export.sc) into <out>/_build/, then feed the results
|
|
23
|
+
// into the SAME merge as the explicit --adapter path, and record the replay
|
|
24
|
+
// recipe into _index/refresh.json.
|
|
25
|
+
//
|
|
26
|
+
// Output shape: docs/codemap-profile.md — one document per container (single
|
|
27
|
+
// meta with module/src/entry, empty-body code blocks with src=/anchor=, and
|
|
28
|
+
// the #calls / #called-by / #unresolved CSV edge tables). Verify with
|
|
29
|
+
// geml codemap verify (geml check + profile reference checks).
|
|
30
|
+
//
|
|
31
|
+
// Adapters (docs/DESIGN-geml-code-graph.md §3):
|
|
32
|
+
// crg code-review-graph SQLite graph.db (tree-sitter level; everything
|
|
33
|
+
// honestly labelled resolution:"heuristic") [P0, default]
|
|
34
|
+
// joern Joern CPG export: run geml-parser/codemap/joern-export.sc inside
|
|
35
|
+
// joern first; --raw points at its outDir [P1]
|
|
36
|
+
//
|
|
37
|
+
// After building, run: geml codemap verify <out-dir>
|
|
38
|
+
import { writeFileSync, mkdirSync, existsSync, readFileSync, statSync } from "node:fs";
|
|
39
|
+
import { join, resolve, basename, dirname, relative, sep } from "node:path";
|
|
40
|
+
import { fileURLToPath } from "node:url";
|
|
41
|
+
import { execFileSync, spawnSync } from "node:child_process";
|
|
42
|
+
import { emit } from "./emit.mjs";
|
|
43
|
+
import { makeExcluder } from "./exclude.mjs";
|
|
44
|
+
import { detectLanguages, indexerCommand, collectSourceFiles } from "./detect.mjs";
|
|
45
|
+
import { loadOrSeedFoldings } from "./foldings.mjs";
|
|
46
|
+
import { detectEntries } from "./entries.mjs";
|
|
47
|
+
import { discoverModuleRoots } from "./normalize.mjs";
|
|
48
|
+
import { recipeFingerprint, trustRecipe, RECIPE_VERSION } from "./recipe-trust.mjs";
|
|
49
|
+
import { buildCrossStackOverlay } from "./cross-stack.mjs";
|
|
50
|
+
|
|
51
|
+
const args = process.argv.slice(2);
|
|
52
|
+
const flag = (name, dflt) => {
|
|
53
|
+
const i = args.indexOf(name);
|
|
54
|
+
return i >= 0 ? args[i + 1] : dflt;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const USAGE = [
|
|
58
|
+
"usage: geml codemap build [--root <repo-root>] # auto-detect languages, index, and merge (--root defaults to the current directory)",
|
|
59
|
+
" or: geml codemap build (--db <graph.db> | --adapter joern|scip --raw <dir|index.scip> [--remap <virtual-dir>])+ [--root <repo-root>] [--out .geml-code-graph] [--build .geml-code-graph/_build] [--container module|dir|file] [--lang <LANG>] [--joern <path>] [--exclude <glob>]... [--no-gitignore] [--history [-m msg]]",
|
|
60
|
+
].join("\n");
|
|
61
|
+
if (args.includes("--help") || args.includes("-h")) { console.log(USAGE); process.exit(0); }
|
|
62
|
+
|
|
63
|
+
// --root defaults to the current directory, so `geml codemap build` with no
|
|
64
|
+
// arguments indexes the repo you're standing in.
|
|
65
|
+
const root = flag("--root", ".");
|
|
66
|
+
const outDir = resolve(flag("--out", ".geml-code-graph"));
|
|
67
|
+
// Intermediates live INSIDE the codemap dir (alongside _index) so a build
|
|
68
|
+
// leaves nothing scattered at the repo root — `.geml-code-graph/_build/`.
|
|
69
|
+
const buildDir = resolve(flag("--build", join(outDir, "_build")));
|
|
70
|
+
|
|
71
|
+
// Adapter inputs are REPEATABLE — one codemap can merge several extractions
|
|
72
|
+
// (e.g. Joern for the Java modules + SCIP for the TypeScript ones). Each
|
|
73
|
+
// `--adapter X` opens a group; the following `--db`/`--raw` belongs to it.
|
|
74
|
+
// A bare `--db` without `--adapter` keeps the historical crg default.
|
|
75
|
+
const inputs = [];
|
|
76
|
+
{
|
|
77
|
+
let cur = null;
|
|
78
|
+
for (let i = 0; i < args.length; i++) {
|
|
79
|
+
if (args[i] === "--adapter") { cur = { adapter: args[++i] }; inputs.push(cur); }
|
|
80
|
+
else if (args[i] === "--db") { if (!cur) { cur = { adapter: "crg" }; inputs.push(cur); } cur.db = args[++i]; cur = null; }
|
|
81
|
+
else if (args[i] === "--raw") { if (!cur) { console.error("--raw needs a preceding --adapter"); process.exit(2); } cur.raw = args[++i]; cur = null; }
|
|
82
|
+
// --remap <virtual dir>: the preceding scip input was produced over an
|
|
83
|
+
// SFC virtual dir (sfc-virtualize.mjs) — the adapter maps shadow paths
|
|
84
|
+
// back to the original .vue/.svelte sources. Recorded into refresh.json
|
|
85
|
+
// so replays keep the remapping.
|
|
86
|
+
else if (args[i] === "--remap") { if (!inputs.length) { console.error("--remap needs a preceding --adapter/--raw group"); process.exit(2); } inputs[inputs.length - 1].remap = args[++i]; }
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
// ---- auto-detect mode -------------------------------------------------------
|
|
90
|
+
// A --root with NO --adapter and NO --db (inputs still empty): detect the
|
|
91
|
+
// languages ourselves, run the right indexer(s) into <out>/_build/, then push
|
|
92
|
+
// the results into `inputs` so the merge below runs UNCHANGED. This is the
|
|
93
|
+
// one-command onboarding path; the explicit --adapter/--db paths are untouched.
|
|
94
|
+
let recordRecipe = null; // { rootAbs, steps } — written to refresh.json after emit
|
|
95
|
+
let detectedLanguages = []; // languages seen in auto-detect; seeds foldings' language conventions
|
|
96
|
+
let entryHints = []; // app-entry hints (entries.mjs), matched to symbols in emit
|
|
97
|
+
if (root && !inputs.length) {
|
|
98
|
+
const rootAbs = resolve(root);
|
|
99
|
+
const excludeGlobs0 = args.flatMap((v, i) => (args[i - 1] === "--exclude" ? [v] : []));
|
|
100
|
+
const { files, manifests, pkgs } = collectSourceFiles(rootAbs);
|
|
101
|
+
const excluder = makeExcluder({
|
|
102
|
+
root: rootAbs, globs: excludeGlobs0, gitignore: !args.includes("--no-gitignore"),
|
|
103
|
+
files: [...files, ...manifests, ...pkgs], run: execFileSync,
|
|
104
|
+
});
|
|
105
|
+
const jobs = detectLanguages(rootAbs, { files, manifests, pkgs, excluder });
|
|
106
|
+
detectedLanguages = [...new Set(jobs.map((j) => j.language))];
|
|
107
|
+
// App-entry hints from manifests/layout/source markers — pure detection now,
|
|
108
|
+
// matched to extracted symbols (or noted file-level) inside emit.
|
|
109
|
+
entryHints = detectEntries(rootAbs, {
|
|
110
|
+
files: files.filter((f) => !excluder(f)),
|
|
111
|
+
manifests: manifests.filter((m) => !excluder(m)),
|
|
112
|
+
pkgs: (pkgs ?? []).filter((p) => !excluder(p)),
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
// --lang forces the Joern frontend (GEML_LANG) — the escape hatch for a
|
|
116
|
+
// mixed repo whose majority language isn't the one you want. Joern jobs only.
|
|
117
|
+
const langOverride = flag("--lang");
|
|
118
|
+
if (langOverride) {
|
|
119
|
+
const L = langOverride.toUpperCase();
|
|
120
|
+
const touched = jobs.filter((j) => j.indexer === "joern");
|
|
121
|
+
for (const j of touched) { j.gemlLang = L; j.signal += ` --lang ${L}`; }
|
|
122
|
+
if (!touched.length) console.error(`--lang ${langOverride}: no Joern language detected to override (ignored)`);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (!jobs.length) {
|
|
126
|
+
console.error(`could not auto-detect a supported language under ${rootAbs}.`);
|
|
127
|
+
console.error("supported: TypeScript/JS, Rust (scip); Java, C, Python, Go, Kotlin (joern).");
|
|
128
|
+
console.error("pass an explicit --adapter scip|joern --raw <in> or --db <graph.db> instead (geml codemap build --help).");
|
|
129
|
+
process.exit(1);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Space-aware quote: wrap a token in double quotes only when it contains
|
|
133
|
+
// whitespace or a quote. Used for (a) the PROGRAM token of a spawned command
|
|
134
|
+
// and (b) recording human-readable recipe steps into refresh.json. The
|
|
135
|
+
// program token must NOT be blanket-quoted: a bare launcher name resolved via
|
|
136
|
+
// PATH (npx / joern) whose .cmd/.bat shim uses %~dp0 breaks if the name is
|
|
137
|
+
// quoted — cmd then resolves %~dp0 against the cwd, not the shim's dir. A
|
|
138
|
+
// spaced launcher PATH is a full path, so quoting it keeps %~dp0 correct.
|
|
139
|
+
// WHEN it does quote, it defers to shq below: escaping only `"` left a
|
|
140
|
+
// trailing backslash (a path ending `...\`) escaping our own closing quote,
|
|
141
|
+
// which merges the next token into this one. Same CRT rules, one source.
|
|
142
|
+
const q = (s) => (/[\s"]/.test(String(s)) ? shq(s) : String(s));
|
|
143
|
+
// Hardened quote for command ARGUMENTS on win32. Node does NOT escape args
|
|
144
|
+
// under shell:true — it only concatenates them (Node DEP0190) — so an
|
|
145
|
+
// unquoted argument such as a source directory named `a&calc` reaching the
|
|
146
|
+
// --output path would break out of the command and run `calc`. ALWAYS wrap in
|
|
147
|
+
// double quotes: inside quotes cmd.exe treats & | < > ( ) ^ and whitespace as
|
|
148
|
+
// literal, neutralizing injection while keeping spaced paths intact. Embedded
|
|
149
|
+
// quotes / trailing backslash runs follow the CRT rules so the child's
|
|
150
|
+
// CommandLineToArgvW recovers the exact token.
|
|
151
|
+
const shq = (s) => `"${String(s).replace(/(\\*)"/g, '$1$1\\"').replace(/(\\+)$/, '$1$1')}"`;
|
|
152
|
+
// Run a command PORTABLY. On Windows we MUST go through cmd.exe (shell:true):
|
|
153
|
+
// npx.cmd / joern.bat / rust-analyzer.bat are .cmd/.bat launchers and modern
|
|
154
|
+
// Node refuses to spawn those with shell:false (EINVAL). Build ONE pre-escaped
|
|
155
|
+
// command string ourselves (never an args array — that is the unescaped
|
|
156
|
+
// DEP0190 path): the program via q (bare names stay bare so a shim's %~dp0
|
|
157
|
+
// resolves), every argument via shq (always quoted, so no argument can
|
|
158
|
+
// inject). On unix we exec the binary directly (no shell, no injection).
|
|
159
|
+
const runCmd = (argv, opts = {}) =>
|
|
160
|
+
(process.platform === "win32"
|
|
161
|
+
? spawnSync([q(argv[0]), ...argv.slice(1).map(shq)].join(" "), { shell: true, ...opts })
|
|
162
|
+
: spawnSync(argv[0], argv.slice(1), opts));
|
|
163
|
+
|
|
164
|
+
// Resolve the Joern launcher, honoring an explicit install location so users
|
|
165
|
+
// (Windows especially) need not put joern on PATH. A --joern / GEML_JOERN
|
|
166
|
+
// value may be the launcher itself OR the unzipped joern-cli DIRECTORY holding
|
|
167
|
+
// it (joern.bat on Windows, joern on unix). Tried in order — --joern flag,
|
|
168
|
+
// then GEML_JOERN, then `joern` on PATH — and the first that answers
|
|
169
|
+
// `--version` wins. GEML_SRC/GEML_OUT/GEML_LANG always pass as ENV VARS, never
|
|
170
|
+
// --param (the Windows joern.bat -> repl-bridge hop mangles --param).
|
|
171
|
+
const launcherName = process.platform === "win32" ? "joern.bat" : "joern";
|
|
172
|
+
const asLauncher = (v) => {
|
|
173
|
+
try { if (statSync(v).isDirectory()) return join(v, launcherName); } catch { /* not a dir: a launcher path or bare command */ }
|
|
174
|
+
return v;
|
|
175
|
+
};
|
|
176
|
+
let joernBin = null;
|
|
177
|
+
const joernJobs = jobs.filter((j) => j.indexer === "joern");
|
|
178
|
+
if (joernJobs.length) {
|
|
179
|
+
// Joern creates a `workspace/` CPG cache in its CWD on startup — even for
|
|
180
|
+
// `--version`. Probe (and, below, run) it FROM the build dir so that cache
|
|
181
|
+
// lands in _build/, never scattered at the repo root. buildDir must exist
|
|
182
|
+
// first (it is also (re)created before emit); mkdir is idempotent.
|
|
183
|
+
mkdirSync(buildDir, { recursive: true });
|
|
184
|
+
for (const cand of [flag("--joern"), process.env.GEML_JOERN, "joern"].filter((v) => v)) {
|
|
185
|
+
const bin = asLauncher(cand);
|
|
186
|
+
const r = runCmd([bin, "--version"], { stdio: "ignore", cwd: buildDir });
|
|
187
|
+
if (!r.error && r.status === 0) { joernBin = bin; break; }
|
|
188
|
+
}
|
|
189
|
+
if (!joernBin) {
|
|
190
|
+
const langs = [...new Set(joernJobs.map((j) => j.language))].join(", ");
|
|
191
|
+
console.error(
|
|
192
|
+
`Joern is required for ${langs} but was not found (looked at --joern, GEML_JOERN, then PATH).\n`
|
|
193
|
+
+ "Install one and retry:\n"
|
|
194
|
+
+ " macOS/Linux:\n"
|
|
195
|
+
+ " mkdir joern && cd joern\n"
|
|
196
|
+
+ ' curl -L "https://github.com/joernio/joern/releases/latest/download/joern-install.sh" -o joern-install.sh\n'
|
|
197
|
+
+ " chmod +x joern-install.sh && ./joern-install.sh\n"
|
|
198
|
+
+ " Windows:\n"
|
|
199
|
+
+ " download joern-cli.zip from https://github.com/joernio/joern/releases , unzip it, then\n"
|
|
200
|
+
+ " add that folder to PATH or pass --joern <unzipped-folder>\n"
|
|
201
|
+
+ "Docs: https://docs.joern.io/installation",
|
|
202
|
+
);
|
|
203
|
+
process.exit(1);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// Same courtesy for Rust: rust-analyzer produces the SCIP index, so probe it
|
|
208
|
+
// BEFORE any slow work and fail with install instructions instead of a
|
|
209
|
+
// mid-build spawn error. (A rustup shim without the component installed also
|
|
210
|
+
// answers `--version` non-zero, so it lands here too.)
|
|
211
|
+
if (jobs.some((j) => j.language === "Rust")) {
|
|
212
|
+
const r = runCmd(["rust-analyzer", "--version"], { stdio: "ignore" });
|
|
213
|
+
if (r.error || r.status !== 0) {
|
|
214
|
+
console.error(
|
|
215
|
+
"rust-analyzer is required for Rust but was not found on PATH (or is not runnable).\n"
|
|
216
|
+
+ "Install it and retry:\n"
|
|
217
|
+
+ " rustup component add rust-analyzer # rustup-managed toolchains\n"
|
|
218
|
+
+ " or download a release binary: https://github.com/rust-lang/rust-analyzer/releases\n"
|
|
219
|
+
+ "and make sure `rust-analyzer --version` works in this shell.",
|
|
220
|
+
);
|
|
221
|
+
process.exit(1);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// Transparent plan before doing any slow work.
|
|
226
|
+
// A monorepo with vendored trees (next.js's src/compiled: 140 package.json
|
|
227
|
+
// bundles) turns the full job list into a wall — summarize past 10.
|
|
228
|
+
if (jobs.length > 10) {
|
|
229
|
+
const byLang = new Map();
|
|
230
|
+
for (const j of jobs) byLang.set(j.language, (byLang.get(j.language) ?? 0) + 1);
|
|
231
|
+
const langs = [...byLang].map(([l, n]) => (n > 1 ? `${l}×${n}` : l)).join(", ");
|
|
232
|
+
const sample = jobs.slice(0, 5).map((j) => j.subroot ?? j.language).join("; ");
|
|
233
|
+
console.error(`detected: ${jobs.length} jobs (${langs}) — e.g. ${sample}; … (vendored trees inflating this? --exclude "path/**" trims them)`);
|
|
234
|
+
} else {
|
|
235
|
+
console.error(`detected: ${jobs.map((j) => `${j.language}${j.subroot ? `[${j.subroot}]` : ""} (${j.signal}) -> ${j.indexer}${j.gemlLang ? `[${j.gemlLang}]` : ""}`).join("; ")}`);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const scriptPath = resolve(dirname(fileURLToPath(import.meta.url)), "joern-export.sc");
|
|
239
|
+
const scriptPosix = scriptPath.replace(/\\/g, "/");
|
|
240
|
+
const sfcScript = resolve(dirname(fileURLToPath(import.meta.url)), "sfc-virtualize.mjs");
|
|
241
|
+
const sfcScriptPosix = sfcScript.replace(/\\/g, "/");
|
|
242
|
+
const relToRoot = (p) => (relative(rootAbs, p).replace(/\\/g, "/") || ".");
|
|
243
|
+
mkdirSync(buildDir, { recursive: true });
|
|
244
|
+
// Structured recipe steps { cwd?, env?, argv:[...] } (security fix R2-1).
|
|
245
|
+
// Attacker-controllable sub-project dir names appear here ONLY as DISCRETE
|
|
246
|
+
// structured values (a step's cwd, or an argv element) — NEVER concatenated
|
|
247
|
+
// into a shell string at rest. refresh executes each step without building an
|
|
248
|
+
// attacker-influenced command line (see codemap/refresh.mjs).
|
|
249
|
+
const indexSteps = [];
|
|
250
|
+
// Build a recorded step's env map, dropping undefined values so the step's
|
|
251
|
+
// fingerprint stays stable (GEML_LANG is unset for scip jobs).
|
|
252
|
+
const envOf = (obj) => {
|
|
253
|
+
const env = {};
|
|
254
|
+
for (const [k, v] of Object.entries(obj)) if (v != null) env[k] = String(v);
|
|
255
|
+
return env;
|
|
256
|
+
};
|
|
257
|
+
|
|
258
|
+
console.error("indexing...");
|
|
259
|
+
const failedLangs = [];
|
|
260
|
+
for (const job of jobs) {
|
|
261
|
+
let cmd = indexerCommand(job, { root: rootAbs, buildDir, scriptPath, sfcScript });
|
|
262
|
+
let preStep = null;
|
|
263
|
+
if (cmd.pre) {
|
|
264
|
+
// SFC job: run the virtualizer first. If it fails (offline npx, exotic
|
|
265
|
+
// SFC syntax), the project must not lose its plain TS coverage — fall
|
|
266
|
+
// back to the sfc-less job and say the gap out loud.
|
|
267
|
+
const pr = runCmd(cmd.pre.argv, {
|
|
268
|
+
cwd: cmd.pre.cwd, stdio: "inherit",
|
|
269
|
+
env: { ...process.env, ...cmd.pre.env },
|
|
270
|
+
});
|
|
271
|
+
if (pr.error || pr.status !== 0) {
|
|
272
|
+
console.error(
|
|
273
|
+
`sfc virtualizer failed for ${job.language}${job.subroot ? `[${job.subroot}]` : ""} `
|
|
274
|
+
+ `(${pr.error ? pr.error.message : `exit ${pr.status}`}) — falling back to plain TS indexing; `
|
|
275
|
+
+ ".vue/.svelte files stay invisible until this is fixed and build re-runs.",
|
|
276
|
+
);
|
|
277
|
+
cmd = indexerCommand({ ...job, sfc: undefined }, { root: rootAbs, buildDir, scriptPath, sfcScript });
|
|
278
|
+
} else {
|
|
279
|
+
// Runs at root (cmd.pre.cwd === root), so no cwd; the virtualizer reads
|
|
280
|
+
// GEML_SRC/GEML_OUT (relative to root) from env. argv[-1] is the script
|
|
281
|
+
// path — record the forward-slash form.
|
|
282
|
+
preStep = {
|
|
283
|
+
env: envOf({ GEML_SRC: relToRoot(cmd.pre.env.GEML_SRC), GEML_OUT: relToRoot(cmd.pre.env.GEML_OUT) }),
|
|
284
|
+
argv: [...cmd.pre.argv.slice(0, -1), sfcScriptPosix],
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
// scip runs the npx launcher; joern runs the resolved launcher. Env
|
|
289
|
+
// (GEML_SRC/OUT/LANG for joern) rides through the spawn options.
|
|
290
|
+
const argv = [job.indexer === "joern" ? joernBin : cmd.argv[0], ...cmd.argv.slice(1)];
|
|
291
|
+
const r = runCmd(argv, {
|
|
292
|
+
cwd: cmd.cwd, stdio: "inherit",
|
|
293
|
+
env: cmd.env ? { ...process.env, ...cmd.env } : process.env,
|
|
294
|
+
});
|
|
295
|
+
if (r.error || r.status !== 0) {
|
|
296
|
+
// One language failing must not sink the others' finished work — keep
|
|
297
|
+
// going, build what succeeded, and say the gap out loud below. Name the
|
|
298
|
+
// subroot so a monorepo says WHICH project's indexer died, not just the
|
|
299
|
+
// language (several TS projects can each have their own scip job).
|
|
300
|
+
const where = job.subroot ? `${job.language} at ${job.subroot}` : job.language;
|
|
301
|
+
console.error(`indexer failed for ${where} (${job.indexer}): ${r.error ? r.error.message : `exit ${r.status}`}`);
|
|
302
|
+
failedLangs.push(job.language);
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
inputs.push({ adapter: cmd.adapter, raw: cmd.raw, remap: cmd.remapDir });
|
|
306
|
+
// Recipe step (paths relative to <root>, the cwd refresh replays in) —
|
|
307
|
+
// successful steps only, so `refresh` replays a recipe that works.
|
|
308
|
+
if (preStep) indexSteps.push(preStep);
|
|
309
|
+
if (job.indexer === "scip") {
|
|
310
|
+
// Subrooted jobs replay from their own dir (SFC jobs from the virtual
|
|
311
|
+
// dir, standalone crates from the crate dir); the output path is written
|
|
312
|
+
// relative to THAT cwd, recorded as step.cwd (omitted when it is root).
|
|
313
|
+
const relRaw = relative(cmd.cwd, cmd.raw).replace(/\\/g, "/");
|
|
314
|
+
const cwdRel = relToRoot(cmd.cwd);
|
|
315
|
+
const step = {};
|
|
316
|
+
if (cwdRel !== ".") step.cwd = cwdRel;
|
|
317
|
+
step.argv = cmd.argv[0] === "npx"
|
|
318
|
+
? [...cmd.argv.slice(0, -1), relRaw]
|
|
319
|
+
: ["rust-analyzer", "scip", ".", "--output", relRaw];
|
|
320
|
+
indexSteps.push(step);
|
|
321
|
+
} else {
|
|
322
|
+
// Joern replays IN the build dir (cmd.cwd), so its workspace cache lands
|
|
323
|
+
// under _build/ on refresh too — not at the repo root. Re-base the env
|
|
324
|
+
// paths on that cwd: GEML_SRC climbs back to the root, GEML_OUT is the raw
|
|
325
|
+
// dir's name (a sibling under _build). Mirrors the subrooted-scip step.
|
|
326
|
+
const relRaw = relative(cmd.cwd, cmd.raw).replace(/\\/g, "/");
|
|
327
|
+
const srcRel = relative(cmd.cwd, rootAbs).replace(/\\/g, "/") || ".";
|
|
328
|
+
const cwdRel = relToRoot(cmd.cwd);
|
|
329
|
+
const step = {
|
|
330
|
+
env: envOf({ GEML_SRC: srcRel, GEML_OUT: relRaw, GEML_LANG: job.gemlLang }),
|
|
331
|
+
argv: ["joern", "--script", scriptPosix],
|
|
332
|
+
};
|
|
333
|
+
if (cwdRel !== ".") step.cwd = cwdRel;
|
|
334
|
+
indexSteps.push(step);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
if (!inputs.length) {
|
|
338
|
+
console.error("every indexer failed — nothing to build.");
|
|
339
|
+
process.exit(1);
|
|
340
|
+
}
|
|
341
|
+
if (failedLangs.length) {
|
|
342
|
+
console.error(`WARNING: continuing WITHOUT ${failedLangs.join(", ")} — the codemap covers the remaining language(s) only. Fix that indexer and re-run build to fill the gap.`);
|
|
343
|
+
}
|
|
344
|
+
console.error("merging...");
|
|
345
|
+
recordRecipe = { rootAbs, indexSteps };
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
const bad = inputs.find((s) => !["crg", "joern", "scip"].includes(s.adapter) || (s.adapter === "crg" ? !s.db : !s.raw));
|
|
349
|
+
if (!inputs.length || bad) {
|
|
350
|
+
console.error(USAGE);
|
|
351
|
+
process.exit(2);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// Extract every input and concatenate — anchors are namespaced by language and
|
|
355
|
+
// path, so inputs don't collide; identical anchors across inputs are dropped
|
|
356
|
+
// with a warning (first input wins).
|
|
357
|
+
const symbols = [];
|
|
358
|
+
const edges = [];
|
|
359
|
+
const seenAnchors = new Set();
|
|
360
|
+
for (const spec of inputs) {
|
|
361
|
+
const { extract } = await import(`./adapters/${spec.adapter}.mjs`);
|
|
362
|
+
const r = extract(spec.adapter === "crg" ? { db: spec.db, root } : { raw: spec.raw, root, remapDir: spec.remap });
|
|
363
|
+
let dropped = 0;
|
|
364
|
+
for (const s of r.symbols) {
|
|
365
|
+
if (seenAnchors.has(s.anchor)) { dropped++; continue; }
|
|
366
|
+
seenAnchors.add(s.anchor);
|
|
367
|
+
symbols.push(s);
|
|
368
|
+
}
|
|
369
|
+
for (const e of r.edges) edges.push(e);
|
|
370
|
+
console.error(`input ${spec.adapter}: ${r.symbols.length} symbols, ${r.edges.length} edges${dropped ? ` (${dropped} duplicate anchors dropped)` : ""}`);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// Exclusion: drop symbols whose source file is git-ignored (default) or
|
|
374
|
+
// matches an explicit --exclude glob. Vendored copies and third-party dumps
|
|
375
|
+
// belong out of the graph; the edge tables key on surviving anchors, so
|
|
376
|
+
// dangling references simply vanish (emit skips edges whose endpoints are
|
|
377
|
+
// gone). --no-gitignore turns off the git-driven half.
|
|
378
|
+
const excludeGlobs = args.flatMap((v, i) => (args[i - 1] === "--exclude" ? [v] : []));
|
|
379
|
+
const excluder = makeExcluder({
|
|
380
|
+
root: resolve(root),
|
|
381
|
+
globs: excludeGlobs,
|
|
382
|
+
gitignore: !args.includes("--no-gitignore"),
|
|
383
|
+
files: [...new Set(symbols.map((s) => s.file))],
|
|
384
|
+
run: execFileSync,
|
|
385
|
+
});
|
|
386
|
+
const kept = symbols.filter((s) => !excluder(s.file));
|
|
387
|
+
const excludedCount = symbols.length - kept.length;
|
|
388
|
+
|
|
389
|
+
// App-entry hints for the EXPLICIT-adapter path too (auto mode computed them
|
|
390
|
+
// alongside language detection): the entry signals live in the repo's
|
|
391
|
+
// manifests and sources, not in how the indexes were produced.
|
|
392
|
+
if (root && !recordRecipe && !entryHints.length) {
|
|
393
|
+
const rootAbs = resolve(root);
|
|
394
|
+
const c = collectSourceFiles(rootAbs);
|
|
395
|
+
const excl = makeExcluder({
|
|
396
|
+
root: rootAbs, globs: excludeGlobs, gitignore: !args.includes("--no-gitignore"),
|
|
397
|
+
files: [...c.files, ...c.manifests, ...c.pkgs], run: execFileSync,
|
|
398
|
+
});
|
|
399
|
+
entryHints = detectEntries(rootAbs, {
|
|
400
|
+
files: c.files.filter((f) => !excl(f)),
|
|
401
|
+
manifests: c.manifests.filter((m) => !excl(m)),
|
|
402
|
+
pkgs: c.pkgs.filter((p) => !excl(p)),
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
if (excludedCount) {
|
|
406
|
+
symbols.length = 0;
|
|
407
|
+
for (const s of kept) symbols.push(s);
|
|
408
|
+
console.error(
|
|
409
|
+
`excluded ${excludedCount} symbol(s) via ${!args.includes("--no-gitignore") ? ".gitignore" : "(gitignore off)"}`
|
|
410
|
+
+ `${excludeGlobs.length ? ` + ${excludeGlobs.length} --exclude glob(s)` : ""}`,
|
|
411
|
+
);
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Cross-stack API links: detect frontend HTTP call sites + backend route
|
|
415
|
+
// declarations across the merged graph and append `http` edges wiring each
|
|
416
|
+
// caller's enclosing symbol to the handler's — the seam that joins a full
|
|
417
|
+
// stack's two otherwise-disjoint trees. Kept OUT of the verified `calls`
|
|
418
|
+
// relation (own edge kind + confidence). Best-effort: a detector failure must
|
|
419
|
+
// never sink an otherwise-good build.
|
|
420
|
+
try {
|
|
421
|
+
const rootAbs = resolve(root);
|
|
422
|
+
const scanFiles = [...new Set(symbols.map((s) => s.file))];
|
|
423
|
+
const { edges: httpEdges, audit } = buildCrossStackOverlay({
|
|
424
|
+
symbols,
|
|
425
|
+
files: scanFiles,
|
|
426
|
+
// `rel` is indexer OUTPUT, not a path this build authored, so a `..`
|
|
427
|
+
// segment must not turn a source read into an escape from the scanned
|
|
428
|
+
// root. Same gate the document resolver uses: resolve first, then require
|
|
429
|
+
// the result to BE the root or sit under it — compared with the separator,
|
|
430
|
+
// so a sibling named `<root>-evil` is not mistaken for a child.
|
|
431
|
+
readText: (rel) => {
|
|
432
|
+
try {
|
|
433
|
+
const p = resolve(rootAbs, ...String(rel).split("/"));
|
|
434
|
+
if (p !== rootAbs && !p.startsWith(rootAbs + sep)) return null;
|
|
435
|
+
return readFileSync(p, "utf8");
|
|
436
|
+
} catch { return null; }
|
|
437
|
+
},
|
|
438
|
+
});
|
|
439
|
+
for (const e of httpEdges) edges.push(e);
|
|
440
|
+
if (httpEdges.length) {
|
|
441
|
+
console.error(
|
|
442
|
+
`cross-stack: ${httpEdges.length} api link(s) across ${audit.endpoints} endpoint(s)`
|
|
443
|
+
+ (audit.divergent.length ? `, ${audit.divergent.length} method-divergent` : "")
|
|
444
|
+
+ (audit.deadRoutes.length ? `, ${audit.deadRoutes.length} uncalled route(s)` : ""));
|
|
445
|
+
mkdirSync(join(outDir, "_index"), { recursive: true });
|
|
446
|
+
const auditPath = join(outDir, "_index", "cross-stack.json");
|
|
447
|
+
const auditContent = JSON.stringify(audit, null, 2) + "\n";
|
|
448
|
+
if (!existsSync(auditPath) || readFileSync(auditPath, "utf8") !== auditContent) writeFileSync(auditPath, auditContent);
|
|
449
|
+
}
|
|
450
|
+
} catch (e) {
|
|
451
|
+
console.error(`cross-stack overlay skipped: ${e.message}`);
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// Exchange format on disk — the layer contract (§3). Deterministic order so
|
|
455
|
+
// the jsonl files diff cleanly across builds.
|
|
456
|
+
symbols.sort((a, b) => a.anchor.localeCompare(b.anchor));
|
|
457
|
+
edges.sort((a, b) =>
|
|
458
|
+
String(a.from ?? a.from_text ?? "").localeCompare(String(b.from ?? b.from_text ?? "")) || a.kind.localeCompare(b.kind)
|
|
459
|
+
|| String(a.to ?? a.to_text).localeCompare(String(b.to ?? b.to_text)));
|
|
460
|
+
mkdirSync(buildDir, { recursive: true });
|
|
461
|
+
const jsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
462
|
+
const writeIfChanged = (p, content) => {
|
|
463
|
+
if (existsSync(p) && readFileSync(p, "utf8") === content) return;
|
|
464
|
+
writeFileSync(p, content);
|
|
465
|
+
};
|
|
466
|
+
writeIfChanged(join(buildDir, "symbols.jsonl"), jsonl(symbols));
|
|
467
|
+
writeIfChanged(join(buildDir, "edges.jsonl"), jsonl(edges));
|
|
468
|
+
|
|
469
|
+
// Container granularity (codemap profile): module = first path segment,
|
|
470
|
+
// dir = containing directory (default), file = one document per source file.
|
|
471
|
+
const containerGranularity = flag("--container", "dir");
|
|
472
|
+
if (!["module", "dir", "file"].includes(containerGranularity)) {
|
|
473
|
+
console.error(`--container must be module|dir|file (got '${containerGranularity}')`);
|
|
474
|
+
process.exit(2);
|
|
475
|
+
}
|
|
476
|
+
// Best-effort commit stamp for index.geml meta.
|
|
477
|
+
let commit;
|
|
478
|
+
try {
|
|
479
|
+
commit = execFileSync("git", ["-C", resolve(root), "rev-parse", "--short", "HEAD"], { encoding: "utf8" }).trim();
|
|
480
|
+
} catch { /* not a git repo */ }
|
|
481
|
+
|
|
482
|
+
// Ceremony-folding config: read _index/foldings.geml, or seed it on this first
|
|
483
|
+
// build from the discovered module roots + detected languages. Human-owned
|
|
484
|
+
// once seeded (never rewritten); threaded into emit for display normalisation.
|
|
485
|
+
const { config: foldings, seeded: foldingsSeeded } = loadOrSeedFoldings({
|
|
486
|
+
outDir,
|
|
487
|
+
moduleRoots: discoverModuleRoots(resolve(root)),
|
|
488
|
+
languages: detectedLanguages,
|
|
489
|
+
});
|
|
490
|
+
if (foldingsSeeded) console.error("seeded _index/foldings.geml — edit to tune module folding");
|
|
491
|
+
|
|
492
|
+
const stats = emit({
|
|
493
|
+
symbols, edges, outDir, buildDir,
|
|
494
|
+
repoName: basename(resolve(root)),
|
|
495
|
+
container: containerGranularity,
|
|
496
|
+
commit,
|
|
497
|
+
root: resolve(root),
|
|
498
|
+
foldings,
|
|
499
|
+
entryHints,
|
|
500
|
+
});
|
|
501
|
+
|
|
502
|
+
// Keep the transient build dir out of version control while the `.geml` graph
|
|
503
|
+
// and `_index/` stay committable (the graph is meant to be committed & shared).
|
|
504
|
+
// `_build/` holds only regenerable intermediates — the *.jsonl exchange files
|
|
505
|
+
// and Joern's `workspace/` CPG cache — so one ignore rule covers them all.
|
|
506
|
+
// Written once; a user's later edits to this file are preserved, never clobbered.
|
|
507
|
+
const ignoreFile = join(outDir, ".gitignore");
|
|
508
|
+
if (!existsSync(ignoreFile)) writeFileSync(ignoreFile, "_build/\n");
|
|
509
|
+
|
|
510
|
+
console.error(
|
|
511
|
+
`geml-code-graph: ${stats.methods} methods (${stats.symbols} symbols), ${stats.edges} edges `
|
|
512
|
+
+ `(${stats.resolved} resolved), ${stats.leaves} leaves, ${stats.entries} app entries -> `
|
|
513
|
+
+ `${stats.containers} containers (${stats.written} of ${stats.docs} files written), `
|
|
514
|
+
+ `${(stats.bytes / 1048576).toFixed(2)} MB -> ${outDir}`,
|
|
515
|
+
);
|
|
516
|
+
// A build that deletes files says which ones. Silence here is how the orphans
|
|
517
|
+
// accumulated in the first place — name them, so a rename that drops a whole
|
|
518
|
+
// naming scheme is visible in the log rather than three renamings later.
|
|
519
|
+
if (stats.pruned?.length) {
|
|
520
|
+
console.error(` pruned ${stats.pruned.length} document(s) no longer produced: ${stats.pruned.join(", ")}`);
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// --history: snapshot every changed document into its .gemlhistory sidecar —
|
|
524
|
+
// the graph's own architectural history (geml history get / revert per node).
|
|
525
|
+
// Targets = documents rewritten this build, plus any document that has no
|
|
526
|
+
// sidecar yet (first run, or --history adopted later).
|
|
527
|
+
if (args.includes("--history")) {
|
|
528
|
+
const histMod = resolve(dirname(fileURLToPath(import.meta.url)), "../dist/history.js");
|
|
529
|
+
if (!existsSync(histMod)) {
|
|
530
|
+
console.error("--history needs the built parser (cd geml-parser && npm install && npm run build)");
|
|
531
|
+
process.exit(1);
|
|
532
|
+
}
|
|
533
|
+
const { save, isCurrent } = await import(`file://${histMod.replace(/\\/g, "/")}`);
|
|
534
|
+
const message = flag("-m", flag("--message", "graph build"));
|
|
535
|
+
const targets = new Set(stats.writtenDocs);
|
|
536
|
+
for (const d of stats.allDocs) {
|
|
537
|
+
if (targets.has(d)) continue;
|
|
538
|
+
const gemlPath = join(outDir, d);
|
|
539
|
+
const sidecar = gemlPath.replace(/\.geml$/, ".gemlhistory");
|
|
540
|
+
// No sidecar yet, or the sidecar tip drifted from the file (a previous
|
|
541
|
+
// save was refused): both need a snapshot even though this build
|
|
542
|
+
// did not rewrite the document.
|
|
543
|
+
if (!existsSync(sidecar) || !isCurrent(sidecar, gemlPath)) targets.add(d);
|
|
544
|
+
}
|
|
545
|
+
let committed = 0;
|
|
546
|
+
const histFailed = [];
|
|
547
|
+
for (const d of [...targets].sort()) {
|
|
548
|
+
const gemlPath = join(outDir, d);
|
|
549
|
+
try {
|
|
550
|
+
save({ gemlPath, historyPath: gemlPath.replace(/\.geml$/, ".gemlhistory"), summary: message });
|
|
551
|
+
committed++;
|
|
552
|
+
} catch (e) {
|
|
553
|
+
// One document's history refusing a save (e.g. the round-trip gate)
|
|
554
|
+
// must not abort the build or the other documents' snapshots. The
|
|
555
|
+
// failing document's previous revision stays intact.
|
|
556
|
+
histFailed.push(d);
|
|
557
|
+
console.error(`history: ${d}: ${e.message}`);
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
console.error(
|
|
561
|
+
`history: committed ${committed} document(s) (${stats.allDocs.length - committed - histFailed.length} unchanged, skipped)`
|
|
562
|
+
+ (histFailed.length ? `; FAILED: ${histFailed.join(", ")}` : ""),
|
|
563
|
+
);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
// Auto mode records the exact replay recipe (index → explicit build → verify)
|
|
567
|
+
// into _index/refresh.json on the FIRST build, so `geml codemap refresh` (and
|
|
568
|
+
// the commit hook) can reproduce it. An existing recipe is left untouched —
|
|
569
|
+
// EXCEPT one whose on-disk schema `version` does not match RECIPE_VERSION:
|
|
570
|
+
// refresh refuses an out-of-date recipe, so a rebuild re-records it in the
|
|
571
|
+
// current format. Judging by a standalone schema version (not the parser
|
|
572
|
+
// version, which bumps every patch) means a FUTURE format change is cleanly
|
|
573
|
+
// detected without a parser bump forcing a needless re-index. This is the
|
|
574
|
+
// "re-run build" upgrade path refresh points users to; a recipe already at the
|
|
575
|
+
// current version stays write-once (not clobbered). Paths are relative to
|
|
576
|
+
// <root>, which is the cwd refresh runs each step in.
|
|
577
|
+
if (recordRecipe) {
|
|
578
|
+
const cfgPath = join(outDir, "_index", "refresh.json");
|
|
579
|
+
let needsRerecord = false;
|
|
580
|
+
if (existsSync(cfgPath)) {
|
|
581
|
+
try { needsRerecord = JSON.parse(readFileSync(cfgPath, "utf8")).version !== RECIPE_VERSION; }
|
|
582
|
+
catch { needsRerecord = true; } // unparseable → re-record clean
|
|
583
|
+
}
|
|
584
|
+
if (!existsSync(cfgPath) || needsRerecord) {
|
|
585
|
+
const rel = (p) => (relative(recordRecipe.rootAbs, p).replace(/\\/g, "/") || ".");
|
|
586
|
+
const relOut = rel(outDir);
|
|
587
|
+
// Structured build + verify steps (security fix R2-1): argv arrays, never a
|
|
588
|
+
// shell string. Each `--adapter/--raw[/--remap]` group is discrete tokens.
|
|
589
|
+
const buildArgv = ["geml", "codemap", "build",
|
|
590
|
+
...inputs.flatMap((s) => ["--adapter", s.adapter, "--raw", rel(s.raw), ...(s.remap ? ["--remap", rel(s.remap)] : [])]),
|
|
591
|
+
"--root", ".", "--out", relOut,
|
|
592
|
+
...(containerGranularity !== "dir" ? ["--container", containerGranularity] : []),
|
|
593
|
+
...(args.includes("--history") ? ["--history"] : []),
|
|
594
|
+
];
|
|
595
|
+
// Parser version — recorded as `generator` PROVENANCE only, never as part of
|
|
596
|
+
// the compatibility check or the fingerprint (it bumps every patch release;
|
|
597
|
+
// judging by it would force a full re-index of every project each release).
|
|
598
|
+
const pkgVersion = (() => {
|
|
599
|
+
try { return JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version; }
|
|
600
|
+
catch { return "?"; }
|
|
601
|
+
})();
|
|
602
|
+
const cfg = {
|
|
603
|
+
version: RECIPE_VERSION,
|
|
604
|
+
generator: `geml ${pkgVersion}`,
|
|
605
|
+
// Project root relative to the codemap dir (refresh runs each step under
|
|
606
|
+
// <root>). Normally outDir is a subdir of root so relative() yields ".."
|
|
607
|
+
// etc.; when --out == --root it yields "" and the project root IS the
|
|
608
|
+
// codemap dir, so record "." — recording ".." would send refresh into
|
|
609
|
+
// the PARENT of the real root.
|
|
610
|
+
root: relative(outDir, recordRecipe.rootAbs).replace(/\\/g, "/") || ".",
|
|
611
|
+
steps: [...recordRecipe.indexSteps, { argv: buildArgv }, { argv: ["geml", "codemap", "verify", relOut] }],
|
|
612
|
+
};
|
|
613
|
+
mkdirSync(join(outDir, "_index"), { recursive: true });
|
|
614
|
+
writeFileSync(cfgPath, JSON.stringify(cfg, null, 2) + "\n");
|
|
615
|
+
console.error(`recorded build recipe -> ${cfgPath}`);
|
|
616
|
+
// Auto-trust the recipe we just authored (security fix C2). The user ran
|
|
617
|
+
// build locally, so their own recipe is trusted by construction and the
|
|
618
|
+
// normal build -> refresh flow needs no prompt. Uses the SAME fingerprint
|
|
619
|
+
// fn as refresh, so the two agree exactly. Best-effort: a trust-store write
|
|
620
|
+
// failure must not fail an otherwise-successful build — the user can still
|
|
621
|
+
// approve later with `geml codemap refresh --trust`.
|
|
622
|
+
try {
|
|
623
|
+
trustRecipe(recipeFingerprint(cfg), outDir);
|
|
624
|
+
} catch (e) {
|
|
625
|
+
console.error(`warning: could not record the codemap recipe as trusted (${e.message}); run \`geml codemap refresh --trust\` after reviewing _index/refresh.json`);
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
console.error(`next: geml codemap verify ${outDir}`);
|