@geml/geml 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +79 -72
- package/codemap/adapters/crg.mjs +109 -0
- package/codemap/adapters/joern.mjs +131 -0
- package/codemap/adapters/scip.mjs +229 -0
- package/codemap/browser-stub.mjs +24 -0
- package/codemap/build.mjs +354 -0
- package/codemap/detect.mjs +185 -0
- package/codemap/emit.mjs +361 -0
- package/codemap/exclude.mjs +52 -0
- package/codemap/joern-export.sc +83 -0
- package/codemap/mcp-server.mjs +143 -0
- package/codemap/normalize.mjs +0 -0
- package/codemap/refresh.mjs +160 -0
- package/codemap/render-all.mjs +64 -0
- package/codemap/serve.mjs +378 -0
- package/codemap/verify.mjs +126 -0
- package/dist/geml.d.ts +5 -0
- package/dist/geml.js +390 -14
- package/dist/history.d.ts +30 -0
- package/dist/history.js +153 -13
- package/dist/render.d.ts +59 -0
- package/dist/render.js +1685 -11
- package/dist/to-md.js +1 -3
- package/package.json +9 -5
package/codemap/emit.mjs
ADDED
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
// geml-code-graph emit — exchange-format symbols/edges → the codemap document
|
|
2
|
+
// tree of docs/DESIGN-codemap-delta.md (§4–6) / docs/codemap-profile.md:
|
|
3
|
+
// one .geml per container (module|dir|file), each with EXACTLY ONE meta
|
|
4
|
+
// (module/src/entry/resolution-default), empty-body `code` blocks per method
|
|
5
|
+
// (src=path#Lx-y, anchor=; symbol-level classes .leaf/.test), and up to three
|
|
6
|
+
// CSV edge tables: #calls (out), #called-by (in, aggregated), #unresolved
|
|
7
|
+
// (blind spots, hidden). Plus index.geml (aggregates) and name-lookup.json.
|
|
8
|
+
//
|
|
9
|
+
// Generated documents are PURE DATA: no diagram blocks (a codemap-aware
|
|
10
|
+
// renderer offers the layered-flow view; embedding elsewhere uses
|
|
11
|
+
// `=== diagram {format=geml-code-graph src=…}`).
|
|
12
|
+
//
|
|
13
|
+
// Emission is deterministic: stable sort orders everywhere; a file is only
|
|
14
|
+
// written when its bytes changed (mtime = "what a change touched").
|
|
15
|
+
import { createHash } from "node:crypto";
|
|
16
|
+
import { mkdirSync, readFileSync, writeFileSync, existsSync } from "node:fs";
|
|
17
|
+
import { dirname, join, posix } from "node:path";
|
|
18
|
+
import { buildNormalizer } from "./normalize.mjs";
|
|
19
|
+
|
|
20
|
+
const esc = (s) => String(s).replace(/`/g, "'");
|
|
21
|
+
const attrVal = (s) => String(s).replace(/"/g, "'");
|
|
22
|
+
// Plain-text cells: no commas/newlines (CSV), and no square brackets — table
|
|
23
|
+
// cells are inline-parsed, so `f[i](&x)` would otherwise read as a LINK with an
|
|
24
|
+
// unresolvable target. Brackets become parens: still readable, never markup.
|
|
25
|
+
const csvCell = (s) => String(s).replace(/[,\r\n]/g, " ").replace(/\[/g, "(").replace(/\]/g, ")").trim();
|
|
26
|
+
const sha6 = (s, len = 6) => createHash("sha256").update(s, "utf8").digest("hex").slice(0, len);
|
|
27
|
+
|
|
28
|
+
// Test territory (path conventions; the avowed heuristic of GEP-0002).
|
|
29
|
+
const TEST_DIR = /(^|\/)(test|tests|testing|__tests__|spec|specs)(\/|$)/i;
|
|
30
|
+
const TEST_FILE = /(^test_|^tests?\.|[._-]tests?\.|\.test\.|\.spec\.)/i;
|
|
31
|
+
const isTestPath = (p) => {
|
|
32
|
+
p = String(p).replace(/\\/g, "/");
|
|
33
|
+
return TEST_DIR.test(p) || TEST_FILE.test(p.slice(p.lastIndexOf("/") + 1));
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
const slugName = (name) => {
|
|
37
|
+
let s = String(name).replace(/[^A-Za-z0-9_-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").slice(0, 32);
|
|
38
|
+
if (!/^[A-Za-z]/.test(s)) s = "s" + s;
|
|
39
|
+
return s;
|
|
40
|
+
};
|
|
41
|
+
const slugPath = (p) => (p === "" || p === "(root)" ? "root" : p.replace(/\//g, "--").replace(/[^A-Za-z0-9_.-]/g, "-"));
|
|
42
|
+
const dirOf = (rel) => { const i = rel.lastIndexOf("/"); return i < 0 ? "(root)" : rel.slice(0, i); };
|
|
43
|
+
const topOf = (rel) => { const i = rel.indexOf("/"); return i < 0 ? "(root)" : rel.slice(0, i); };
|
|
44
|
+
|
|
45
|
+
export function emit({ symbols, edges, outDir, buildDir, repoName, container = "dir", commit, root }) {
|
|
46
|
+
const byAnchor = new Map(symbols.map((s) => [s.anchor, s]));
|
|
47
|
+
const methods = symbols.filter((s) => s.kind === "Function" || s.kind === "Test");
|
|
48
|
+
const files = symbols.filter((s) => s.kind === "File");
|
|
49
|
+
|
|
50
|
+
// ---- containers ----
|
|
51
|
+
const containerOf = (s) =>
|
|
52
|
+
container === "file" ? s.file : container === "module" ? topOf(s.file) : dirOf(s.file);
|
|
53
|
+
// Display-path normalisation: strip each module's shared ceremony prefix so
|
|
54
|
+
// `module=`/doc names read as the real structure. Applies in every container
|
|
55
|
+
// mode (dir and file) — a file-mode container path carries the same source
|
|
56
|
+
// roots (geml-parser/src/render.ts -> geml-parser/render.ts). Grouping still
|
|
57
|
+
// keys on the TRUE path (containerOf) and `src=` stays the true path — only
|
|
58
|
+
// the displayed module path shortens. root may be absent (older callers / crg
|
|
59
|
+
// tier): then displayOf is the identity.
|
|
60
|
+
const normMap = root ? buildNormalizer(root, methods.map(containerOf), { repoName, fileMode: container === "file" }) : new Map();
|
|
61
|
+
const displayOf = (name) => normMap.get(name) ?? name;
|
|
62
|
+
const containers = new Map(); // name -> { docName, methods[], files[] }
|
|
63
|
+
const taken = new Set(["index.geml"]);
|
|
64
|
+
const containerFor = (name) => {
|
|
65
|
+
if (!containers.has(name)) {
|
|
66
|
+
let doc = `${slugPath(displayOf(name))}.geml`;
|
|
67
|
+
for (let i = 2; taken.has(doc); i++) doc = `${slugPath(displayOf(name))}-${i}.geml`;
|
|
68
|
+
taken.add(doc);
|
|
69
|
+
containers.set(name, { docName: doc, methods: [], files: [] });
|
|
70
|
+
}
|
|
71
|
+
return containers.get(name);
|
|
72
|
+
};
|
|
73
|
+
for (const s of methods) containerFor(containerOf(s)).methods.push(s);
|
|
74
|
+
for (const s of files) {
|
|
75
|
+
const c = containers.get(containerOf(s));
|
|
76
|
+
if (c) c.files.push(s); // only files that actually host methods
|
|
77
|
+
}
|
|
78
|
+
const docOfAnchor = new Map();
|
|
79
|
+
for (const [name, c] of containers) {
|
|
80
|
+
for (const s of [...c.methods, ...c.files]) docOfAnchor.set(s.anchor, c.docName);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// ---- block ids: short name when unique in its doc, else name-<sha6(anchor)> ----
|
|
84
|
+
const idOf = new Map(); // anchor -> id
|
|
85
|
+
for (const [, c] of containers) {
|
|
86
|
+
const byName = new Map();
|
|
87
|
+
for (const s of [...c.methods, ...c.files]) {
|
|
88
|
+
const base = slugName(s.name);
|
|
89
|
+
if (!byName.has(base)) byName.set(base, []);
|
|
90
|
+
byName.get(base).push(s);
|
|
91
|
+
}
|
|
92
|
+
for (const [base, list] of byName) {
|
|
93
|
+
if (list.length === 1) { idOf.set(list[0].anchor, base); continue; }
|
|
94
|
+
// Escalate hash length only when the default 6 hex chars actually collide
|
|
95
|
+
// within this name group — keeps ids short in the common case.
|
|
96
|
+
let len = 6;
|
|
97
|
+
let ids;
|
|
98
|
+
for (;;) {
|
|
99
|
+
ids = list.map((s) => `${base}-${sha6(s.anchor, len)}`);
|
|
100
|
+
if (new Set(ids).size === ids.length) break;
|
|
101
|
+
// sha256 hex is 64 chars — beyond that only IDENTICAL anchors can
|
|
102
|
+
// still collide, which is a caller bug (build.mjs dedupes anchors):
|
|
103
|
+
// fail loudly instead of escalating forever.
|
|
104
|
+
if (len >= 64) throw new Error(`emit: duplicate anchors in name group "${base}" — anchors must be unique`);
|
|
105
|
+
len += 2;
|
|
106
|
+
}
|
|
107
|
+
list.forEach((s, i) => idOf.set(s.anchor, ids[i]));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// ---- edges (stage A: the `calls` relation only) ----
|
|
112
|
+
const calls = edges.filter((e) => e.kind === "calls");
|
|
113
|
+
const outCalls = new Map(); // anchor -> total outgoing (incl. unresolved) — leaf rule
|
|
114
|
+
const inBySym = new Map(); // target anchor -> [{fromAnchor, kind, site, confidence}]
|
|
115
|
+
const outBySym = new Map(); // source anchor -> [{toAnchor, kind, confidence}] resolved
|
|
116
|
+
const unresBySym = new Map(); // source anchor -> Set(to_text)
|
|
117
|
+
const addIn = (target, rec) => {
|
|
118
|
+
if (!byAnchor.has(target)) return;
|
|
119
|
+
if (!inBySym.has(target)) inBySym.set(target, []);
|
|
120
|
+
inBySym.get(target).push(rec);
|
|
121
|
+
};
|
|
122
|
+
for (const e of calls) {
|
|
123
|
+
if (!docOfAnchor.has(e.from)) continue;
|
|
124
|
+
outCalls.set(e.from, (outCalls.get(e.from) ?? 0) + 1);
|
|
125
|
+
if (e.to !== undefined && docOfAnchor.has(e.to)) {
|
|
126
|
+
if (!outBySym.has(e.from)) outBySym.set(e.from, []);
|
|
127
|
+
const conf = e.confidence === "high" || !e.confidence ? "" : e.confidence;
|
|
128
|
+
outBySym.get(e.from).push({ to: e.to, kind: "call", confidence: conf });
|
|
129
|
+
addIn(e.to, { from: e.from, kind: "call", site: e.site, confidence: conf });
|
|
130
|
+
for (const c of e.candidates ?? []) {
|
|
131
|
+
if (!docOfAnchor.has(c)) continue;
|
|
132
|
+
outBySym.get(e.from).push({ to: c, kind: "candidate", confidence: "" });
|
|
133
|
+
addIn(c, { from: e.from, kind: "candidate", site: e.site, confidence: "" });
|
|
134
|
+
}
|
|
135
|
+
} else if (e.to_text) {
|
|
136
|
+
if (!unresBySym.has(e.from)) unresBySym.set(e.from, new Set());
|
|
137
|
+
unresBySym.get(e.from).add(e.to_text);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
const isLeaf = (s) =>
|
|
141
|
+
(s.kind === "Function" || s.kind === "Test") &&
|
|
142
|
+
!(outCalls.get(s.anchor) > 0) && (inBySym.get(s.anchor)?.length ?? 0) >= 1;
|
|
143
|
+
// Bean-style accessors that also call nothing: pure noise in a flow view.
|
|
144
|
+
// Marked so renderers can hide them by default; the edge tables keep them.
|
|
145
|
+
const isAccessor = (s) => isLeaf(s) && /^(get|set|is)(?![a-z])/.test(s.name);
|
|
146
|
+
|
|
147
|
+
// ---- entry: called from outside its container, or an app entry (main) ----
|
|
148
|
+
const isEntry = (s) => {
|
|
149
|
+
if (s.entry) return true;
|
|
150
|
+
const doc = docOfAnchor.get(s.anchor);
|
|
151
|
+
return (inBySym.get(s.anchor) ?? []).some((r) => docOfAnchor.get(r.from) !== doc);
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
// A reference to `anchor` as seen from `fromDoc`.
|
|
155
|
+
const refTo = (anchor, fromDoc) => {
|
|
156
|
+
const doc = docOfAnchor.get(anchor);
|
|
157
|
+
const id = idOf.get(anchor);
|
|
158
|
+
return doc === fromDoc ? `#${id}` : `${posix.relative(posix.dirname(fromDoc), doc)}#${id}`;
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
// ---- write helper: deterministic, only-on-change ----
|
|
162
|
+
const stats = { docs: 0, written: 0, bytes: 0 };
|
|
163
|
+
const allDocs = [];
|
|
164
|
+
const writtenDocs = [];
|
|
165
|
+
const writeIfChanged = (relPath, content) => {
|
|
166
|
+
const p = join(outDir, relPath);
|
|
167
|
+
mkdirSync(dirname(p), { recursive: true });
|
|
168
|
+
stats.docs++;
|
|
169
|
+
stats.bytes += content.length;
|
|
170
|
+
if (relPath.endsWith(".geml")) allDocs.push(relPath);
|
|
171
|
+
if (existsSync(p) && readFileSync(p, "utf8") === content) return false;
|
|
172
|
+
writeFileSync(p, content);
|
|
173
|
+
stats.written++;
|
|
174
|
+
if (relPath.endsWith(".geml")) writtenDocs.push(relPath);
|
|
175
|
+
return true;
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
const RESOLUTION_DEFAULT = symbols.some((s) => s.resolution === "cpg") ? "cpg" : "heuristic";
|
|
179
|
+
const csv = (id, columns, rows, extraAttrs = "") => {
|
|
180
|
+
if (!rows.length) return null; // empty tables are not generated
|
|
181
|
+
// Column width by loop, not Math.max(...spread) — a spread call over a
|
|
182
|
+
// repo-scale table's rows blows the argument limit (same failure class
|
|
183
|
+
// as the build.mjs merge).
|
|
184
|
+
const widths = columns.map((c, i) => {
|
|
185
|
+
let w = c.length;
|
|
186
|
+
for (const r of rows) { const l = String(r[i] ?? "").length; if (l > w) w = l; }
|
|
187
|
+
return w;
|
|
188
|
+
});
|
|
189
|
+
const line = (cells) => cells.map((v, i) =>
|
|
190
|
+
i === cells.length - 1 ? String(v ?? "") : (String(v ?? "") + ",").padEnd(widths[i] + 2)).join("").replace(/\s+$/, "");
|
|
191
|
+
return `=== table {#${id} format=csv${extraAttrs}}\n${line(columns)}\n${rows.map(line).join("\n")}\n===\n`;
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
// ---- container documents ----
|
|
195
|
+
const indexRows = [];
|
|
196
|
+
const appEntries = [];
|
|
197
|
+
for (const [name, c] of [...containers.entries()].sort((a, b) => a[1].docName.localeCompare(b[1].docName))) {
|
|
198
|
+
const doc = c.docName;
|
|
199
|
+
c.methods.sort((a, b) => a.file.localeCompare(b.file) || (a.line_start ?? 0) - (b.line_start ?? 0) || a.anchor.localeCompare(b.anchor));
|
|
200
|
+
|
|
201
|
+
const entries = c.methods.filter(isEntry);
|
|
202
|
+
for (const s of c.methods) if (s.entry) appEntries.push(s.anchor);
|
|
203
|
+
const testCount = c.methods.filter((s) => isTestPath(s.file)).length;
|
|
204
|
+
// src= = the TRUE source directory (real path, for locating code); module=
|
|
205
|
+
// and the heading = the normalised DISPLAY path (ceremony stripped).
|
|
206
|
+
const srcDir = container === "file" ? name : name === "(root)" ? "" : `${name}/`;
|
|
207
|
+
const disp = displayOf(name);
|
|
208
|
+
const dispLabel = disp === "(root)" ? "root" : disp;
|
|
209
|
+
|
|
210
|
+
const chunks = [
|
|
211
|
+
"=== meta\n"
|
|
212
|
+
+ `module = ${csvCell(dispLabel)}\n`
|
|
213
|
+
+ (srcDir ? `src = ${csvCell(srcDir)}\n` : "")
|
|
214
|
+
+ (entries.length ? `entry = ${entries.map((s) => `#${idOf.get(s.anchor)}`).join(" ")}\n` : "")
|
|
215
|
+
+ `resolution-default = ${RESOLUTION_DEFAULT}\n===\n`,
|
|
216
|
+
`# ${esc(dispLabel)}\n`,
|
|
217
|
+
];
|
|
218
|
+
|
|
219
|
+
// method blocks, grouped under a `##` file heading when the container spans
|
|
220
|
+
// several files (containment = document structure)
|
|
221
|
+
const byFile = new Map();
|
|
222
|
+
for (const s of c.methods) {
|
|
223
|
+
if (!byFile.has(s.file)) byFile.set(s.file, []);
|
|
224
|
+
byFile.get(s.file).push(s);
|
|
225
|
+
}
|
|
226
|
+
const multiFile = byFile.size > 1;
|
|
227
|
+
const fileSymByPath = new Map(c.files.map((f) => [f.file, f]));
|
|
228
|
+
for (const [file, list] of [...byFile.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
229
|
+
if (multiFile) {
|
|
230
|
+
const fileSym = fileSymByPath.get(file);
|
|
231
|
+
const base = file.split("/").pop();
|
|
232
|
+
chunks.push(fileSym ? `## ${esc(base)} {#${idOf.get(fileSym.anchor)}}\n` : `## ${esc(base)}\n`);
|
|
233
|
+
}
|
|
234
|
+
for (const s of list) {
|
|
235
|
+
const cls = `${isTestPath(s.file) ? " .test" : ""}${isLeaf(s) ? " .leaf" : ""}${isAccessor(s) ? " .accessor" : ""}${s.flow_crit ? " .flow-entry" : ""}`;
|
|
236
|
+
const src = `${s.file}${s.line_start !== undefined ? `#L${s.line_start}-${s.line_end ?? s.line_start}` : ""}`;
|
|
237
|
+
// The display name rides along whenever id sanitisation changed it
|
|
238
|
+
// ("RenderCtx.block" -> id RenderCtx-block): renderers label nodes
|
|
239
|
+
// with the real name, ids stay reference-grammar clean.
|
|
240
|
+
const id = idOf.get(s.anchor);
|
|
241
|
+
const nameAttr = s.name !== id ? ` name="${attrVal(s.name)}"` : "";
|
|
242
|
+
chunks.push(`=== code {#${id}${cls}${nameAttr} src=${attrVal(src)} anchor="${attrVal(s.anchor)}"}\n===\n`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// #calls
|
|
247
|
+
const callRows = [];
|
|
248
|
+
for (const s of c.methods) {
|
|
249
|
+
for (const r of (outBySym.get(s.anchor) ?? [])
|
|
250
|
+
.sort((x, y) => (x.kind === y.kind ? refTo(x.to, doc).localeCompare(refTo(y.to, doc)) : 0))) {
|
|
251
|
+
callRows.push([`#${idOf.get(s.anchor)}`, refTo(r.to, doc), r.kind, r.confidence]);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
const callsTable = csv("calls", ["from", "to", "kind", "confidence"], callRows);
|
|
255
|
+
if (callsTable) chunks.push(callsTable);
|
|
256
|
+
|
|
257
|
+
// #called-by (aggregated in-edges)
|
|
258
|
+
const inRows = [];
|
|
259
|
+
for (const s of c.methods) {
|
|
260
|
+
const recs = (inBySym.get(s.anchor) ?? [])
|
|
261
|
+
.sort((x, y) => (x.site?.file ?? "").localeCompare(y.site?.file ?? "") || (x.site?.line ?? 0) - (y.site?.line ?? 0) || x.from.localeCompare(y.from));
|
|
262
|
+
for (const r of recs) {
|
|
263
|
+
const site = r.site ? `${r.site.file}:${r.site.line}` : "";
|
|
264
|
+
inRows.push([refTo(r.from, doc), `#${idOf.get(s.anchor)}`, r.kind, csvCell(site)]);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
const inTable = csv("called-by", ["from", "to", "kind", "site"], inRows);
|
|
268
|
+
if (inTable) chunks.push(inTable);
|
|
269
|
+
|
|
270
|
+
// #unresolved (hidden)
|
|
271
|
+
const unRows = [];
|
|
272
|
+
for (const s of c.methods) {
|
|
273
|
+
for (const t of [...(unresBySym.get(s.anchor) ?? [])].sort()) {
|
|
274
|
+
unRows.push([`#${idOf.get(s.anchor)}`, csvCell(t)]);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
const unTable = csv("unresolved", ["from", "to"], unRows, " hidden");
|
|
278
|
+
if (unTable) chunks.push(unTable);
|
|
279
|
+
|
|
280
|
+
writeIfChanged(doc, chunks.join("\n"));
|
|
281
|
+
indexRows.push({ module: dispLabel, doc, methods: c.methods.length, entries: entries.length, tests: testCount });
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// ---- index.geml ----
|
|
285
|
+
appEntries.sort((a, b) => docOfAnchor.get(a).localeCompare(docOfAnchor.get(b)) || a.localeCompare(b));
|
|
286
|
+
const moduleEdges = new Map(); // "fromDoc toDoc" -> count (cross-container resolved calls)
|
|
287
|
+
for (const [from, recs] of outBySym) {
|
|
288
|
+
const fd = docOfAnchor.get(from);
|
|
289
|
+
for (const r of recs) {
|
|
290
|
+
if (r.kind !== "call") continue;
|
|
291
|
+
const td = docOfAnchor.get(r.to);
|
|
292
|
+
if (fd === td) continue;
|
|
293
|
+
const key = `${fd} ${td}`;
|
|
294
|
+
moduleEdges.set(key, (moduleEdges.get(key) ?? 0) + 1);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
const modName = (doc) => indexRows.find((r) => r.doc === doc)?.module ?? doc;
|
|
298
|
+
const index = [
|
|
299
|
+
"=== meta\n"
|
|
300
|
+
+ `repo = ${csvCell(repoName)}\n`
|
|
301
|
+
+ (commit ? `commit = ${csvCell(commit)}\n` : "")
|
|
302
|
+
+ `container = ${container}\n`
|
|
303
|
+
+ (appEntries.length ? `entry = ${appEntries.map((a) => `${docOfAnchor.get(a)}#${idOf.get(a)}`).join(" ")}\n` : "")
|
|
304
|
+
+ `resolution-default = ${RESOLUTION_DEFAULT}\n===\n`,
|
|
305
|
+
`# Code map — ${esc(repoName)}\n`,
|
|
306
|
+
csv("modules", ["module", "doc", "methods", "entries", "tests"],
|
|
307
|
+
indexRows.sort((a, b) => b.methods - a.methods)
|
|
308
|
+
.map((r) => [csvCell(r.module), r.doc, r.methods, r.entries, r.tests])) ?? "",
|
|
309
|
+
csv("module-edges", ["from", "to", "calls"],
|
|
310
|
+
[...moduleEdges.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
311
|
+
.map(([k, n]) => { const [fd, td] = k.split(" "); return [csvCell(modName(fd)), csvCell(modName(td)), n]; })) ?? "",
|
|
312
|
+
].filter(Boolean).join("\n");
|
|
313
|
+
writeIfChanged("index.geml", index);
|
|
314
|
+
|
|
315
|
+
// ---- name-lookup ----
|
|
316
|
+
// Class-qualified names ("Cls.method") are ALSO findable by the bare member
|
|
317
|
+
// name — an agent asking "who is handleLogin" should not need to know the
|
|
318
|
+
// class first; ambiguity across classes is intrinsic and the lookup already
|
|
319
|
+
// answers with every candidate.
|
|
320
|
+
const lookup = new Map();
|
|
321
|
+
const addLookup = (name, s) => {
|
|
322
|
+
if (!lookup.has(name)) lookup.set(name, []);
|
|
323
|
+
lookup.get(name).push({ anchor: s.anchor, doc: docOfAnchor.get(s.anchor), id: idOf.get(s.anchor) });
|
|
324
|
+
};
|
|
325
|
+
for (const s of methods) {
|
|
326
|
+
addLookup(s.name, s);
|
|
327
|
+
const dot = s.name.indexOf(".");
|
|
328
|
+
if (dot > 0 && dot < s.name.length - 1) addLookup(s.name.slice(dot + 1), s);
|
|
329
|
+
}
|
|
330
|
+
const sortedLookup = {};
|
|
331
|
+
for (const name of [...lookup.keys()].sort()) {
|
|
332
|
+
sortedLookup[name] = lookup.get(name).sort((a, b) => a.anchor.localeCompare(b.anchor));
|
|
333
|
+
}
|
|
334
|
+
writeIfChanged("_index/name-lookup.json", JSON.stringify(sortedLookup, null, 2) + "\n");
|
|
335
|
+
|
|
336
|
+
// ---- edges-manifest (internal) ----
|
|
337
|
+
if (buildDir) {
|
|
338
|
+
const manifest = {};
|
|
339
|
+
for (const a of [...outBySym.keys()].sort()) {
|
|
340
|
+
manifest[a] = outBySym.get(a).map((r) => ({ kind: r.kind, to: r.to }))
|
|
341
|
+
.sort((x, y) => (x.kind + x.to).localeCompare(y.kind + y.to));
|
|
342
|
+
}
|
|
343
|
+
mkdirSync(buildDir, { recursive: true });
|
|
344
|
+
const p = join(buildDir, "edges-manifest.json");
|
|
345
|
+
const content = JSON.stringify(manifest, null, 1) + "\n";
|
|
346
|
+
if (!existsSync(p) || readFileSync(p, "utf8") !== content) writeFileSync(p, content);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
return {
|
|
350
|
+
...stats,
|
|
351
|
+
allDocs,
|
|
352
|
+
writtenDocs,
|
|
353
|
+
containers: containers.size,
|
|
354
|
+
symbols: symbols.length,
|
|
355
|
+
methods: methods.length,
|
|
356
|
+
edges: edges.length,
|
|
357
|
+
resolved: calls.filter((e) => e.to !== undefined && docOfAnchor.has(e.to)).length,
|
|
358
|
+
leaves: methods.filter((s) => isLeaf(s)).length,
|
|
359
|
+
entries: appEntries.length,
|
|
360
|
+
};
|
|
361
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Source exclusion for the codemap build.
|
|
2
|
+
//
|
|
3
|
+
// Two mechanisms, both matching on a symbol's repo-relative POSIX file path:
|
|
4
|
+
// 1. .gitignore — the default. Whatever git ignores (vendored copies, build
|
|
5
|
+
// output, dependency dumps) never enters the graph. Uses `git check-ignore`
|
|
6
|
+
// so the semantics are exactly git's, including un-committed .gitignore
|
|
7
|
+
// edits (check-ignore reads the working tree).
|
|
8
|
+
// 2. --exclude <glob> — explicit, repeatable, for paths git still tracks that
|
|
9
|
+
// you nonetheless don't want in the graph.
|
|
10
|
+
// Neither touches the raw indexer output; excluded symbols are dropped before
|
|
11
|
+
// emit, and the edge tables (which key on surviving anchors) follow.
|
|
12
|
+
|
|
13
|
+
import { execFileSync as _execFileSync } from "node:child_process";
|
|
14
|
+
|
|
15
|
+
// Minimal gitignore-flavoured glob: `**` spans path separators, `*` stays
|
|
16
|
+
// within a segment, everything else is literal. Anchored to the whole path.
|
|
17
|
+
export function globToRegExp(glob) {
|
|
18
|
+
let re = "";
|
|
19
|
+
for (let i = 0; i < glob.length; i++) {
|
|
20
|
+
const c = glob[i];
|
|
21
|
+
if (c === "*") {
|
|
22
|
+
if (glob[i + 1] === "*") { re += ".*"; i++; if (glob[i + 1] === "/") i++; }
|
|
23
|
+
else re += "[^/]*";
|
|
24
|
+
} else if ("\\^$+?.()|{}[]".includes(c)) {
|
|
25
|
+
re += "\\" + c;
|
|
26
|
+
} else {
|
|
27
|
+
re += c;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return new RegExp("^" + re + "$");
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Ask git which of `files` it ignores. Returns a Set of the ignored paths.
|
|
34
|
+
// check-ignore exits 1 when nothing matches and 128 when git is unavailable /
|
|
35
|
+
// the dir is not a repo — both mean "ignore nothing", not a build failure.
|
|
36
|
+
export function gitIgnored(root, files, exec = _execFileSync) {
|
|
37
|
+
if (!files.length) return new Set();
|
|
38
|
+
try {
|
|
39
|
+
const out = exec("git", ["-C", root, "check-ignore", "--stdin"], { input: files.join("\n"), encoding: "utf8" });
|
|
40
|
+
return new Set(out.split(/\r?\n/).filter(Boolean));
|
|
41
|
+
} catch (e) {
|
|
42
|
+
const out = e && e.stdout ? String(e.stdout) : "";
|
|
43
|
+
return new Set(out.split(/\r?\n/).filter(Boolean));
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Build a predicate (file) => shouldExclude.
|
|
48
|
+
export function makeExcluder({ root, globs = [], gitignore = true, files = [], exec } = {}) {
|
|
49
|
+
const res = globs.map(globToRegExp);
|
|
50
|
+
const ignored = gitignore ? gitIgnored(root, files, exec) : new Set();
|
|
51
|
+
return (file) => ignored.has(file) || res.some((r) => r.test(file));
|
|
52
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// geml-code-graph Joern export (P1, DESIGN §3.4).
|
|
2
|
+
// Runs INSIDE joern; emits raw method/call records as JSONL for adapters/joern.mjs.
|
|
3
|
+
//
|
|
4
|
+
// Parameters come from ENVIRONMENT VARIABLES, not --param: on Windows the
|
|
5
|
+
// joern.bat -> repl-bridge.bat hop re-tokenizes %* and cmd.exe treats `=` as a
|
|
6
|
+
// delimiter, so `--param k=v` never survives intact. Env vars pass through
|
|
7
|
+
// every layer on every OS:
|
|
8
|
+
//
|
|
9
|
+
// GEML_SRC=/abs/path/to/src GEML_OUT=/abs/path/to/build/raw \
|
|
10
|
+
// joern --script geml-parser/codemap/joern-export.sc
|
|
11
|
+
//
|
|
12
|
+
// Output:
|
|
13
|
+
// <GEML_OUT>/methods.jsonl one record per internal method
|
|
14
|
+
// <GEML_OUT>/calls.jsonl one record per call site, callees resolved by Joern
|
|
15
|
+
//
|
|
16
|
+
// Identity: methods are keyed by fullName|signature|filename — the adapter
|
|
17
|
+
// mints anchors and stable ids from these; this script stays dumb on purpose.
|
|
18
|
+
import java.io.{File, PrintWriter}
|
|
19
|
+
|
|
20
|
+
@main def exec(): Unit = {
|
|
21
|
+
val codeDir = sys.env.getOrElse("GEML_SRC", { System.err.println("GEML_SRC not set"); sys.exit(2) })
|
|
22
|
+
val outDir = sys.env.getOrElse("GEML_OUT", { System.err.println("GEML_OUT not set"); sys.exit(2) })
|
|
23
|
+
// GEML_LANG (optional): force a frontend in mixed-language repos, where
|
|
24
|
+
// auto-detection may pick the majority language instead of the intended one.
|
|
25
|
+
// Values are Joern's --language names: JAVASRC, NEWC, PYTHONSRC, JSSRC, …
|
|
26
|
+
sys.env.get("GEML_LANG") match {
|
|
27
|
+
case Some(lang) => importCode(inputPath = codeDir, projectName = "geml-code-graph", language = lang)
|
|
28
|
+
case None => importCode(inputPath = codeDir, projectName = "geml-code-graph")
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
def esc(s: String): String =
|
|
32
|
+
s.replace("\\", "\\\\").replace("\"", "\\\"")
|
|
33
|
+
.replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t")
|
|
34
|
+
def jstr(s: String): String = "\"" + esc(s) + "\""
|
|
35
|
+
|
|
36
|
+
new File(outDir).mkdirs()
|
|
37
|
+
|
|
38
|
+
val skipName = (n: String) => n == "<global>" || n.startsWith("<operator>") || n.startsWith("<clinit>")
|
|
39
|
+
|
|
40
|
+
// ---- methods ----
|
|
41
|
+
val mOut = new PrintWriter(new File(outDir, "methods.jsonl"), "UTF-8")
|
|
42
|
+
cpg.method.filter(m => !m.isExternal && !skipName(m.name)).foreach { m =>
|
|
43
|
+
mOut.println(
|
|
44
|
+
s"""{"name":${jstr(m.name)},"fullName":${jstr(m.fullName)},"signature":${jstr(m.signature)},""" +
|
|
45
|
+
s""""file":${jstr(m.filename)},"lineStart":${m.lineNumber.map(_.toString).getOrElse("null")},""" +
|
|
46
|
+
s""""lineEnd":${m.lineNumberEnd.map(_.toString).getOrElse("null")}}"""
|
|
47
|
+
)
|
|
48
|
+
}
|
|
49
|
+
mOut.close()
|
|
50
|
+
|
|
51
|
+
// ---- calls ----
|
|
52
|
+
// For each call site inside an internal method: Joern-resolved callees.
|
|
53
|
+
// Several internal callees = dispatch candidates (the adapter keeps them ALL,
|
|
54
|
+
// per the "never force a single candidate" red line). No internal callee =
|
|
55
|
+
// unresolved from the graph's point of view (external / pointer call).
|
|
56
|
+
// Operator calls are noise (arithmetic, casts, field access) EXCEPT
|
|
57
|
+
// <operator>.pointerCall — a function-pointer invocation is a real dispatch
|
|
58
|
+
// site the graph cannot resolve statically, so it must surface as an
|
|
59
|
+
// unresolved call (blind spots are shown, not hidden). Its readable label is
|
|
60
|
+
// the source expression itself.
|
|
61
|
+
val cOut = new PrintWriter(new File(outDir, "calls.jsonl"), "UTF-8")
|
|
62
|
+
cpg.call.filterNot(c => c.name.startsWith("<operator>") && c.name != "<operator>.pointerCall").foreach { c =>
|
|
63
|
+
val caller = c.method
|
|
64
|
+
if (!caller.isExternal && !skipName(caller.name)) {
|
|
65
|
+
val callees = c.callee.l
|
|
66
|
+
val internal = callees.filter(m => !m.isExternal && !skipName(m.name))
|
|
67
|
+
val tos = internal.map(m =>
|
|
68
|
+
s"""{"fullName":${jstr(m.fullName)},"signature":${jstr(m.signature)},"file":${jstr(m.filename)}}"""
|
|
69
|
+
).mkString("[", ",", "]")
|
|
70
|
+
val label =
|
|
71
|
+
if (c.name == "<operator>.pointerCall") c.code.takeWhile(_ != '\n').take(48)
|
|
72
|
+
else c.name
|
|
73
|
+
cOut.println(
|
|
74
|
+
s"""{"callerFullName":${jstr(caller.fullName)},"callerSignature":${jstr(caller.signature)},""" +
|
|
75
|
+
s""""callerFile":${jstr(caller.filename)},"name":${jstr(label)},""" +
|
|
76
|
+
s""""line":${c.lineNumber.map(_.toString).getOrElse("null")},"callees":$tos}"""
|
|
77
|
+
)
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
cOut.close()
|
|
81
|
+
|
|
82
|
+
println(s"geml-code-graph joern-export: done -> $outDir")
|
|
83
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// geml-code-graph MCP server — the thin consumption wrapper of DESIGN §8 (P2).
|
|
3
|
+
// Three navigation tools over a built graph/ directory, each "give an
|
|
4
|
+
// identifier, get readable text back" (the original proposal's 2.6):
|
|
5
|
+
// resolve_name name -> candidate anchors (doc + block id)
|
|
6
|
+
// open_symbol doc + id -> that symbol's block, verbatim
|
|
7
|
+
// get_backlinks doc + id -> the symbol's backlink block (who calls it)
|
|
8
|
+
//
|
|
9
|
+
// Zero dependencies: newline-delimited JSON-RPC 2.0 over stdio (the MCP stdio
|
|
10
|
+
// transport). Register e.g.:
|
|
11
|
+
// claude mcp add geml-code-graph -e GEML_GRAPH_DIR=/abs/path/to/graph \
|
|
12
|
+
// -- geml codemap mcp
|
|
13
|
+
// The graph dir comes from GEML_GRAPH_DIR or a per-call `graph_dir` argument.
|
|
14
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
15
|
+
import { join, resolve, dirname } from "node:path";
|
|
16
|
+
import { fileURLToPath } from "node:url";
|
|
17
|
+
import { createInterface } from "node:readline";
|
|
18
|
+
|
|
19
|
+
// blockSpans from the reference parser (its CLI entry is guarded, so importing
|
|
20
|
+
// is side-effect free). Falls back with a clear error if the parser isn't built.
|
|
21
|
+
const parserPath = resolve(dirname(fileURLToPath(import.meta.url)), "../dist/geml.js");
|
|
22
|
+
if (!existsSync(parserPath)) {
|
|
23
|
+
console.error("geml-code-graph mcp: build the parser first (cd geml-parser && npm install && npm run build)");
|
|
24
|
+
process.exit(1);
|
|
25
|
+
}
|
|
26
|
+
const { blockSpans } = await import(`file://${parserPath.replace(/\\/g, "/")}`);
|
|
27
|
+
const splitLines = (s) => s.split(/(?<=\n)/);
|
|
28
|
+
|
|
29
|
+
const graphDirOf = (args) => resolve(args?.graph_dir ?? process.env.GEML_GRAPH_DIR ?? ".geml-code-graph");
|
|
30
|
+
|
|
31
|
+
const readBlock = (graphDir, doc, id) => {
|
|
32
|
+
const p = join(graphDir, doc);
|
|
33
|
+
if (!existsSync(p)) throw new Error(`no such document: ${doc} (graph dir: ${graphDir})`);
|
|
34
|
+
const source = readFileSync(p, "utf8");
|
|
35
|
+
const span = blockSpans(source).get(id.replace(/^#/, ""));
|
|
36
|
+
if (!span) throw new Error(`no block with id \`${id}\` in ${doc}`);
|
|
37
|
+
return splitLines(source).slice(span.start, span.end).join("");
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const TOOLS = [
|
|
41
|
+
{
|
|
42
|
+
name: "resolve_name",
|
|
43
|
+
description: "Find a function/class by name in the code graph. Returns candidate anchors with the document and block id to open. Multiple candidates = real ambiguity (overloads/same name) — inspect each, never assume.",
|
|
44
|
+
inputSchema: {
|
|
45
|
+
type: "object",
|
|
46
|
+
properties: {
|
|
47
|
+
name: { type: "string", description: "Exact symbol name (function/class short name)" },
|
|
48
|
+
graph_dir: { type: "string", description: "Graph directory (default: $GEML_GRAPH_DIR or ./.geml-code-graph)" },
|
|
49
|
+
},
|
|
50
|
+
required: ["name"],
|
|
51
|
+
},
|
|
52
|
+
run: (args) => {
|
|
53
|
+
const lookupPath = join(graphDirOf(args), "_index/name-lookup.json");
|
|
54
|
+
if (!existsSync(lookupPath)) throw new Error(`no name-lookup at ${lookupPath} — build the graph first`);
|
|
55
|
+
const lookup = JSON.parse(readFileSync(lookupPath, "utf8"));
|
|
56
|
+
const hits = lookup[args.name];
|
|
57
|
+
if (!hits?.length) return `no symbol named \`${args.name}\` in the graph`;
|
|
58
|
+
return JSON.stringify(hits, null, 1);
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
name: "open_symbol",
|
|
63
|
+
description: "Open ONE symbol's block from the code graph (its callees as checked references, confidence annotations, called-by pointer). Equivalent to following a link. Get doc+id from resolve_name.",
|
|
64
|
+
inputSchema: {
|
|
65
|
+
type: "object",
|
|
66
|
+
properties: {
|
|
67
|
+
doc: { type: "string", description: "Document path relative to the codemap dir, e.g. hashtable.c.geml" },
|
|
68
|
+
id: { type: "string", description: "Block id, e.g. hashtableFind (or #calls / #called-by for the edge tables)" },
|
|
69
|
+
graph_dir: { type: "string", description: "Graph directory (default: $GEML_GRAPH_DIR or ./.geml-code-graph)" },
|
|
70
|
+
},
|
|
71
|
+
required: ["doc", "id"],
|
|
72
|
+
},
|
|
73
|
+
run: (args) => readBlock(graphDirOf(args), args.doc, args.id),
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
name: "get_backlinks",
|
|
77
|
+
description: "Who calls this symbol: opens its backlink block (callers with file:line sites, each a followable reference). Absence means no RESOLVED callers — never proof of none.",
|
|
78
|
+
inputSchema: {
|
|
79
|
+
type: "object",
|
|
80
|
+
properties: {
|
|
81
|
+
doc: { type: "string", description: "The symbol's document path, e.g. hashtable.c.geml" },
|
|
82
|
+
id: { type: "string", description: "The symbol's block id (e.g. hashtableFind); omit to get the whole #called-by table" },
|
|
83
|
+
graph_dir: { type: "string", description: "Codemap directory (default: $GEML_GRAPH_DIR or ./.geml-code-graph)" },
|
|
84
|
+
},
|
|
85
|
+
required: ["doc"],
|
|
86
|
+
},
|
|
87
|
+
run: (args) => {
|
|
88
|
+
// codemap profile: in-edges live in the SAME document's #called-by table.
|
|
89
|
+
let table;
|
|
90
|
+
try {
|
|
91
|
+
table = readBlock(graphDirOf(args), args.doc, "called-by");
|
|
92
|
+
} catch {
|
|
93
|
+
return `no #called-by table in ${args.doc} — no resolved callers recorded (under heuristic extraction this is a blind spot, not proof of none)`;
|
|
94
|
+
}
|
|
95
|
+
if (!args.id) return table;
|
|
96
|
+
const id = args.id.replace(/^#/, "");
|
|
97
|
+
const lines = table.split("\n");
|
|
98
|
+
const hits = lines.filter((l, i) => i < 2 || new RegExp(`,\\s*#${id}\\s*,`).test(l));
|
|
99
|
+
return hits.length > 2 ? hits.join("\n")
|
|
100
|
+
: `no resolved callers of #${id} in ${args.doc} (blind spots live in the #unresolved table)`;
|
|
101
|
+
},
|
|
102
|
+
},
|
|
103
|
+
];
|
|
104
|
+
|
|
105
|
+
// ---- newline-delimited JSON-RPC 2.0 over stdio ----
|
|
106
|
+
const reply = (id, result) => process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id, result }) + "\n");
|
|
107
|
+
const replyError = (id, code, message) =>
|
|
108
|
+
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id, error: { code, message } }) + "\n");
|
|
109
|
+
|
|
110
|
+
createInterface({ input: process.stdin }).on("line", (line) => {
|
|
111
|
+
line = line.trim();
|
|
112
|
+
if (!line) return;
|
|
113
|
+
let msg;
|
|
114
|
+
try { msg = JSON.parse(line); } catch { return; }
|
|
115
|
+
const { id, method, params } = msg;
|
|
116
|
+
try {
|
|
117
|
+
if (method === "initialize") {
|
|
118
|
+
reply(id, {
|
|
119
|
+
protocolVersion: params?.protocolVersion ?? "2024-11-05",
|
|
120
|
+
capabilities: { tools: {} },
|
|
121
|
+
serverInfo: { name: "geml-code-graph", version: "0.2.0" },
|
|
122
|
+
});
|
|
123
|
+
} else if (method === "notifications/initialized" || method?.startsWith("notifications/")) {
|
|
124
|
+
// notifications get no response
|
|
125
|
+
} else if (method === "ping") {
|
|
126
|
+
reply(id, {});
|
|
127
|
+
} else if (method === "tools/list") {
|
|
128
|
+
reply(id, { tools: TOOLS.map(({ name, description, inputSchema }) => ({ name, description, inputSchema })) });
|
|
129
|
+
} else if (method === "tools/call") {
|
|
130
|
+
const tool = TOOLS.find((t) => t.name === params?.name);
|
|
131
|
+
if (!tool) { replyError(id, -32602, `unknown tool: ${params?.name}`); return; }
|
|
132
|
+
try {
|
|
133
|
+
reply(id, { content: [{ type: "text", text: tool.run(params?.arguments ?? {}) }] });
|
|
134
|
+
} catch (e) {
|
|
135
|
+
reply(id, { content: [{ type: "text", text: `error: ${e.message}` }], isError: true });
|
|
136
|
+
}
|
|
137
|
+
} else if (id !== undefined) {
|
|
138
|
+
replyError(id, -32601, `method not found: ${method}`);
|
|
139
|
+
}
|
|
140
|
+
} catch (e) {
|
|
141
|
+
if (id !== undefined) replyError(id, -32603, String(e?.message ?? e));
|
|
142
|
+
}
|
|
143
|
+
});
|
|
Binary file
|