@geml/geml 1.0.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +109 -72
- package/codemap/adapters/crg.mjs +120 -0
- package/codemap/adapters/joern.mjs +131 -0
- package/codemap/adapters/scip.mjs +658 -0
- package/codemap/browser-stub.mjs +29 -0
- package/codemap/build.mjs +579 -0
- package/codemap/detect.mjs +399 -0
- package/codemap/emit.mjs +432 -0
- package/codemap/entries.mjs +129 -0
- package/codemap/exclude.mjs +52 -0
- package/codemap/find.mjs +63 -0
- package/codemap/foldings.mjs +110 -0
- package/codemap/joern-export.sc +83 -0
- package/codemap/mcp-server.mjs +172 -0
- package/codemap/normalize.mjs +272 -0
- package/codemap/recipe-trust.mjs +103 -0
- package/codemap/refresh.mjs +310 -0
- package/codemap/render-all.mjs +64 -0
- package/codemap/serve.mjs +578 -0
- package/codemap/sfc-virtualize.mjs +367 -0
- package/codemap/verify.mjs +143 -0
- package/dist/from-md.js +66 -7
- package/dist/geml.d.ts +7 -1
- package/dist/geml.js +700 -52
- package/dist/history.d.ts +30 -0
- package/dist/history.js +212 -32
- package/dist/inline.d.ts +2 -1
- package/dist/inline.js +61 -8
- package/dist/render-html.d.ts +3 -0
- package/dist/render-html.js +95 -0
- package/dist/render.d.ts +91 -2
- package/dist/render.js +1916 -50
- package/dist/serialize.js +22 -2
- package/dist/table.js +40 -5
- package/dist/to-md.js +5 -3
- package/package.json +63 -54
package/codemap/emit.mjs
ADDED
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
// geml-code-graph emit — exchange-format symbols/edges → the codemap document
|
|
2
|
+
// tree of docs/DESIGN-codemap-delta.md (§4–6) / docs/codemap-profile.md:
|
|
3
|
+
// one .geml per container (module|dir|file), each with EXACTLY ONE meta
|
|
4
|
+
// (module/src/entry/resolution-default), empty-body `code` blocks per method
|
|
5
|
+
// (src=path#Lx-y, anchor=; symbol-level classes .leaf/.test), and up to three
|
|
6
|
+
// CSV edge tables: #calls (out), #called-by (in, aggregated), #unresolved
|
|
7
|
+
// (blind spots, hidden). Plus index.geml (aggregates) and name-lookup.json.
|
|
8
|
+
//
|
|
9
|
+
// Generated documents are PURE DATA: no diagram blocks (a codemap-aware
|
|
10
|
+
// renderer offers the layered-flow view; embedding elsewhere uses
|
|
11
|
+
// `=== diagram {format=geml-code-graph src=…}`).
|
|
12
|
+
//
|
|
13
|
+
// Emission is deterministic: stable sort orders everywhere; a file is only
|
|
14
|
+
// written when its bytes changed (mtime = "what a change touched").
|
|
15
|
+
import { createHash } from "node:crypto";
|
|
16
|
+
import { mkdirSync, readFileSync, writeFileSync, existsSync } from "node:fs";
|
|
17
|
+
import { dirname, join, posix } from "node:path";
|
|
18
|
+
import { buildNormalizer } from "./normalize.mjs";
|
|
19
|
+
|
|
20
|
+
const esc = (s) => String(s).replace(/`/g, "'");
|
|
21
|
+
// Attribute values live on the block-header LINE: a newline inside one (e.g.
|
|
22
|
+
// scip's anonymous-type-literal descriptors embed the literal's multi-line
|
|
23
|
+
// text in the symbol) would truncate the header mid-value — the block still
|
|
24
|
+
// parses as raw text but its #id never registers, and every edge to it
|
|
25
|
+
// dangles (found on next.js: `recursiveCopy().({ filter... })`). Collapse all
|
|
26
|
+
// whitespace runs to single spaces alongside the quote swap.
|
|
27
|
+
const attrVal = (s) => String(s).replace(/"/g, "'").replace(/\s+/g, " ");
|
|
28
|
+
// Plain-text cells: no commas/newlines (CSV), and no square brackets — table
|
|
29
|
+
// cells are inline-parsed, so `f[i](&x)` would otherwise read as a LINK with an
|
|
30
|
+
// unresolvable target. Brackets become parens: still readable, never markup.
|
|
31
|
+
const csvCell = (s) => String(s).replace(/[,\r\n]/g, " ").replace(/\[/g, "(").replace(/\]/g, ")").trim();
|
|
32
|
+
const sha6 = (s, len = 6) => createHash("sha256").update(s, "utf8").digest("hex").slice(0, len);
|
|
33
|
+
|
|
34
|
+
// Test territory (path conventions; the avowed heuristic of GEP-0002).
|
|
35
|
+
const TEST_DIR = /(^|\/)(test|tests|testing|__tests__|spec|specs)(\/|$)/i;
|
|
36
|
+
const TEST_FILE = /(^test_|^tests?\.|[._-]tests?\.|\.test\.|\.spec\.)/i;
|
|
37
|
+
const isTestPath = (p) => {
|
|
38
|
+
p = String(p).replace(/\\/g, "/");
|
|
39
|
+
return TEST_DIR.test(p) || TEST_FILE.test(p.slice(p.lastIndexOf("/") + 1));
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const slugName = (name) => {
|
|
43
|
+
let s = String(name).replace(/[^A-Za-z0-9_-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").slice(0, 32);
|
|
44
|
+
if (!/^[A-Za-z]/.test(s)) s = "s" + s;
|
|
45
|
+
return s;
|
|
46
|
+
};
|
|
47
|
+
const slugPath = (p) => (p === "" || p === "(root)" ? "root" : p.replace(/\//g, "--").replace(/[^A-Za-z0-9_.-]/g, "-"));
|
|
48
|
+
const dirOf = (rel) => { const i = rel.lastIndexOf("/"); return i < 0 ? "(root)" : rel.slice(0, i); };
|
|
49
|
+
const topOf = (rel) => { const i = rel.indexOf("/"); return i < 0 ? "(root)" : rel.slice(0, i); };
|
|
50
|
+
|
|
51
|
+
export function emit({ symbols, edges, outDir, buildDir, repoName, container = "dir", commit, root, foldings, entryHints }) {
|
|
52
|
+
const byAnchor = new Map(symbols.map((s) => [s.anchor, s]));
|
|
53
|
+
const methods = symbols.filter((s) => s.kind === "Function" || s.kind === "Test");
|
|
54
|
+
const files = symbols.filter((s) => s.kind === "File");
|
|
55
|
+
|
|
56
|
+
// ---- app-entry hints (codemap/entries.mjs) ----
|
|
57
|
+
// A named hint marks its method; a file-level hint (SPA bootstrap, Nuxt app
|
|
58
|
+
// shell…) marks the file's only method, or — when the entry is top-level
|
|
59
|
+
// code with no function symbol at all — falls through to a doc-level
|
|
60
|
+
// `app-entry-file` note. The adapters' own name==="main" flag rides along;
|
|
61
|
+
// every marked entry carries `via` (the convention that identified it).
|
|
62
|
+
const fileHints = [];
|
|
63
|
+
{
|
|
64
|
+
const methodsByFile = new Map();
|
|
65
|
+
for (const s of methods) {
|
|
66
|
+
if (!methodsByFile.has(s.file)) methodsByFile.set(s.file, []);
|
|
67
|
+
methodsByFile.get(s.file).push(s);
|
|
68
|
+
}
|
|
69
|
+
for (const h of entryHints ?? []) {
|
|
70
|
+
const list = methodsByFile.get(h.file) ?? [];
|
|
71
|
+
let hit;
|
|
72
|
+
if (h.name) hit = list.find((s) => s.name === h.name || s.name.endsWith(`::${h.name}`) || s.name.endsWith(`.${h.name}`));
|
|
73
|
+
else if (list.length === 1) hit = list[0];
|
|
74
|
+
if (hit) { hit.entry = true; hit.entryVia ??= h.via; }
|
|
75
|
+
else fileHints.push(h);
|
|
76
|
+
}
|
|
77
|
+
for (const s of methods) if (s.entry && !s.entryVia) s.entryVia = "main";
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// ---- containers ----
|
|
81
|
+
const containerOf = (s) =>
|
|
82
|
+
container === "file" ? s.file : container === "module" ? topOf(s.file) : dirOf(s.file);
|
|
83
|
+
// Display-path normalisation: strip each module's shared ceremony prefix so
|
|
84
|
+
// `module=`/doc names read as the real structure. Applies in every container
|
|
85
|
+
// mode (dir and file) — a file-mode container path carries the same source
|
|
86
|
+
// roots (geml-parser/src/render.ts -> geml-parser/render.ts). Grouping still
|
|
87
|
+
// keys on the TRUE path (containerOf) and `src=` stays the true path — only
|
|
88
|
+
// the displayed module path shortens. root may be absent (older callers / crg
|
|
89
|
+
// tier): then displayOf is the identity.
|
|
90
|
+
const normMap = root ? buildNormalizer(root, methods.map(containerOf), { repoName, fileMode: container === "file", config: foldings }) : new Map();
|
|
91
|
+
const displayOf = (name) => normMap.get(name) ?? name;
|
|
92
|
+
const containers = new Map(); // name -> { docName, methods[], files[] }
|
|
93
|
+
const taken = new Set(["index.geml"]);
|
|
94
|
+
const containerFor = (name) => {
|
|
95
|
+
if (!containers.has(name)) {
|
|
96
|
+
let doc = `${slugPath(displayOf(name))}.geml`;
|
|
97
|
+
for (let i = 2; taken.has(doc); i++) doc = `${slugPath(displayOf(name))}-${i}.geml`;
|
|
98
|
+
taken.add(doc);
|
|
99
|
+
containers.set(name, { docName: doc, methods: [], files: [] });
|
|
100
|
+
}
|
|
101
|
+
return containers.get(name);
|
|
102
|
+
};
|
|
103
|
+
for (const s of methods) containerFor(containerOf(s)).methods.push(s);
|
|
104
|
+
for (const s of files) {
|
|
105
|
+
const c = containers.get(containerOf(s));
|
|
106
|
+
if (c) c.files.push(s); // only files that actually host methods
|
|
107
|
+
}
|
|
108
|
+
const docOfAnchor = new Map();
|
|
109
|
+
for (const [name, c] of containers) {
|
|
110
|
+
for (const s of [...c.methods, ...c.files]) docOfAnchor.set(s.anchor, c.docName);
|
|
111
|
+
}
|
|
112
|
+
// File-level app-entry hints, grouped under the container that holds the
|
|
113
|
+
// file (a hint whose file grew no container at all has nowhere honest to
|
|
114
|
+
// land and is dropped).
|
|
115
|
+
const fileHintsByDoc = new Map(); // docName -> [{file, via}]
|
|
116
|
+
for (const h of fileHints) {
|
|
117
|
+
const c = containers.get(containerOf({ file: h.file }));
|
|
118
|
+
if (!c) continue;
|
|
119
|
+
if (!fileHintsByDoc.has(c.docName)) fileHintsByDoc.set(c.docName, []);
|
|
120
|
+
fileHintsByDoc.get(c.docName).push(h);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// ---- block ids: short name when unique in its doc, else name-<sha6(anchor)> ----
|
|
124
|
+
const idOf = new Map(); // anchor -> id
|
|
125
|
+
for (const [, c] of containers) {
|
|
126
|
+
const byName = new Map();
|
|
127
|
+
for (const s of [...c.methods, ...c.files]) {
|
|
128
|
+
const base = slugName(s.name);
|
|
129
|
+
if (!byName.has(base)) byName.set(base, []);
|
|
130
|
+
byName.get(base).push(s);
|
|
131
|
+
}
|
|
132
|
+
for (const [base, list] of byName) {
|
|
133
|
+
if (list.length === 1) { idOf.set(list[0].anchor, base); continue; }
|
|
134
|
+
// Escalate hash length only when the default 6 hex chars actually collide
|
|
135
|
+
// within this name group — keeps ids short in the common case.
|
|
136
|
+
let len = 6;
|
|
137
|
+
let ids;
|
|
138
|
+
for (;;) {
|
|
139
|
+
ids = list.map((s) => `${base}-${sha6(s.anchor, len)}`);
|
|
140
|
+
if (new Set(ids).size === ids.length) break;
|
|
141
|
+
// sha256 hex is 64 chars — beyond that only IDENTICAL anchors can
|
|
142
|
+
// still collide, which is a caller bug (build.mjs dedupes anchors):
|
|
143
|
+
// fail loudly instead of escalating forever.
|
|
144
|
+
if (len >= 64) throw new Error(`emit: duplicate anchors in name group "${base}" — anchors must be unique`);
|
|
145
|
+
len += 2;
|
|
146
|
+
}
|
|
147
|
+
list.forEach((s, i) => idOf.set(s.anchor, ids[i]));
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// ---- edges (stage A: the `calls` relation only) ----
|
|
152
|
+
const calls = edges.filter((e) => e.kind === "calls");
|
|
153
|
+
const outCalls = new Map(); // anchor -> total outgoing (incl. unresolved) — leaf rule
|
|
154
|
+
const inBySym = new Map(); // target anchor -> [{fromAnchor, kind, site, confidence}]
|
|
155
|
+
const outBySym = new Map(); // source anchor -> [{toAnchor, kind, confidence}] resolved
|
|
156
|
+
const unresBySym = new Map(); // source anchor -> Set(to_text)
|
|
157
|
+
const addIn = (target, rec) => {
|
|
158
|
+
if (!byAnchor.has(target)) return;
|
|
159
|
+
if (!inBySym.has(target)) inBySym.set(target, []);
|
|
160
|
+
inBySym.get(target).push(rec);
|
|
161
|
+
};
|
|
162
|
+
for (const e of calls) {
|
|
163
|
+
if (!docOfAnchor.has(e.from)) continue;
|
|
164
|
+
outCalls.set(e.from, (outCalls.get(e.from) ?? 0) + 1);
|
|
165
|
+
if (e.to !== undefined && docOfAnchor.has(e.to)) {
|
|
166
|
+
if (!outBySym.has(e.from)) outBySym.set(e.from, []);
|
|
167
|
+
const conf = e.confidence === "high" || !e.confidence ? "" : e.confidence;
|
|
168
|
+
outBySym.get(e.from).push({ to: e.to, kind: "call", confidence: conf });
|
|
169
|
+
addIn(e.to, { from: e.from, kind: "call", site: e.site, confidence: conf });
|
|
170
|
+
for (const c of e.candidates ?? []) {
|
|
171
|
+
if (!docOfAnchor.has(c)) continue;
|
|
172
|
+
outBySym.get(e.from).push({ to: c, kind: "candidate", confidence: "" });
|
|
173
|
+
addIn(c, { from: e.from, kind: "candidate", site: e.site, confidence: "" });
|
|
174
|
+
}
|
|
175
|
+
} else if (e.to_text) {
|
|
176
|
+
if (!unresBySym.has(e.from)) unresBySym.set(e.from, new Set());
|
|
177
|
+
unresBySym.get(e.from).add(e.to_text);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
const isLeaf = (s) =>
|
|
181
|
+
(s.kind === "Function" || s.kind === "Test") &&
|
|
182
|
+
!(outCalls.get(s.anchor) > 0) && (inBySym.get(s.anchor)?.length ?? 0) >= 1;
|
|
183
|
+
// Bean-style accessors that also call nothing: pure noise in a flow view.
|
|
184
|
+
// Marked so renderers can hide them by default; the edge tables keep them.
|
|
185
|
+
const isAccessor = (s) => isLeaf(s) && /^(get|set|is)(?![a-z])/.test(s.name);
|
|
186
|
+
|
|
187
|
+
// ---- entry: called from outside its container, or an app entry (main) ----
|
|
188
|
+
const isEntry = (s) => {
|
|
189
|
+
if (s.entry) return true;
|
|
190
|
+
const doc = docOfAnchor.get(s.anchor);
|
|
191
|
+
return (inBySym.get(s.anchor) ?? []).some((r) => docOfAnchor.get(r.from) !== doc);
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
// A reference to `anchor` as seen from `fromDoc`.
|
|
195
|
+
const refTo = (anchor, fromDoc) => {
|
|
196
|
+
const doc = docOfAnchor.get(anchor);
|
|
197
|
+
const id = idOf.get(anchor);
|
|
198
|
+
return doc === fromDoc ? `#${id}` : `${posix.relative(posix.dirname(fromDoc), doc)}#${id}`;
|
|
199
|
+
};
|
|
200
|
+
|
|
201
|
+
// ---- write helper: deterministic, only-on-change ----
|
|
202
|
+
const stats = { docs: 0, written: 0, bytes: 0 };
|
|
203
|
+
const allDocs = [];
|
|
204
|
+
const writtenDocs = [];
|
|
205
|
+
const writeIfChanged = (relPath, content) => {
|
|
206
|
+
const p = join(outDir, relPath);
|
|
207
|
+
mkdirSync(dirname(p), { recursive: true });
|
|
208
|
+
stats.docs++;
|
|
209
|
+
stats.bytes += content.length;
|
|
210
|
+
if (relPath.endsWith(".geml")) allDocs.push(relPath);
|
|
211
|
+
if (existsSync(p) && readFileSync(p, "utf8") === content) return false;
|
|
212
|
+
writeFileSync(p, content);
|
|
213
|
+
stats.written++;
|
|
214
|
+
if (relPath.endsWith(".geml")) writtenDocs.push(relPath);
|
|
215
|
+
return true;
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
const RESOLUTION_DEFAULT = symbols.some((s) => s.resolution === "cpg") ? "cpg" : "heuristic";
|
|
219
|
+
const csv = (id, columns, rows, extraAttrs = "") => {
|
|
220
|
+
if (!rows.length) return null; // empty tables are not generated
|
|
221
|
+
// Column width by loop, not Math.max(...spread) — a spread call over a
|
|
222
|
+
// repo-scale table's rows blows the argument limit (same failure class
|
|
223
|
+
// as the build.mjs merge).
|
|
224
|
+
const widths = columns.map((c, i) => {
|
|
225
|
+
let w = c.length;
|
|
226
|
+
for (const r of rows) { const l = String(r[i] ?? "").length; if (l > w) w = l; }
|
|
227
|
+
return w;
|
|
228
|
+
});
|
|
229
|
+
const line = (cells) => cells.map((v, i) =>
|
|
230
|
+
i === cells.length - 1 ? String(v ?? "") : (String(v ?? "") + ",").padEnd(widths[i] + 2)).join("").replace(/\s+$/, "");
|
|
231
|
+
return `=== table {#${id} format=csv${extraAttrs}}\n${line(columns)}\n${rows.map(line).join("\n")}\n===\n`;
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
// ---- container documents ----
|
|
235
|
+
const indexRows = [];
|
|
236
|
+
const appEntries = [];
|
|
237
|
+
for (const [name, c] of [...containers.entries()].sort((a, b) => a[1].docName.localeCompare(b[1].docName))) {
|
|
238
|
+
const doc = c.docName;
|
|
239
|
+
c.methods.sort((a, b) => a.file.localeCompare(b.file) || (a.line_start ?? 0) - (b.line_start ?? 0) || a.anchor.localeCompare(b.anchor));
|
|
240
|
+
|
|
241
|
+
const entries = c.methods.filter(isEntry);
|
|
242
|
+
for (const s of c.methods) if (s.entry) appEntries.push(s.anchor);
|
|
243
|
+
const testCount = c.methods.filter((s) => isTestPath(s.file)).length;
|
|
244
|
+
// src= = the TRUE source directory (real path, for locating code); module=
|
|
245
|
+
// and the heading = the normalised DISPLAY path (ceremony stripped).
|
|
246
|
+
const srcDir = container === "file" ? name : name === "(root)" ? "" : `${name}/`;
|
|
247
|
+
const disp = displayOf(name);
|
|
248
|
+
const dispLabel = disp === "(root)" ? "root" : disp;
|
|
249
|
+
|
|
250
|
+
const chunks = [
|
|
251
|
+
"=== meta\n"
|
|
252
|
+
+ `module = ${csvCell(dispLabel)}\n`
|
|
253
|
+
+ (srcDir ? `src = ${csvCell(srcDir)}\n` : "")
|
|
254
|
+
+ (entries.length ? `entry = ${entries.map((s) => `#${idOf.get(s.anchor)}`).join(" ")}\n` : "")
|
|
255
|
+
// app-entry: WHERE the program starts (main, a mount, a worker handler)
|
|
256
|
+
// — a separate, much rarer list than entry= (the container's inbound
|
|
257
|
+
// call surface). File-level entries (top-level bootstrap code with no
|
|
258
|
+
// function symbol) are named by path instead of a block reference.
|
|
259
|
+
+ (c.methods.some((s) => s.entry)
|
|
260
|
+
? `app-entry = ${c.methods.filter((s) => s.entry).map((s) => `#${idOf.get(s.anchor)} (${s.entryVia})`).join(" ")}\n` : "")
|
|
261
|
+
+ (fileHintsByDoc.get(doc) ?? []).map((h) => `app-entry-file = ${csvCell(h.file)} (${h.via})\n`).join("")
|
|
262
|
+
+ `resolution-default = ${RESOLUTION_DEFAULT}\n===\n`,
|
|
263
|
+
`# ${esc(dispLabel)}\n`,
|
|
264
|
+
];
|
|
265
|
+
|
|
266
|
+
// method blocks, grouped under a `##` file heading when the container spans
|
|
267
|
+
// several files (containment = document structure)
|
|
268
|
+
const byFile = new Map();
|
|
269
|
+
for (const s of c.methods) {
|
|
270
|
+
if (!byFile.has(s.file)) byFile.set(s.file, []);
|
|
271
|
+
byFile.get(s.file).push(s);
|
|
272
|
+
}
|
|
273
|
+
const multiFile = byFile.size > 1;
|
|
274
|
+
const fileSymByPath = new Map(c.files.map((f) => [f.file, f]));
|
|
275
|
+
for (const [file, list] of [...byFile.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
276
|
+
if (multiFile) {
|
|
277
|
+
const fileSym = fileSymByPath.get(file);
|
|
278
|
+
const base = file.split("/").pop();
|
|
279
|
+
chunks.push(fileSym ? `## ${esc(base)} {#${idOf.get(fileSym.anchor)}}\n` : `## ${esc(base)}\n`);
|
|
280
|
+
}
|
|
281
|
+
for (const s of list) {
|
|
282
|
+
const cls = `${isTestPath(s.file) ? " .test" : ""}${isLeaf(s) ? " .leaf" : ""}${isAccessor(s) ? " .accessor" : ""}${s.flow_crit ? " .flow-entry" : ""}${s.entry ? " .app-entry" : ""}`;
|
|
283
|
+
const src = `${s.file}${s.line_start !== undefined ? `#L${s.line_start}-${s.line_end ?? s.line_start}` : ""}`;
|
|
284
|
+
// The display name rides along whenever id sanitisation changed it
|
|
285
|
+
// ("RenderCtx.block" -> id RenderCtx-block): renderers label nodes
|
|
286
|
+
// with the real name, ids stay reference-grammar clean.
|
|
287
|
+
const id = idOf.get(s.anchor);
|
|
288
|
+
const nameAttr = s.name !== id ? ` name="${attrVal(s.name)}"` : "";
|
|
289
|
+
const viaAttr = s.entry && s.entryVia ? ` entry-via="${attrVal(s.entryVia)}"` : "";
|
|
290
|
+
chunks.push(`=== code {#${id}${cls}${nameAttr}${viaAttr} src=${attrVal(src)} anchor="${attrVal(s.anchor)}"}\n===\n`);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// #calls
|
|
295
|
+
const callRows = [];
|
|
296
|
+
for (const s of c.methods) {
|
|
297
|
+
for (const r of (outBySym.get(s.anchor) ?? [])
|
|
298
|
+
.sort((x, y) => (x.kind === y.kind ? refTo(x.to, doc).localeCompare(refTo(y.to, doc)) : 0))) {
|
|
299
|
+
callRows.push([`#${idOf.get(s.anchor)}`, refTo(r.to, doc), r.kind, r.confidence]);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
const callsTable = csv("calls", ["from", "to", "kind", "confidence"], callRows);
|
|
303
|
+
if (callsTable) chunks.push(callsTable);
|
|
304
|
+
|
|
305
|
+
// #called-by (aggregated in-edges)
|
|
306
|
+
const inRows = [];
|
|
307
|
+
for (const s of c.methods) {
|
|
308
|
+
const recs = (inBySym.get(s.anchor) ?? [])
|
|
309
|
+
.sort((x, y) => (x.site?.file ?? "").localeCompare(y.site?.file ?? "") || (x.site?.line ?? 0) - (y.site?.line ?? 0) || x.from.localeCompare(y.from));
|
|
310
|
+
for (const r of recs) {
|
|
311
|
+
const site = r.site ? `${r.site.file}:${r.site.line}` : "";
|
|
312
|
+
inRows.push([refTo(r.from, doc), `#${idOf.get(s.anchor)}`, r.kind, csvCell(site)]);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
const inTable = csv("called-by", ["from", "to", "kind", "site"], inRows);
|
|
316
|
+
if (inTable) chunks.push(inTable);
|
|
317
|
+
|
|
318
|
+
// #unresolved (hidden)
|
|
319
|
+
const unRows = [];
|
|
320
|
+
for (const s of c.methods) {
|
|
321
|
+
for (const t of [...(unresBySym.get(s.anchor) ?? [])].sort()) {
|
|
322
|
+
unRows.push([`#${idOf.get(s.anchor)}`, csvCell(t)]);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
const unTable = csv("unresolved", ["from", "to"], unRows, " hidden");
|
|
326
|
+
if (unTable) chunks.push(unTable);
|
|
327
|
+
|
|
328
|
+
writeIfChanged(doc, chunks.join("\n"));
|
|
329
|
+
indexRows.push({ module: dispLabel, doc, methods: c.methods.length, entries: entries.length, tests: testCount });
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// ---- index.geml ----
|
|
333
|
+
appEntries.sort((a, b) => docOfAnchor.get(a).localeCompare(docOfAnchor.get(b)) || a.localeCompare(b));
|
|
334
|
+
const moduleEdges = new Map(); // "fromDoc toDoc" -> count (cross-container resolved calls)
|
|
335
|
+
for (const [from, recs] of outBySym) {
|
|
336
|
+
const fd = docOfAnchor.get(from);
|
|
337
|
+
for (const r of recs) {
|
|
338
|
+
if (r.kind !== "call") continue;
|
|
339
|
+
const td = docOfAnchor.get(r.to);
|
|
340
|
+
if (fd === td) continue;
|
|
341
|
+
const key = `${fd} ${td}`;
|
|
342
|
+
moduleEdges.set(key, (moduleEdges.get(key) ?? 0) + 1);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
const modName = (doc) => indexRows.find((r) => r.doc === doc)?.module ?? doc;
|
|
346
|
+
const index = [
|
|
347
|
+
"=== meta\n"
|
|
348
|
+
+ `repo = ${csvCell(repoName)}\n`
|
|
349
|
+
+ (commit ? `commit = ${csvCell(commit)}\n` : "")
|
|
350
|
+
+ `container = ${container}\n`
|
|
351
|
+
+ (appEntries.length ? `entry = ${appEntries.map((a) => `${docOfAnchor.get(a)}#${idOf.get(a)}`).join(" ")}\n` : "")
|
|
352
|
+
// Documents whose app entry is FILE-level (top-level bootstrap code, no
|
|
353
|
+
// function symbol) — its own key so entry= keeps its doc#id grammar.
|
|
354
|
+
+ (fileHintsByDoc.size ? `app-entry-docs = ${[...fileHintsByDoc.keys()].sort().join(" ")}\n` : "")
|
|
355
|
+
+ `resolution-default = ${RESOLUTION_DEFAULT}\n===\n`,
|
|
356
|
+
`# Code map — ${esc(repoName)}\n`,
|
|
357
|
+
csv("modules", ["module", "doc", "methods", "entries", "tests"],
|
|
358
|
+
indexRows.sort((a, b) => b.methods - a.methods)
|
|
359
|
+
.map((r) => [csvCell(r.module), r.doc, r.methods, r.entries, r.tests])) ?? "",
|
|
360
|
+
csv("module-edges", ["from", "to", "calls"],
|
|
361
|
+
[...moduleEdges.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
362
|
+
.map(([k, n]) => { const [fd, td] = k.split(" "); return [csvCell(modName(fd)), csvCell(modName(td)), n]; })) ?? "",
|
|
363
|
+
].filter(Boolean).join("\n");
|
|
364
|
+
writeIfChanged("index.geml", index);
|
|
365
|
+
|
|
366
|
+
// ---- name-lookup ----
|
|
367
|
+
// Class-qualified names ("Cls.method") are ALSO findable by the bare member
|
|
368
|
+
// name — an agent asking "who is handleLogin" should not need to know the
|
|
369
|
+
// class first; ambiguity across classes is intrinsic and the lookup already
|
|
370
|
+
// answers with every candidate.
|
|
371
|
+
const lookup = new Map();
|
|
372
|
+
const addLookup = (name, s) => {
|
|
373
|
+
if (!lookup.has(name)) lookup.set(name, []);
|
|
374
|
+
lookup.get(name).push({ anchor: s.anchor, doc: docOfAnchor.get(s.anchor), id: idOf.get(s.anchor) });
|
|
375
|
+
};
|
|
376
|
+
for (const s of methods) {
|
|
377
|
+
addLookup(s.name, s);
|
|
378
|
+
const dot = s.name.indexOf(".");
|
|
379
|
+
if (dot > 0 && dot < s.name.length - 1) addLookup(s.name.slice(dot + 1), s);
|
|
380
|
+
else {
|
|
381
|
+
// Rust members qualify with "::" (Widget::area) — same bare-name alias.
|
|
382
|
+
const c = s.name.indexOf("::");
|
|
383
|
+
if (c > 0 && c < s.name.length - 2) addLookup(s.name.slice(c + 2), s);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
const sortedLookup = {};
|
|
387
|
+
for (const name of [...lookup.keys()].sort()) {
|
|
388
|
+
sortedLookup[name] = lookup.get(name).sort((a, b) => a.anchor.localeCompare(b.anchor));
|
|
389
|
+
}
|
|
390
|
+
writeIfChanged("_index/name-lookup.json", JSON.stringify(sortedLookup, null, 2) + "\n");
|
|
391
|
+
|
|
392
|
+
// Compact search index for the viewer's name -> node typeahead: a flat
|
|
393
|
+
// [name, doc, id] list with the (bulky) anchors dropped, so it stays small
|
|
394
|
+
// even on huge repos. Emitted as a JS global so a STATIC page can load it via
|
|
395
|
+
// `<script src>` (a file:// page can't fetch() a sibling file, but a script
|
|
396
|
+
// tag is exempt); serve also searches it server-side for big graphs.
|
|
397
|
+
const searchIndex = [];
|
|
398
|
+
for (const name of Object.keys(sortedLookup)) {
|
|
399
|
+
for (const c of sortedLookup[name]) searchIndex.push([name, c.doc, c.id]);
|
|
400
|
+
}
|
|
401
|
+
writeIfChanged("_index/search-index.js", "window.__gemlSearch=" + JSON.stringify(searchIndex) + ";\n");
|
|
402
|
+
|
|
403
|
+
// ---- edges-manifest (internal) ----
|
|
404
|
+
if (buildDir) {
|
|
405
|
+
const manifest = {};
|
|
406
|
+
for (const a of [...outBySym.keys()].sort()) {
|
|
407
|
+
manifest[a] = outBySym.get(a).map((r) => ({ kind: r.kind, to: r.to }))
|
|
408
|
+
.sort((x, y) => (x.kind + x.to).localeCompare(y.kind + y.to));
|
|
409
|
+
}
|
|
410
|
+
mkdirSync(buildDir, { recursive: true });
|
|
411
|
+
const p = join(buildDir, "edges-manifest.json");
|
|
412
|
+
const content = JSON.stringify(manifest, null, 1) + "\n";
|
|
413
|
+
if (!existsSync(p) || readFileSync(p, "utf8") !== content) writeFileSync(p, content);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
return {
|
|
417
|
+
...stats,
|
|
418
|
+
allDocs,
|
|
419
|
+
writtenDocs,
|
|
420
|
+
containers: containers.size,
|
|
421
|
+
symbols: symbols.length,
|
|
422
|
+
methods: methods.length,
|
|
423
|
+
edges: edges.length,
|
|
424
|
+
// A `calls` edge is "resolved" only when it actually yields a table row —
|
|
425
|
+
// that needs BOTH endpoints known (the calls loop above skips any edge
|
|
426
|
+
// whose FROM anchor is unknown). Counting by target alone inflated the
|
|
427
|
+
// figure for a dangling-FROM edge that produced no row.
|
|
428
|
+
resolved: calls.filter((e) => docOfAnchor.has(e.from) && e.to !== undefined && docOfAnchor.has(e.to)).length,
|
|
429
|
+
leaves: methods.filter((s) => isLeaf(s)).length,
|
|
430
|
+
entries: appEntries.length + [...fileHintsByDoc.values()].reduce((n, a) => n + a.length, 0),
|
|
431
|
+
};
|
|
432
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
// geml-code-graph app-entry detection — WHERE does this repo start running?
|
|
2
|
+
//
|
|
3
|
+
// Emits entry HINTS ({ file, via, name? }) from three signal tiers, each
|
|
4
|
+
// carrying an honest `via` label (the codemap never claims an entry without
|
|
5
|
+
// saying what convention identified it):
|
|
6
|
+
// L2 manifest/layout Cargo [[bin]] & src/main.rs & src/bin/*, package.json
|
|
7
|
+
// bin, wrangler.toml main, Nuxt app.vue, Next root
|
|
8
|
+
// page, SvelteKit root route, Django manage.py,
|
|
9
|
+
// python __main__.py
|
|
10
|
+
// L3 source markers workers-rs #[event(...)], createApp().mount() /
|
|
11
|
+
// createRoot() / svelte mount (SPA bootstraps),
|
|
12
|
+
// .listen() (node servers), export default { fetch }
|
|
13
|
+
// (JS workers), Flask()/FastAPI() apps,
|
|
14
|
+
// @SpringBootApplication
|
|
15
|
+
// (L1 — a function literally named `main` — is already flagged by the scip
|
|
16
|
+
// and joern adapters at extraction time; hints here ADD to it.)
|
|
17
|
+
//
|
|
18
|
+
// Pure by design: given precomputed { files, manifests, pkgs } lists it walks
|
|
19
|
+
// nothing; `readText`/`readJson` are injectable, and every source peek is
|
|
20
|
+
// bounded to a handful of conventional entry files per project — never a
|
|
21
|
+
// repo-wide grep. A hint is only emitted for files the build actually indexes
|
|
22
|
+
// (present in `files`), so a pkg-bin pointing at dist/ never leaks in.
|
|
23
|
+
import { readFileSync } from "node:fs";
|
|
24
|
+
import { join } from "node:path";
|
|
25
|
+
|
|
26
|
+
const dirOf = (p) => (p.includes("/") ? p.slice(0, p.lastIndexOf("/")) : "");
|
|
27
|
+
|
|
28
|
+
export function detectEntries(root, { files = [], manifests = [], pkgs = [], readText, readJson } = {}) {
|
|
29
|
+
readText ??= (p) => readFileSync(p, "utf8");
|
|
30
|
+
readJson ??= (p) => JSON.parse(readText(p));
|
|
31
|
+
const fileSet = new Set(files);
|
|
32
|
+
const hints = [];
|
|
33
|
+
const seen = new Set();
|
|
34
|
+
const add = (file, via, name) => {
|
|
35
|
+
if (!file || !fileSet.has(file)) return;
|
|
36
|
+
const k = `${file}${via}${name ?? ""}`;
|
|
37
|
+
if (seen.has(k)) return;
|
|
38
|
+
seen.add(k);
|
|
39
|
+
hints.push(name ? { file, via, name } : { file, via });
|
|
40
|
+
};
|
|
41
|
+
const tryText = (rel) => {
|
|
42
|
+
try { return readText(join(root, ...rel.split("/"))); } catch { return null; }
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
// ---- Rust: cargo bin targets + workers-rs event handlers ----
|
|
46
|
+
for (const m of manifests.filter((x) => x.endsWith("Cargo.toml"))) {
|
|
47
|
+
const dir = dirOf(m);
|
|
48
|
+
const at = (rel) => (dir ? `${dir}/${rel}` : rel);
|
|
49
|
+
add(at("src/main.rs"), "cargo-bin", "main");
|
|
50
|
+
for (const f of files) if (f.startsWith(at("src/bin/")) && f.endsWith(".rs")) add(f, "cargo-bin", "main");
|
|
51
|
+
const toml = tryText(m);
|
|
52
|
+
if (toml) {
|
|
53
|
+
for (const b of toml.matchAll(/^\[\[bin\]\][^[]*/gm)) {
|
|
54
|
+
const p = /path\s*=\s*"([^"]+)"/.exec(b[0]);
|
|
55
|
+
if (p) add(at(p[1].replace(/\\/g, "/")), "cargo-bin", "main");
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
for (const rel of ["src/main.rs", "src/lib.rs"]) {
|
|
59
|
+
const t = fileSet.has(at(rel)) ? tryText(at(rel)) : null;
|
|
60
|
+
if (!t) continue;
|
|
61
|
+
for (const ev of t.matchAll(/#\[event\((\w+)[^)]*\)\]\s*(?:pub\s+)?(?:async\s+)?fn\s+([A-Za-z_][A-Za-z0-9_]*)/g)) {
|
|
62
|
+
add(at(rel), `worker-${ev[1]}`, ev[2]);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// ---- Node/TS/frontends: one look per package ----
|
|
68
|
+
for (const p of pkgs) {
|
|
69
|
+
const dir = dirOf(p);
|
|
70
|
+
const at = (rel) => (dir ? `${dir}/${rel}` : rel);
|
|
71
|
+
let pkg = {};
|
|
72
|
+
try { pkg = readJson(join(root, ...p.split("/"))) ?? {}; } catch { /* unreadable manifest */ }
|
|
73
|
+
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
74
|
+
const norm = (v) => (typeof v === "string" ? v.replace(/^\.\//, "").replace(/\\/g, "/") : null);
|
|
75
|
+
const bins = typeof pkg.bin === "string" ? [pkg.bin] : Object.values(pkg.bin ?? {});
|
|
76
|
+
for (const b of bins) { const f = norm(b); if (f) add(at(f), "pkg-bin"); }
|
|
77
|
+
const wrangler = tryText(at("wrangler.toml"));
|
|
78
|
+
if (wrangler) {
|
|
79
|
+
const mm = /^\s*main\s*=\s*"([^"]+)"/m.exec(wrangler);
|
|
80
|
+
if (mm) add(at(norm(mm[1])), "worker-fetch");
|
|
81
|
+
}
|
|
82
|
+
// Nuxt: the app shell is the entry; individual pages are routes, not
|
|
83
|
+
// program starts — deliberately NOT flooded into app-entries.
|
|
84
|
+
if (deps.nuxt || fileSet.has(at("nuxt.config.ts")) || fileSet.has(at("nuxt.config.js"))) {
|
|
85
|
+
if (fileSet.has(at("app.vue"))) add(at("app.vue"), "nuxt-app");
|
|
86
|
+
else add(at("pages/index.vue"), "nuxt-page");
|
|
87
|
+
}
|
|
88
|
+
if (deps.next) {
|
|
89
|
+
for (const rel of ["app/page.tsx", "app/page.jsx", "src/app/page.tsx", "pages/index.tsx", "pages/index.jsx", "src/pages/index.tsx"]) {
|
|
90
|
+
if (fileSet.has(at(rel))) { add(at(rel), "next-page"); break; }
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (deps["@sveltejs/kit"]) add(at("src/routes/+page.svelte"), "kit-route");
|
|
94
|
+
// SPA bootstrap / server start markers — conventional entry files only.
|
|
95
|
+
for (const rel of ["src/main.ts", "src/main.tsx", "src/main.js", "src/main.jsx",
|
|
96
|
+
"src/index.ts", "src/index.tsx", "src/index.js", "index.ts", "index.js",
|
|
97
|
+
"src/server.ts", "src/server.js", "server.js", "src/app.ts", "app.js"]) {
|
|
98
|
+
const f = at(rel);
|
|
99
|
+
if (!fileSet.has(f)) continue;
|
|
100
|
+
const t = tryText(f);
|
|
101
|
+
if (!t) continue;
|
|
102
|
+
if (/createApp\s*\(/.test(t) && /\.mount\s*\(/.test(t)) add(f, "vue-mount");
|
|
103
|
+
else if (/createRoot\s*\(|ReactDOM\.render\s*\(/.test(t)) add(f, "react-mount");
|
|
104
|
+
else if (deps.svelte && /\bnew\s+\w+\s*\(\s*\{[^}]*target|\bmount\s*\(/.test(t)) add(f, "svelte-mount");
|
|
105
|
+
if (/\.listen\s*\(/.test(t)) add(f, "server-listen");
|
|
106
|
+
if (/export\s+default\s*\{[^}]*\bfetch\b/s.test(t)) add(f, "worker-fetch");
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// ---- Python ----
|
|
111
|
+
for (const f of files) {
|
|
112
|
+
if (/(^|\/)manage\.py$/.test(f)) add(f, "django-manage");
|
|
113
|
+
else if (/(^|\/)__main__\.py$/.test(f)) add(f, "py-main");
|
|
114
|
+
else if (/(^|\/)(app|main|wsgi|asgi)\.py$/.test(f)) {
|
|
115
|
+
const t = tryText(f);
|
|
116
|
+
if (t && /\bFlask\s*\(|\bFastAPI\s*\(/.test(t)) add(f, "wsgi-app");
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// ---- Java: Spring Boot (convention-named files only, never a repo grep) ----
|
|
121
|
+
for (const f of files) {
|
|
122
|
+
if (/Application\.java$/.test(f)) {
|
|
123
|
+
const t = tryText(f);
|
|
124
|
+
if (t && /@SpringBootApplication/.test(t)) add(f, "spring-boot", "main");
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
return hints;
|
|
129
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Source exclusion for the codemap build.
|
|
2
|
+
//
|
|
3
|
+
// Two mechanisms, both matching on a symbol's repo-relative POSIX file path:
|
|
4
|
+
// 1. .gitignore — the default. Whatever git ignores (vendored copies, build
|
|
5
|
+
// output, dependency dumps) never enters the graph. Uses `git check-ignore`
|
|
6
|
+
// so the semantics are exactly git's, including un-committed .gitignore
|
|
7
|
+
// edits (check-ignore reads the working tree).
|
|
8
|
+
// 2. --exclude <glob> — explicit, repeatable, for paths git still tracks that
|
|
9
|
+
// you nonetheless don't want in the graph.
|
|
10
|
+
// Neither touches the raw indexer output; excluded symbols are dropped before
|
|
11
|
+
// emit, and the edge tables (which key on surviving anchors) follow.
|
|
12
|
+
|
|
13
|
+
import { execFileSync as _execFileSync } from "node:child_process";
|
|
14
|
+
|
|
15
|
+
// Minimal gitignore-flavoured glob: `**` spans path separators, `*` stays
|
|
16
|
+
// within a segment, everything else is literal. Anchored to the whole path.
|
|
17
|
+
export function globToRegExp(glob) {
|
|
18
|
+
let re = "";
|
|
19
|
+
for (let i = 0; i < glob.length; i++) {
|
|
20
|
+
const c = glob[i];
|
|
21
|
+
if (c === "*") {
|
|
22
|
+
if (glob[i + 1] === "*") { re += ".*"; i++; if (glob[i + 1] === "/") i++; }
|
|
23
|
+
else re += "[^/]*";
|
|
24
|
+
} else if ("\\^$+?.()|{}[]".includes(c)) {
|
|
25
|
+
re += "\\" + c;
|
|
26
|
+
} else {
|
|
27
|
+
re += c;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return new RegExp("^" + re + "$");
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Ask git which of `files` it ignores. Returns a Set of the ignored paths.
|
|
34
|
+
// check-ignore exits 1 when nothing matches and 128 when git is unavailable /
|
|
35
|
+
// the dir is not a repo — both mean "ignore nothing", not a build failure.
|
|
36
|
+
export function gitIgnored(root, files, exec = _execFileSync) {
|
|
37
|
+
if (!files.length) return new Set();
|
|
38
|
+
try {
|
|
39
|
+
const out = exec("git", ["-C", root, "check-ignore", "--stdin"], { input: files.join("\n"), encoding: "utf8" });
|
|
40
|
+
return new Set(out.split(/\r?\n/).filter(Boolean));
|
|
41
|
+
} catch (e) {
|
|
42
|
+
const out = e && e.stdout ? String(e.stdout) : "";
|
|
43
|
+
return new Set(out.split(/\r?\n/).filter(Boolean));
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Build a predicate (file) => shouldExclude.
|
|
48
|
+
export function makeExcluder({ root, globs = [], gitignore = true, files = [], exec } = {}) {
|
|
49
|
+
const res = globs.map(globToRegExp);
|
|
50
|
+
const ignored = gitignore ? gitIgnored(root, files, exec) : new Set();
|
|
51
|
+
return (file) => ignored.has(file) || res.some((r) => r.test(file));
|
|
52
|
+
}
|