@geml/geml 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +79 -72
- package/codemap/adapters/crg.mjs +109 -0
- package/codemap/adapters/joern.mjs +131 -0
- package/codemap/adapters/scip.mjs +229 -0
- package/codemap/browser-stub.mjs +24 -0
- package/codemap/build.mjs +354 -0
- package/codemap/detect.mjs +185 -0
- package/codemap/emit.mjs +361 -0
- package/codemap/exclude.mjs +52 -0
- package/codemap/joern-export.sc +83 -0
- package/codemap/mcp-server.mjs +143 -0
- package/codemap/normalize.mjs +0 -0
- package/codemap/refresh.mjs +160 -0
- package/codemap/render-all.mjs +64 -0
- package/codemap/serve.mjs +378 -0
- package/codemap/verify.mjs +126 -0
- package/dist/geml.d.ts +5 -0
- package/dist/geml.js +390 -14
- package/dist/history.d.ts +30 -0
- package/dist/history.js +153 -13
- package/dist/render.d.ts +59 -0
- package/dist/render.js +1685 -11
- package/dist/to-md.js +1 -3
- package/package.json +9 -5
package/LICENSE
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2026 GEML contributors
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 GEML contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,72 +1,79 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
geml
|
|
43
|
-
geml --
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<picture>
|
|
3
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/geml-spec/geml/main/docs/assets/logo/geml-logo-dark.svg">
|
|
4
|
+
<img src="https://raw.githubusercontent.com/geml-spec/geml/main/docs/assets/logo/geml-logo-light.svg" alt="GEML" width="300">
|
|
5
|
+
</picture>
|
|
6
|
+
</p>
|
|
7
|
+
|
|
8
|
+
# @geml/geml
|
|
9
|
+
|
|
10
|
+
Reference parser, validator, renderer, and CLI for **GEML** — the General
|
|
11
|
+
Expressive Markup Language: a plain-text document format that stays legible to
|
|
12
|
+
people and reliable for machines. Every kind of structured content — code,
|
|
13
|
+
tables, diagrams, math, callouts, metadata — is carried on **one** primitive,
|
|
14
|
+
the typed block:
|
|
15
|
+
|
|
16
|
+
```
|
|
17
|
+
=== code {#hello lang=python}
|
|
18
|
+
print("hi")
|
|
19
|
+
===
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
References are checked at build time (a dangling `#id` is an error, not a silent
|
|
23
|
+
dead link), and the parser emits a document-model JSON with `diagnostics`, so
|
|
24
|
+
agents and CI get a structured pass/fail signal.
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
npm install -g @geml/geml # global CLI — installs the `geml` command
|
|
30
|
+
# or, per project:
|
|
31
|
+
npm install @geml/geml # library + local bin
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Requires Node ≥ 18.
|
|
35
|
+
|
|
36
|
+
## CLI
|
|
37
|
+
|
|
38
|
+
Every command reads a file path, or `-` for stdin. Exit codes: `0` ok ·
|
|
39
|
+
`1` document/operation error · `2` usage error.
|
|
40
|
+
|
|
41
|
+
```sh
|
|
42
|
+
geml check file.geml # validate only: diagnostics + exit code
|
|
43
|
+
geml check --json file.geml # machine-readable: diagnostics array (or {"error":…} on IO failure)
|
|
44
|
+
geml file.geml # full document-model JSON
|
|
45
|
+
geml render file.geml -o out.html # one self-contained, interactive HTML file
|
|
46
|
+
geml export file.geml -o out.md # project to GitHub-Flavored Markdown (lossy; notes on stderr)
|
|
47
|
+
geml convert in.md -o out.geml # Markdown -> GEML
|
|
48
|
+
geml fmt file.geml # canonical re-format (idempotent)
|
|
49
|
+
geml history <commit|verify|show|restore> file.geml [...] # .gemlhistory sidecar
|
|
50
|
+
geml --help | --version # --version --json prints {"parser","spec"}
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The agent loop: write `.geml` → `geml check` → fix on non-zero → done.
|
|
54
|
+
|
|
55
|
+
## Library
|
|
56
|
+
|
|
57
|
+
```js
|
|
58
|
+
import { parse, serialize, renderHtml, gemlToMd, mdToGeml } from "@geml/geml";
|
|
59
|
+
|
|
60
|
+
const doc = parse(src); // { kind:"document", children, ids, diagnostics }
|
|
61
|
+
const ok = !doc.diagnostics.some(d => d.severity === "error");
|
|
62
|
+
const html = renderHtml(doc); // one self-contained HTML string
|
|
63
|
+
const md = gemlToMd(doc).md; // GitHub-Flavored Markdown (lossy)
|
|
64
|
+
const geml = mdToGeml(markdown).geml; // the inverse
|
|
65
|
+
const canonical = serialize(doc); // GEML text; parse(serialize(parse(x))) is stable
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`parse(src, { resolveDoc })` enables cross-document reference checking — pass a
|
|
69
|
+
function that returns another file's source by path (or `null`).
|
|
70
|
+
|
|
71
|
+
## Documentation
|
|
72
|
+
|
|
73
|
+
Full normative spec, history-sidecar spec, and format comparison live in the
|
|
74
|
+
[repository](https://github.com/geml-spec/geml). The spec is itself
|
|
75
|
+
written in GEML (`GEML-spec.geml`) and parsed clean on every test run.
|
|
76
|
+
|
|
77
|
+
## License
|
|
78
|
+
|
|
79
|
+
MIT.
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
// geml-code-graph adapter: code-review-graph SQLite (`graph.db`, tree-sitter based) →
|
|
2
|
+
// the exchange format of docs/DESIGN-geml-code-graph.md §3 (symbols + edges).
|
|
3
|
+
//
|
|
4
|
+
// Everything this adapter emits is syntax-level extraction, so per §3.3 every
|
|
5
|
+
// symbol and edge carries resolution:"heuristic"; resolved targets get
|
|
6
|
+
// confidence:"medium", unresolved ones become `to_text` rows (confidence:"low").
|
|
7
|
+
// CONTAINS is not an edge — containment is document structure (GEP-0002).
|
|
8
|
+
//
|
|
9
|
+
// Beyond the core schema, symbols may carry optional navigation flags the
|
|
10
|
+
// engine knows about: `entry` (a `main` function) and `flow_crit` (max
|
|
11
|
+
// criticality of execution flows entered at this symbol, when >= 0.6).
|
|
12
|
+
import { DatabaseSync } from "node:sqlite";
|
|
13
|
+
|
|
14
|
+
const EDGE_KIND = {
|
|
15
|
+
CALLS: "calls",
|
|
16
|
+
IMPORTS_FROM: "imports",
|
|
17
|
+
INHERITS: "inherits",
|
|
18
|
+
TESTED_BY: "tested-by",
|
|
19
|
+
REFERENCES: "references",
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export function extract({ db: dbPath, root }) {
|
|
23
|
+
const db = new DatabaseSync(dbPath);
|
|
24
|
+
const rootFs = root.replace(/\\/g, "/").replace(/\/?$/, "/");
|
|
25
|
+
const rel = (p) => {
|
|
26
|
+
p = String(p).replace(/\\/g, "/");
|
|
27
|
+
return p.startsWith(rootFs) ? p.slice(rootFs.length) : p;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
const rows = db.prepare(
|
|
31
|
+
"SELECT id, kind, name, qualified_name, file_path, line_start, line_end, language, is_test FROM nodes",
|
|
32
|
+
).all();
|
|
33
|
+
|
|
34
|
+
// anchor = "<lang>:<relfile>#<name>", File symbols just "<lang>:<relfile>".
|
|
35
|
+
// Same-file same-name collisions get ~2, ~3 … ordered by line_start so the
|
|
36
|
+
// numbering is stable across rebuilds (§4.2 / risk 3).
|
|
37
|
+
const byKey = new Map();
|
|
38
|
+
for (const r of rows) {
|
|
39
|
+
const key = `${r.language ?? "unknown"}:${rel(r.file_path)}#${r.name}`;
|
|
40
|
+
if (!byKey.has(key)) byKey.set(key, []);
|
|
41
|
+
byKey.get(key).push(r);
|
|
42
|
+
}
|
|
43
|
+
const anchorOf = new Map(); // node rowid -> anchor
|
|
44
|
+
for (const [key, list] of byKey) {
|
|
45
|
+
list.sort((a, b) => (a.line_start ?? 0) - (b.line_start ?? 0) || a.id - b.id);
|
|
46
|
+
list.forEach((r, i) => {
|
|
47
|
+
const base = r.kind === "File" ? key.slice(0, key.lastIndexOf("#")) : key;
|
|
48
|
+
anchorOf.set(r.id, i === 0 ? base : `${base}~${i + 1}`);
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Navigation flags from engine-specific tables.
|
|
53
|
+
const mains = new Set(
|
|
54
|
+
db.prepare("SELECT id FROM nodes WHERE kind='Function' AND name='main'").all().map((r) => r.id),
|
|
55
|
+
);
|
|
56
|
+
const flowCrit = new Map();
|
|
57
|
+
try {
|
|
58
|
+
for (const r of db.prepare(
|
|
59
|
+
"SELECT entry_point_id id, max(criticality) c FROM flows GROUP BY 1 HAVING c >= 0.6",
|
|
60
|
+
).all()) flowCrit.set(r.id, r.c);
|
|
61
|
+
} catch { /* no flows table */ }
|
|
62
|
+
|
|
63
|
+
const symbols = rows.map((r) => {
|
|
64
|
+
const s = {
|
|
65
|
+
anchor: anchorOf.get(r.id),
|
|
66
|
+
lang: r.language ?? "unknown",
|
|
67
|
+
kind: r.kind,
|
|
68
|
+
name: r.name,
|
|
69
|
+
file: rel(r.file_path),
|
|
70
|
+
line_start: r.line_start ?? undefined,
|
|
71
|
+
line_end: r.line_end ?? undefined,
|
|
72
|
+
is_test: r.is_test ? true : undefined,
|
|
73
|
+
entry: mains.has(r.id) ? true : undefined,
|
|
74
|
+
flow_crit: flowCrit.get(r.id),
|
|
75
|
+
resolution: "heuristic",
|
|
76
|
+
};
|
|
77
|
+
return s;
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
const idByQual = new Map(rows.map((r) => [r.qualified_name, r.id]));
|
|
81
|
+
const edges = [];
|
|
82
|
+
for (const e of db.prepare(
|
|
83
|
+
"SELECT kind, source_qualified, target_qualified, file_path, line FROM edges",
|
|
84
|
+
).all()) {
|
|
85
|
+
const kind = EDGE_KIND[e.kind];
|
|
86
|
+
if (!kind) continue; // CONTAINS and anything unknown
|
|
87
|
+
const fromId = idByQual.get(e.source_qualified);
|
|
88
|
+
if (fromId === undefined) continue; // dangling source: nothing to attach to
|
|
89
|
+
const toId = idByQual.get(e.target_qualified);
|
|
90
|
+
const edge = {
|
|
91
|
+
kind,
|
|
92
|
+
from: anchorOf.get(fromId),
|
|
93
|
+
resolution: "heuristic",
|
|
94
|
+
site: { file: rel(e.file_path), line: e.line ?? 0 },
|
|
95
|
+
};
|
|
96
|
+
if (toId !== undefined) {
|
|
97
|
+
edge.to = anchorOf.get(toId);
|
|
98
|
+
edge.confidence = "medium";
|
|
99
|
+
} else {
|
|
100
|
+
// Keep only a readable short name for the unresolved target (§5.2's
|
|
101
|
+
// calls-unresolved line) — qualified names here are often paths.
|
|
102
|
+
edge.to_text = String(e.target_qualified).replace(/\\/g, "/").split("/").pop();
|
|
103
|
+
edge.confidence = "low";
|
|
104
|
+
}
|
|
105
|
+
edges.push(edge);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return { symbols, edges };
|
|
109
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// geml-code-graph adapter: Joern CPG export → exchange format (DESIGN §3.2/3.4).
|
|
2
|
+
//
|
|
3
|
+
// Consumes the raw JSONL written by joern-export.sc (methods.jsonl +
|
|
4
|
+
// calls.jsonl) — so the node side never touches the JVM. Everything here is
|
|
5
|
+
// resolution:"cpg". Confidence mapping (§3.4):
|
|
6
|
+
// exactly one internal callee → high
|
|
7
|
+
// several internal callees (dispatch) → first as `to` + the rest as
|
|
8
|
+
// `candidates`, medium
|
|
9
|
+
// no internal callee (external/pointer) → to_text, low (unresolved)
|
|
10
|
+
import { readFileSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
|
|
13
|
+
const LANG_BY_EXT = {
|
|
14
|
+
c: "c", h: "c", cc: "cpp", cpp: "cpp", cxx: "cpp", hpp: "cpp", hh: "cpp",
|
|
15
|
+
java: "java", js: "javascript", mjs: "javascript", ts: "typescript",
|
|
16
|
+
py: "python", kt: "kotlin", go: "go", rb: "ruby", swift: "swift",
|
|
17
|
+
cs: "csharp", php: "php",
|
|
18
|
+
};
|
|
19
|
+
const langOf = (file) => LANG_BY_EXT[file.split(".").pop()?.toLowerCase()] ?? "unknown";
|
|
20
|
+
|
|
21
|
+
const readJsonl = (p) => readFileSync(p, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l));
|
|
22
|
+
|
|
23
|
+
export function extract({ raw, root }) {
|
|
24
|
+
const rootFs = root ? root.replace(/\\/g, "/").replace(/\/?$/, "/") : "";
|
|
25
|
+
const rel = (p) => {
|
|
26
|
+
p = String(p).replace(/\\/g, "/");
|
|
27
|
+
return rootFs && p.startsWith(rootFs) ? p.slice(rootFs.length) : p.replace(/^\//, "");
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
const methods = readJsonl(join(raw, "methods.jsonl"));
|
|
31
|
+
const calls = readJsonl(join(raw, "calls.jsonl"));
|
|
32
|
+
|
|
33
|
+
// Identity: fullName|signature|file — the same tuple joern-export.sc keys on.
|
|
34
|
+
const keyOf = (m) => `${m.fullName}|${m.signature}|${rel(m.file)}`;
|
|
35
|
+
|
|
36
|
+
// Some frontends (javasrc2cpg: constructors, overload/bridge pairs) emit
|
|
37
|
+
// several method records with an IDENTICAL fullName|signature|file — one
|
|
38
|
+
// logical method must become one symbol, so dedupe by key first, keeping the
|
|
39
|
+
// record with the widest line span (the real body over a synthetic stub).
|
|
40
|
+
const byKey = new Map();
|
|
41
|
+
for (const m of methods) {
|
|
42
|
+
const k = keyOf(m);
|
|
43
|
+
const prev = byKey.get(k);
|
|
44
|
+
if (!prev || (m.lineEnd ?? 0) - (m.lineStart ?? 0) > (prev.lineEnd ?? 0) - (prev.lineStart ?? 0)) {
|
|
45
|
+
byKey.set(k, m);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const uniqueMethods = [...byKey.values()];
|
|
49
|
+
|
|
50
|
+
// Anchors: "<lang>:<relfile>#<name>(<sig>)"; same-file same-name-and-sig
|
|
51
|
+
// duplicates that remain distinct (different fullName) get ~2, ~3 by line order.
|
|
52
|
+
const byAnchorBase = new Map();
|
|
53
|
+
for (const m of uniqueMethods) {
|
|
54
|
+
const base = `${langOf(m.file)}:${rel(m.file)}#${m.name}(${m.signature})`;
|
|
55
|
+
if (!byAnchorBase.has(base)) byAnchorBase.set(base, []);
|
|
56
|
+
byAnchorBase.get(base).push(m);
|
|
57
|
+
}
|
|
58
|
+
const anchorByKey = new Map();
|
|
59
|
+
for (const [base, list] of byAnchorBase) {
|
|
60
|
+
list.sort((a, b) => (a.lineStart ?? 0) - (b.lineStart ?? 0));
|
|
61
|
+
list.forEach((m, i) => anchorByKey.set(keyOf(m), i === 0 ? base : `${base}~${i + 1}`));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Class-qualified display names: in a class language a method's natural
|
|
65
|
+
// short name is `Cls.method` — bare names ("invoke", "execute") repeat in
|
|
66
|
+
// every other class and read as noise. Constructors are `Cls.new`, static
|
|
67
|
+
// initialisers `Cls.static{}`. C (no classes: fullName has no dot) keeps
|
|
68
|
+
// plain names. The anchor keeps the raw identity — presentation only.
|
|
69
|
+
const displayName = (m) => {
|
|
70
|
+
const head = String(m.fullName ?? "").split(":")[0]; // pkg.Outer$Inner.method
|
|
71
|
+
const di = head.lastIndexOf(".");
|
|
72
|
+
const owner = di > 0 ? head.slice(0, di) : "";
|
|
73
|
+
const simple = (owner.split(".").pop() || "").split("$").pop() || "";
|
|
74
|
+
if (m.name === "<init>") return simple ? `${simple}.new` : "new";
|
|
75
|
+
if (m.name === "<clinit>") return simple ? `${simple}.static{}` : "static{}";
|
|
76
|
+
return simple ? `${simple}.${m.name}` : m.name;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const symbols = [];
|
|
80
|
+
const seenFiles = new Set();
|
|
81
|
+
for (const m of uniqueMethods) {
|
|
82
|
+
const file = rel(m.file);
|
|
83
|
+
symbols.push({
|
|
84
|
+
anchor: anchorByKey.get(keyOf(m)),
|
|
85
|
+
lang: langOf(file),
|
|
86
|
+
kind: "Function",
|
|
87
|
+
name: displayName(m),
|
|
88
|
+
file,
|
|
89
|
+
line_start: m.lineStart ?? undefined,
|
|
90
|
+
line_end: m.lineEnd ?? undefined,
|
|
91
|
+
signature: m.signature || undefined,
|
|
92
|
+
entry: m.name === "main" ? true : undefined,
|
|
93
|
+
resolution: "cpg",
|
|
94
|
+
});
|
|
95
|
+
// Derive one File symbol per source file so emit gets stable heading ids.
|
|
96
|
+
if (!seenFiles.has(file)) {
|
|
97
|
+
seenFiles.add(file);
|
|
98
|
+
symbols.push({
|
|
99
|
+
anchor: `${langOf(file)}:${file}`,
|
|
100
|
+
lang: langOf(file),
|
|
101
|
+
kind: "File",
|
|
102
|
+
name: file.split("/").pop(),
|
|
103
|
+
file,
|
|
104
|
+
resolution: "cpg",
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const edges = [];
|
|
110
|
+
for (const c of calls) {
|
|
111
|
+
const from = anchorByKey.get(`${c.callerFullName}|${c.callerSignature}|${rel(c.callerFile)}`);
|
|
112
|
+
if (!from) continue;
|
|
113
|
+
const site = { file: rel(c.callerFile), line: c.line ?? 0 };
|
|
114
|
+
const targets = (c.callees ?? [])
|
|
115
|
+
.map((t) => anchorByKey.get(`${t.fullName}|${t.signature}|${rel(t.file)}`))
|
|
116
|
+
.filter(Boolean);
|
|
117
|
+
if (targets.length === 0) {
|
|
118
|
+
edges.push({ kind: "calls", from, to_text: c.name, resolution: "cpg", confidence: "low", site });
|
|
119
|
+
} else if (targets.length === 1) {
|
|
120
|
+
edges.push({ kind: "calls", from, to: targets[0], resolution: "cpg", confidence: "high", site });
|
|
121
|
+
} else {
|
|
122
|
+
const [first, ...rest] = [...new Set(targets)].sort();
|
|
123
|
+
edges.push({
|
|
124
|
+
kind: "calls", from, to: first, resolution: "cpg", confidence: "medium",
|
|
125
|
+
note: `dispatch, ${targets.length} candidates`, candidates: rest, site,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return { symbols, edges };
|
|
131
|
+
}
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
// geml-code-graph adapter: SCIP index (index.scip, protobuf) → exchange format.
|
|
2
|
+
//
|
|
3
|
+
// Reads the protobuf DIRECTLY with a minimal embedded wire-format reader — the
|
|
4
|
+
// scip CLI ships no Windows binary, and the fields we need are few. Everything
|
|
5
|
+
// here is compiler-grade resolution, so edges are resolution:"cpg"; a direct
|
|
6
|
+
// hit is confidence:"high"; a call to an interface/abstract member with known
|
|
7
|
+
// implementations becomes medium + candidates; references to symbols not
|
|
8
|
+
// defined in the project become to_text (unresolved, low).
|
|
9
|
+
//
|
|
10
|
+
// Produce the index with scip-typescript (TS/JS):
|
|
11
|
+
// npx --yes @sourcegraph/scip-typescript index --output index.scip
|
|
12
|
+
//
|
|
13
|
+
// Caller attribution: a reference occurrence belongs to the innermost function
|
|
14
|
+
// DEFINITION whose enclosing_range contains it. scip-typescript emits
|
|
15
|
+
// enclosing_range on definition occurrences; if absent we fall back to "the
|
|
16
|
+
// nearest preceding definition in the file" and mark the adapter degraded.
|
|
17
|
+
import { readFileSync } from "node:fs";
|
|
18
|
+
import { resolve as resolvePath } from "node:path";
|
|
19
|
+
|
|
20
|
+
// ---- minimal protobuf wire reader ------------------------------------------
|
|
21
|
+
function varint(buf, p) {
|
|
22
|
+
let x = 0n, s = 0n, b;
|
|
23
|
+
do { b = buf[p.i++]; x |= BigInt(b & 0x7f) << s; s += 7n; } while (b & 0x80);
|
|
24
|
+
return x;
|
|
25
|
+
}
|
|
26
|
+
// Iterate fields of one message region [start, end): yields {no, wt, val|sub}.
|
|
27
|
+
function* fields(buf, start, end) {
|
|
28
|
+
const p = { i: start };
|
|
29
|
+
while (p.i < end) {
|
|
30
|
+
const key = Number(varint(buf, p));
|
|
31
|
+
const no = key >> 3, wt = key & 7;
|
|
32
|
+
if (wt === 0) yield { no, wt, val: varint(buf, p) };
|
|
33
|
+
else if (wt === 1) { yield { no, wt, val: buf.readBigUInt64LE(p.i) }; p.i += 8; }
|
|
34
|
+
else if (wt === 2) { const len = Number(varint(buf, p)); yield { no, wt, a: p.i, b: p.i + len }; p.i += len; }
|
|
35
|
+
else if (wt === 5) { yield { no, wt, val: BigInt(buf.readUInt32LE(p.i)) }; p.i += 4; }
|
|
36
|
+
else throw new Error(`scip: unsupported wire type ${wt}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const str = (buf, f) => buf.toString("utf8", f.a, f.b);
|
|
40
|
+
// repeated int32, packed (len-delimited varints) or single varint value
|
|
41
|
+
function packedInts(buf, f, out) {
|
|
42
|
+
if (f.wt === 0) { out.push(Number(f.val)); return; }
|
|
43
|
+
const p = { i: f.a };
|
|
44
|
+
while (p.i < f.b) out.push(Number(varint(buf, p)));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// ---- SCIP field numbers (scip.proto) ---------------------------------------
|
|
48
|
+
// Index: metadata=1, documents=2, external_symbols=3
|
|
49
|
+
// Document: relative_path=1, occurrences=2, symbols=3, language=4
|
|
50
|
+
// Occurrence: range=1, symbol=2, symbol_roles=3, enclosing_range=7
|
|
51
|
+
// SymbolInformation: symbol=1, relationships=4, display_name=6
|
|
52
|
+
// Relationship: symbol=1, is_implementation=3
|
|
53
|
+
const ROLE_DEFINITION = 0x1;
|
|
54
|
+
|
|
55
|
+
function parseScip(path) {
|
|
56
|
+
const buf = readFileSync(path);
|
|
57
|
+
const docs = [];
|
|
58
|
+
let projectRoot = "";
|
|
59
|
+
for (const f of fields(buf, 0, buf.length)) {
|
|
60
|
+
if (f.no === 1 && f.wt === 2) {
|
|
61
|
+
// Metadata → project_root (field 3): the directory the indexer ran in.
|
|
62
|
+
// Needed to re-anchor document paths when a SUBPROJECT of the repo was
|
|
63
|
+
// indexed (scip paths are relative to the indexed project, not the repo).
|
|
64
|
+
for (const m of fields(buf, f.a, f.b)) {
|
|
65
|
+
if (m.no === 3 && m.wt === 2) projectRoot = str(buf, m);
|
|
66
|
+
}
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (f.no !== 2 || f.wt !== 2) continue;
|
|
70
|
+
const doc = { path: "", occ: [], rel: [] };
|
|
71
|
+
for (const d of fields(buf, f.a, f.b)) {
|
|
72
|
+
if (d.no === 1 && d.wt === 2) doc.path = str(buf, d);
|
|
73
|
+
else if (d.no === 2 && d.wt === 2) {
|
|
74
|
+
const o = { range: [], symbol: "", roles: 0, enclosing: [] };
|
|
75
|
+
for (const x of fields(buf, d.a, d.b)) {
|
|
76
|
+
if (x.no === 1) packedInts(buf, x, o.range);
|
|
77
|
+
else if (x.no === 2 && x.wt === 2) o.symbol = str(buf, x);
|
|
78
|
+
else if (x.no === 3 && x.wt === 0) o.roles = Number(x.val);
|
|
79
|
+
else if (x.no === 7) packedInts(buf, x, o.enclosing);
|
|
80
|
+
}
|
|
81
|
+
doc.occ.push(o);
|
|
82
|
+
} else if (d.no === 3 && d.wt === 2) {
|
|
83
|
+
// SymbolInformation → implementation relationships only
|
|
84
|
+
let sym = "";
|
|
85
|
+
const impl = [];
|
|
86
|
+
for (const x of fields(buf, d.a, d.b)) {
|
|
87
|
+
if (x.no === 1 && x.wt === 2) sym = str(buf, x);
|
|
88
|
+
else if (x.no === 4 && x.wt === 2) {
|
|
89
|
+
let rsym = "", isImpl = false;
|
|
90
|
+
for (const r of fields(buf, x.a, x.b)) {
|
|
91
|
+
if (r.no === 1 && r.wt === 2) rsym = str(buf, r);
|
|
92
|
+
else if (r.no === 3 && r.wt === 0) isImpl = r.val !== 0n;
|
|
93
|
+
}
|
|
94
|
+
if (isImpl && rsym) impl.push(rsym);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (impl.length) doc.rel.push({ sym, impl });
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
docs.push(doc);
|
|
101
|
+
}
|
|
102
|
+
return { docs, projectRoot };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// ---- SCIP symbol grammar helpers -------------------------------------------
|
|
106
|
+
// e.g. "scip-typescript npm @geml/geml 1.0.0 src/`geml.ts`/parse()."
|
|
107
|
+
const isFuncSym = (s) => s.endsWith("().");
|
|
108
|
+
const nameOf = (s) => {
|
|
109
|
+
// Class members read class-qualified (`RenderCtx.block`), constructors as
|
|
110
|
+
// `Cls.new` — free functions (no `Owner#` scope) keep their plain name.
|
|
111
|
+
if (/`?<constructor>`?\(\)\.$/.test(s)) {
|
|
112
|
+
const cm = /([A-Za-z0-9_$]+)#`?<constructor>`?\(\)\.$/.exec(s);
|
|
113
|
+
return cm ? `${cm[1]}.new` : "new";
|
|
114
|
+
}
|
|
115
|
+
const m = /(?:([A-Za-z0-9_$]+)#)?([^\/#.`]+)\(\)\.$/.exec(s);
|
|
116
|
+
if (m) return m[1] ? `${m[1]}.${m[2]}` : m[2];
|
|
117
|
+
return s.split("/").pop() ?? s;
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
export function extract({ raw: scipPath, root }) {
|
|
121
|
+
const { docs, projectRoot } = parseScip(scipPath);
|
|
122
|
+
// scip-typescript emits OS-native separators in relative_path on Windows;
|
|
123
|
+
// the codemap profile is posix throughout.
|
|
124
|
+
for (const d of docs) d.path = d.path.replace(/\\/g, "/");
|
|
125
|
+
// Document paths are relative to the INDEXED project (metadata.project_root),
|
|
126
|
+
// which may be a subdirectory of the codemap's --root. Re-anchor them so a
|
|
127
|
+
// multi-language merge keeps one coherent repo-relative path space.
|
|
128
|
+
if (projectRoot && root) {
|
|
129
|
+
const norm = (p) => p.replace(/^file:\/\/\/?/, "").replace(/\\/g, "/").replace(/\/+$/, "").toLowerCase();
|
|
130
|
+
const rootN = norm(resolvePath(root));
|
|
131
|
+
const projN = norm(decodeURIComponent(projectRoot));
|
|
132
|
+
if (projN !== rootN && projN.startsWith(rootN + "/")) {
|
|
133
|
+
const prefix = decodeURIComponent(projectRoot).replace(/^file:\/\/\/?/, "").replace(/\\/g, "/").replace(/\/+$/, "").slice(rootN.length + 1);
|
|
134
|
+
for (const d of docs) d.path = `${prefix}/${d.path}`;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// range = [startLine, startChar, endLine(, endChar)] (0-based); normalize.
|
|
139
|
+
const spanOf = (r) => (r.length === 3 ? [r[0], r[0]] : [r[0], r[2]]);
|
|
140
|
+
|
|
141
|
+
// 1. definitions of function symbols
|
|
142
|
+
const defs = new Map(); // symbol -> {file, name, line_start, line_end, encl:[sl,el]}
|
|
143
|
+
for (const d of docs) {
|
|
144
|
+
for (const o of d.occ) {
|
|
145
|
+
if (!(o.roles & ROLE_DEFINITION) || !isFuncSym(o.symbol)) continue;
|
|
146
|
+
const [nl] = spanOf(o.range);
|
|
147
|
+
const encl = o.enclosing.length ? spanOf(o.enclosing) : [nl, nl];
|
|
148
|
+
const prev = defs.get(o.symbol);
|
|
149
|
+
// keep the widest definition (impl over overload signatures)
|
|
150
|
+
if (!prev || encl[1] - encl[0] > prev.encl[1] - prev.encl[0]) {
|
|
151
|
+
defs.set(o.symbol, { file: d.path, name: nameOf(o.symbol), line_start: encl[0] + 1, line_end: encl[1] + 1, encl });
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
const enclosingDegraded = [...defs.values()].every((v) => v.encl[0] === v.encl[1]);
|
|
156
|
+
|
|
157
|
+
// implementations map: interface/abstract member symbol -> implementing symbols
|
|
158
|
+
const implOf = new Map();
|
|
159
|
+
for (const d of docs) {
|
|
160
|
+
for (const { sym, impl } of d.rel) {
|
|
161
|
+
for (const target of impl) {
|
|
162
|
+
if (!implOf.has(target)) implOf.set(target, []);
|
|
163
|
+
implOf.get(target).push(sym);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const symbols = [];
|
|
169
|
+
const seenFiles = new Set();
|
|
170
|
+
for (const [sym, v] of defs) {
|
|
171
|
+
symbols.push({
|
|
172
|
+
anchor: sym, lang: "typescript", kind: "Function", name: v.name,
|
|
173
|
+
file: v.file, line_start: v.line_start, line_end: v.line_end,
|
|
174
|
+
entry: v.name === "main" ? true : undefined,
|
|
175
|
+
resolution: "cpg",
|
|
176
|
+
});
|
|
177
|
+
if (!seenFiles.has(v.file)) {
|
|
178
|
+
seenFiles.add(v.file);
|
|
179
|
+
symbols.push({ anchor: `file:${v.file}`, lang: "typescript", kind: "File", name: v.file.split("/").pop(), file: v.file, resolution: "cpg" });
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// 2. calls: reference occurrences of function symbols, attributed to the
|
|
184
|
+
// innermost containing definition of the same file.
|
|
185
|
+
const perFileDefs = new Map(); // file -> defs sorted by span size asc
|
|
186
|
+
for (const [sym, v] of defs) {
|
|
187
|
+
if (!perFileDefs.has(v.file)) perFileDefs.set(v.file, []);
|
|
188
|
+
perFileDefs.get(v.file).push({ sym, ...v });
|
|
189
|
+
}
|
|
190
|
+
for (const list of perFileDefs.values()) list.sort((a, b) => (a.encl[1] - a.encl[0]) - (b.encl[1] - b.encl[0]));
|
|
191
|
+
const callerAt = (file, line) => {
|
|
192
|
+
const list = perFileDefs.get(file);
|
|
193
|
+
if (!list) return undefined;
|
|
194
|
+
if (!enclosingDegraded) {
|
|
195
|
+
for (const d of list) if (line >= d.encl[0] && line <= d.encl[1]) return d.sym; // innermost first (sorted asc)
|
|
196
|
+
return undefined;
|
|
197
|
+
}
|
|
198
|
+
// degraded: nearest preceding definition start
|
|
199
|
+
let best;
|
|
200
|
+
for (const d of list) if (d.encl[0] <= line && (!best || d.encl[0] > best.encl[0])) best = d;
|
|
201
|
+
return best?.sym;
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
const edges = [];
|
|
205
|
+
for (const d of docs) {
|
|
206
|
+
for (const o of d.occ) {
|
|
207
|
+
if ((o.roles & ROLE_DEFINITION) || !isFuncSym(o.symbol)) continue;
|
|
208
|
+
const [line] = spanOf(o.range);
|
|
209
|
+
const from = callerAt(d.path, line);
|
|
210
|
+
if (!from || from === o.symbol) continue;
|
|
211
|
+
const site = { file: d.path, line: line + 1 };
|
|
212
|
+
if (defs.has(o.symbol)) {
|
|
213
|
+
const impls = (implOf.get(o.symbol) ?? []).filter((s) => defs.has(s));
|
|
214
|
+
if (impls.length) {
|
|
215
|
+
edges.push({ kind: "calls", from, to: o.symbol, resolution: "cpg", confidence: "medium", note: `dispatch, ${impls.length + 1} candidates`, candidates: [...new Set(impls)].sort(), site });
|
|
216
|
+
} else {
|
|
217
|
+
edges.push({ kind: "calls", from, to: o.symbol, resolution: "cpg", confidence: "high", site });
|
|
218
|
+
}
|
|
219
|
+
} else {
|
|
220
|
+
edges.push({ kind: "calls", from, to_text: nameOf(o.symbol), resolution: "cpg", confidence: "low", site });
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
if (enclosingDegraded) {
|
|
226
|
+
console.error("scip adapter: no enclosing_range in this index — caller attribution degraded to nearest-preceding definition");
|
|
227
|
+
}
|
|
228
|
+
return { symbols, edges };
|
|
229
|
+
}
|