@geml/geml 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +79 -72
- package/codemap/adapters/crg.mjs +109 -0
- package/codemap/adapters/joern.mjs +131 -0
- package/codemap/adapters/scip.mjs +229 -0
- package/codemap/browser-stub.mjs +24 -0
- package/codemap/build.mjs +354 -0
- package/codemap/detect.mjs +185 -0
- package/codemap/emit.mjs +361 -0
- package/codemap/exclude.mjs +52 -0
- package/codemap/joern-export.sc +83 -0
- package/codemap/mcp-server.mjs +143 -0
- package/codemap/normalize.mjs +0 -0
- package/codemap/refresh.mjs +160 -0
- package/codemap/render-all.mjs +64 -0
- package/codemap/serve.mjs +378 -0
- package/codemap/verify.mjs +126 -0
- package/dist/geml.d.ts +5 -0
- package/dist/geml.js +390 -14
- package/dist/history.d.ts +30 -0
- package/dist/history.js +153 -13
- package/dist/render.d.ts +59 -0
- package/dist/render.js +1685 -11
- package/dist/to-md.js +1 -3
- package/package.json +9 -5
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// geml codemap refresh — re-run a codemap's RECORDED build recipe so the
|
|
3
|
+
// graph stays consistent with the code.
|
|
4
|
+
//
|
|
5
|
+
// geml codemap refresh [codemap-dir] run the recipe now
|
|
6
|
+
// geml codemap refresh [codemap-dir] --background detach, return at once
|
|
7
|
+
// geml codemap refresh [codemap-dir] --hook Claude Code hook adapter
|
|
8
|
+
//
|
|
9
|
+
// The recipe lives at <codemap-dir>/_index/refresh.json — written once, after
|
|
10
|
+
// the first successful build (the geml-code-graph skill records the exact
|
|
11
|
+
// index/build/verify commands it ran):
|
|
12
|
+
//
|
|
13
|
+
// { "root": "..", "steps": ["npx --yes @sourcegraph/scip-typescript index …",
|
|
14
|
+
// "geml codemap build --adapter scip --raw index.scip --root . --out .geml-code-graph --history",
|
|
15
|
+
// "geml codemap verify .geml-code-graph"] }
|
|
16
|
+
//
|
|
17
|
+
// Steps run sequentially with the project root as cwd; the run is skipped
|
|
18
|
+
// when git HEAD hasn't moved past the commit the codemap was built from —
|
|
19
|
+
// read from index.geml's own meta (`commit = <sha>`, stamped by build), so
|
|
20
|
+
// refresh.json stays a pure, human-reviewable recipe that no tool rewrites.
|
|
21
|
+
// Output goes to _index/refresh.log.
|
|
22
|
+
//
|
|
23
|
+
// --hook mode is a PostToolUse adapter: it reads the hook payload from stdin,
|
|
24
|
+
// exits 0 immediately unless the tool ran a `git commit`, and otherwise
|
|
25
|
+
// starts the refresh DETACHED so the commit is never blocked on an indexer.
|
|
26
|
+
// A project without refresh.json is simply not opted in (silent exit 0).
|
|
27
|
+
//
|
|
28
|
+
// --commit: after a successful refresh, commit the refreshed codemap files as
|
|
29
|
+
// their own follow-up commit (chore(codemap): …), so the graph travels with
|
|
30
|
+
// the code on the next push instead of lingering as working-tree churn. The
|
|
31
|
+
// commit is surgical (pathspec = the codemap dir only) and guarded: it is
|
|
32
|
+
// skipped when HEAD moved mid-refresh or a merge is in progress. Loop-safe by
|
|
33
|
+
// construction — the follow-up commit changes no indexed source file, so the
|
|
34
|
+
// refresh it triggers takes the no-source-change skip and stops.
|
|
35
|
+
import { readFileSync, existsSync, appendFileSync, openSync, closeSync } from "node:fs";
|
|
36
|
+
import { join, resolve, relative } from "node:path";
|
|
37
|
+
import { spawnSync, spawn } from "node:child_process";
|
|
38
|
+
import { isSourcePath } from "./detect.mjs";
|
|
39
|
+
|
|
40
|
+
const args = process.argv.slice(2);
|
|
41
|
+
const hookMode = args.includes("--hook");
|
|
42
|
+
const background = args.includes("--background");
|
|
43
|
+
// --force: rebuild even when the repo commit is unchanged — the recipe's
|
|
44
|
+
// up-to-date check watches the CODE, but a toolchain upgrade (new adapter
|
|
45
|
+
// naming, new emit shape) changes the OUTPUT for the same code.
|
|
46
|
+
const force = args.includes("--force");
|
|
47
|
+
const autoCommit = args.includes("--commit");
|
|
48
|
+
if (args.includes("--help")) {
|
|
49
|
+
console.error("usage: geml codemap refresh [codemap-dir] [--force] [--commit] [--background|--hook] (dir defaults to ./.geml-code-graph)");
|
|
50
|
+
process.exit(2);
|
|
51
|
+
}
|
|
52
|
+
const dir = args.find((a) => !a.startsWith("--")) || ".geml-code-graph";
|
|
53
|
+
const cmDir = resolve(dir);
|
|
54
|
+
const cfgPath = join(cmDir, "_index", "refresh.json");
|
|
55
|
+
const logPath = join(cmDir, "_index", "refresh.log");
|
|
56
|
+
|
|
57
|
+
if (!existsSync(cfgPath)) {
|
|
58
|
+
if (hookMode) process.exit(0); // no recipe = this project has not opted in
|
|
59
|
+
console.error(`error: ${cfgPath} not found — record the build recipe there first (see the geml-code-graph skill)`);
|
|
60
|
+
process.exit(1);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (hookMode) {
|
|
64
|
+
// PostToolUse payload on stdin; only a git commit warrants a refresh.
|
|
65
|
+
let cmd = "";
|
|
66
|
+
try { cmd = JSON.parse(readFileSync(0, "utf8"))?.tool_input?.command ?? ""; } catch { /* not JSON: ignore */ }
|
|
67
|
+
if (!/(^|[;&|]\s*)(\S+\s+)?git\s+(\S+\s+)*commit\b/.test(cmd)) process.exit(0);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (hookMode || background) {
|
|
71
|
+
const child = spawn(process.execPath, [process.argv[1], cmDir, ...(force ? ["--force"] : []), ...(autoCommit ? ["--commit"] : [])], { detached: true, stdio: "ignore" });
|
|
72
|
+
child.unref();
|
|
73
|
+
console.error(`codemap refresh: running in background (log: ${logPath})`);
|
|
74
|
+
process.exit(0);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const cfg = JSON.parse(readFileSync(cfgPath, "utf8"));
|
|
78
|
+
const root = resolve(cmDir, cfg.root ?? "..");
|
|
79
|
+
let head;
|
|
80
|
+
try {
|
|
81
|
+
const r = spawnSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" });
|
|
82
|
+
head = r.status === 0 ? r.stdout.trim() : undefined;
|
|
83
|
+
} catch { /* no git: refresh unconditionally */ }
|
|
84
|
+
// The commit this codemap was built from: build stamps it into index.geml's
|
|
85
|
+
// meta (`commit = <short-sha>`), so the graph itself carries the baseline and
|
|
86
|
+
// refresh.json is never rewritten. A legacy `last_commit` in refresh.json is
|
|
87
|
+
// honored as a fallback for codemaps built before the meta stamp.
|
|
88
|
+
let builtFrom;
|
|
89
|
+
try {
|
|
90
|
+
const m = /^commit = "?([0-9a-fA-F]{4,40})"?\r?$/m.exec(readFileSync(join(cmDir, "index.geml"), "utf8").slice(0, 4000));
|
|
91
|
+
if (m) builtFrom = m[1];
|
|
92
|
+
} catch { /* no index yet: first build */ }
|
|
93
|
+
if (!builtFrom && cfg.last_commit) builtFrom = cfg.last_commit;
|
|
94
|
+
if (!force && head && builtFrom && head.startsWith(builtFrom)) {
|
|
95
|
+
console.error(`codemap refresh: up to date at ${head.slice(0, 10)} (--force to rebuild anyway)`);
|
|
96
|
+
process.exit(0);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// HEAD moved past the built-from commit, but if no INDEXED source file
|
|
100
|
+
// changed in between (docs, config, CI only) the graph can't have changed —
|
|
101
|
+
// skip the slow re-index. --force, a first build (no baseline), or an
|
|
102
|
+
// uncomputable diff all fall through and rebuild.
|
|
103
|
+
if (!force && head && builtFrom) {
|
|
104
|
+
let changed;
|
|
105
|
+
try {
|
|
106
|
+
const r = spawnSync("git", ["-C", root, "diff", "--name-only", builtFrom, head], { encoding: "utf8" });
|
|
107
|
+
if (r.status === 0) changed = r.stdout.split("\n").filter(Boolean);
|
|
108
|
+
} catch { /* diff unavailable: fall through and rebuild */ }
|
|
109
|
+
if (changed && !changed.some(isSourcePath)) {
|
|
110
|
+
console.error(`codemap refresh: no source files changed since ${builtFrom.slice(0, 10)} — skipped (${changed.length} non-source file(s); --force to rebuild)`);
|
|
111
|
+
process.exit(0);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
appendFileSync(logPath, `\n[${new Date().toISOString()}] refresh @ ${head ?? "no-git"}\n`);
|
|
116
|
+
for (const step of cfg.steps ?? []) {
|
|
117
|
+
appendFileSync(logPath, `$ ${step}\n`);
|
|
118
|
+
// Stream the step's output STRAIGHT into the log file. Capturing it in
|
|
119
|
+
// memory (spawnSync + encoding) hits the default 1MB maxBuffer — Joern's
|
|
120
|
+
// INFO firehose blew it and the child got killed mid-export (exit null).
|
|
121
|
+
// A file descriptor has no such limit, and the log tails live.
|
|
122
|
+
const fd = openSync(logPath, "a");
|
|
123
|
+
const r = spawnSync(step, { shell: true, cwd: root, stdio: ["ignore", fd, fd] });
|
|
124
|
+
closeSync(fd);
|
|
125
|
+
if (r.status !== 0) {
|
|
126
|
+
const why = r.status ?? r.signal ?? r.error?.message ?? "killed";
|
|
127
|
+
appendFileSync(logPath, `FAILED (exit ${why})\n`);
|
|
128
|
+
console.error(`codemap refresh: step failed (exit ${why}): ${step}\n log: ${logPath}`);
|
|
129
|
+
process.exit(1);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
appendFileSync(logPath, "ok\n");
|
|
133
|
+
console.error(`codemap refresh: done${head ? ` @ ${head.slice(0, 10)}` : ""} (${(cfg.steps ?? []).length} step(s))`);
|
|
134
|
+
|
|
135
|
+
// --commit: land the refreshed codemap as its own follow-up commit so it
|
|
136
|
+
// rides the next push with the code. Guards: HEAD must not have moved while
|
|
137
|
+
// the indexer ran (a switched branch or new commit means these files belong
|
|
138
|
+
// to a different base — leave them in the tree), and never during a merge.
|
|
139
|
+
if (autoCommit && head) {
|
|
140
|
+
const g = (...a) => spawnSync("git", ["-C", root, ...a], { encoding: "utf8" });
|
|
141
|
+
const headNow = g("rev-parse", "HEAD").stdout?.trim();
|
|
142
|
+
const merging = g("rev-parse", "-q", "--verify", "MERGE_HEAD").status === 0;
|
|
143
|
+
if (headNow !== head || merging) {
|
|
144
|
+
console.error(`codemap refresh: not auto-committing (${merging ? "merge in progress" : "HEAD moved during the refresh"}) — refreshed files left in the working tree`);
|
|
145
|
+
} else {
|
|
146
|
+
const rel = relative(root, cmDir).replace(/\\/g, "/") || ".";
|
|
147
|
+
// Runtime noise in _index (refresh/serve logs, serve.pid) never belongs in
|
|
148
|
+
// the commit — and this very run appends to refresh.log after committing.
|
|
149
|
+
const spec = ["--", rel, `:(exclude)${rel}/_index/refresh.log`, `:(exclude)${rel}/_index/serve.log`, `:(exclude)${rel}/_index/serve.pid`];
|
|
150
|
+
g("add", "-A", ...spec); // new pages need staging; pathspec keeps it surgical
|
|
151
|
+
const c = g("commit", "-m", `chore(codemap): refresh for ${head.slice(0, 7)}`, ...spec);
|
|
152
|
+
if (c.status === 0) {
|
|
153
|
+
const sha = g("rev-parse", "--short", "HEAD").stdout?.trim();
|
|
154
|
+
appendFileSync(logPath, `auto-commit ${sha}\n`);
|
|
155
|
+
console.error(`codemap refresh: committed as ${sha} (chore(codemap): refresh for ${head.slice(0, 7)})`);
|
|
156
|
+
} else {
|
|
157
|
+
console.error(`codemap refresh: nothing to commit (codemap unchanged)`);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// geml codemap render — render every codemap document to a sibling .html.
|
|
3
|
+
//
|
|
4
|
+
// geml codemap render [codemap-dir]
|
|
5
|
+
//
|
|
6
|
+
// The output folder then works with NO server: open index.html straight from
|
|
7
|
+
// disk (file://). Module click-through opens each container page inside the
|
|
8
|
+
// graph area (nested frame), so the whole map is browsable offline — this is
|
|
9
|
+
// the "copy the folder to someone" mode. For a live view that never goes
|
|
10
|
+
// stale, use `geml codemap serve` instead.
|
|
11
|
+
import { readdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
12
|
+
import { join, basename } from "node:path";
|
|
13
|
+
import { parse, renderHtml } from "../dist/geml.js";
|
|
14
|
+
|
|
15
|
+
if (process.argv[2] === "--help" || process.argv[2] === "-h") {
|
|
16
|
+
console.error("usage: geml codemap render [codemap-dir] (dir defaults to ./.geml-code-graph)");
|
|
17
|
+
process.exit(2);
|
|
18
|
+
}
|
|
19
|
+
const dir = process.argv[2] || ".geml-code-graph";
|
|
20
|
+
|
|
21
|
+
// One shared cache for the whole batch: every page's graph slice crosses the
|
|
22
|
+
// same neighbour documents, and a fresh parse per page turns N pages into
|
|
23
|
+
// O(N x working set) — hours at repo scale. A one-shot process has no
|
|
24
|
+
// staleness to worry about, so cache unconditionally (the whole codemap's
|
|
25
|
+
// text + parsed docs live in memory for the duration of the run).
|
|
26
|
+
const texts = new Map(); // rel -> text | null
|
|
27
|
+
const parsed = new Map(); // text -> Document
|
|
28
|
+
const loadDoc = (rel) => {
|
|
29
|
+
if (!texts.has(rel)) {
|
|
30
|
+
try { texts.set(rel, readFileSync(join(dir, rel), "utf8")); } catch { texts.set(rel, null); }
|
|
31
|
+
}
|
|
32
|
+
return texts.get(rel);
|
|
33
|
+
};
|
|
34
|
+
const parseDoc = (s) => {
|
|
35
|
+
let d = parsed.get(s);
|
|
36
|
+
if (!d) { d = parse(s); parsed.set(s, d); }
|
|
37
|
+
return d;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
let n = 0;
|
|
41
|
+
const failed = [];
|
|
42
|
+
let files;
|
|
43
|
+
try {
|
|
44
|
+
files = readdirSync(dir);
|
|
45
|
+
} catch {
|
|
46
|
+
console.error(`error: cannot read directory ${dir}`);
|
|
47
|
+
process.exit(1);
|
|
48
|
+
}
|
|
49
|
+
for (const f of files) {
|
|
50
|
+
if (!f.endsWith(".geml")) continue;
|
|
51
|
+
try {
|
|
52
|
+
const text = loadDoc(f);
|
|
53
|
+
if (text === null) throw new Error("unreadable");
|
|
54
|
+
const doc = parseDoc(text);
|
|
55
|
+
const html = renderHtml(doc, { source: basename(f), loadDoc, parseDoc });
|
|
56
|
+
writeFileSync(join(dir, f.replace(/\.geml$/, ".html")), html);
|
|
57
|
+
n++;
|
|
58
|
+
} catch (e) {
|
|
59
|
+
failed.push(f);
|
|
60
|
+
console.error(`render: ${f}: ${e.message}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
console.error(`rendered ${n} page(s) -> ${dir}${failed.length ? `; FAILED: ${failed.join(", ")}` : ""}`);
|
|
64
|
+
process.exit(failed.length ? 1 : 0);
|
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// geml codemap serve — live viewer for a codemap directory.
|
|
3
|
+
//
|
|
4
|
+
// geml codemap serve [codemap-dir] [--port 8140] foreground
|
|
5
|
+
// geml codemap serve [codemap-dir] [--port 8140] --background survives the session
|
|
6
|
+
// geml codemap serve [codemap-dir] --stop stop a background server
|
|
7
|
+
// geml codemap serve [codemap-dir] --watch editing-time sync: re-run the
|
|
8
|
+
// recorded recipe when indexed sources change (30s quiet)
|
|
9
|
+
//
|
|
10
|
+
// Every *.html request is rendered FROM ITS *.geml AT REQUEST TIME, so the
|
|
11
|
+
// pages are never stale: rebuild the codemap (or upgrade the renderer) and a
|
|
12
|
+
// browser refresh shows the new state — no pre-render step. Pre-rendered
|
|
13
|
+
// static .html files (from `geml codemap render`) are served only when no
|
|
14
|
+
// .geml source exists for the path.
|
|
15
|
+
//
|
|
16
|
+
// --background detaches the server from the launching process (an agent
|
|
17
|
+
// session ending must not take the viewer down): stdio goes to
|
|
18
|
+
// _index/serve.log, the pid lands in _index/serve.pid, and the parent waits
|
|
19
|
+
// until the port actually answers before reporting the URL.
|
|
20
|
+
//
|
|
21
|
+
// Local viewer by design: binds 127.0.0.1. HEAD is answered without a body —
|
|
22
|
+
// the in-page navigation probes targets before embedding them.
|
|
23
|
+
import { createServer } from "node:http";
|
|
24
|
+
import { readFileSync, writeFileSync, existsSync, statSync, mkdirSync, openSync, unlinkSync, readdirSync, watch } from "node:fs";
|
|
25
|
+
import { join, resolve, sep, basename, dirname } from "node:path";
|
|
26
|
+
import { spawn } from "node:child_process";
|
|
27
|
+
import { fileURLToPath } from "node:url";
|
|
28
|
+
import { parse, renderHtml } from "../dist/geml.js";
|
|
29
|
+
import { buildCodeGraph } from "../dist/render.js";
|
|
30
|
+
import { isSourcePath, SKIP_DIRS } from "./detect.mjs";
|
|
31
|
+
|
|
32
|
+
// Where this package's compiled ESM lives — served under /_dist/ so pages can
|
|
33
|
+
// import the parser in the browser (live in-place navigation).
|
|
34
|
+
const DIST_DIR = resolve(join(dirname(fileURLToPath(import.meta.url)), "..", "dist"));
|
|
35
|
+
|
|
36
|
+
const args = process.argv.slice(2);
|
|
37
|
+
const portIdx = args.indexOf("--port");
|
|
38
|
+
const port = portIdx >= 0 ? Number(args[portIdx + 1]) : 8140;
|
|
39
|
+
const background = args.includes("--background");
|
|
40
|
+
const stop = args.includes("--stop");
|
|
41
|
+
const noWarm = args.includes("--no-warm");
|
|
42
|
+
const noOpen = args.includes("--no-open");
|
|
43
|
+
const watchMode = args.includes("--watch");
|
|
44
|
+
const cacheIdx = args.indexOf("--cache-mb");
|
|
45
|
+
const cacheMb = cacheIdx >= 0 ? Number(args[cacheIdx + 1]) : 256;
|
|
46
|
+
if (args.includes("--help") || args.includes("-h") || !Number.isInteger(port) || port <= 0 || !(cacheMb > 0)) {
|
|
47
|
+
console.error("usage: geml codemap serve [codemap-dir] [--port 8140] [--cache-mb 256] [--no-warm] [--no-open] [--watch] [--background|--stop] (dir defaults to ./.geml-code-graph)");
|
|
48
|
+
process.exit(2);
|
|
49
|
+
}
|
|
50
|
+
const dir = args.find((a, i) => !a.startsWith("--") && (portIdx < 0 || i !== portIdx + 1) && (cacheIdx < 0 || i !== cacheIdx + 1)) || ".geml-code-graph";
|
|
51
|
+
const root = resolve(dir);
|
|
52
|
+
const runDir = join(root, "_index");
|
|
53
|
+
const pidPath = join(runDir, "serve.pid");
|
|
54
|
+
const logPath = join(runDir, "serve.log");
|
|
55
|
+
|
|
56
|
+
// Project root for the source route: a method node's src path is
|
|
57
|
+
// project-root-relative, so it always misses inside the codemap dir. The
|
|
58
|
+
// recorded build recipe knows the root (_index/refresh.json "root", relative
|
|
59
|
+
// to the codemap dir); without one, assume the codemap sits at <root>/<dir>.
|
|
60
|
+
let srcRoot = resolve(root, "..");
|
|
61
|
+
try { srcRoot = resolve(root, JSON.parse(readFileSync(join(root, "_index", "refresh.json"), "utf8")).root ?? ".."); } catch { /* no recipe: parent */ }
|
|
62
|
+
|
|
63
|
+
if (stop) {
|
|
64
|
+
if (!existsSync(pidPath)) { console.error("codemap serve: no pid file — nothing to stop"); process.exit(0); }
|
|
65
|
+
const pid = Number(readFileSync(pidPath, "utf8").trim());
|
|
66
|
+
try {
|
|
67
|
+
process.kill(pid);
|
|
68
|
+
console.error(`codemap serve: stopped (pid ${pid})`);
|
|
69
|
+
} catch {
|
|
70
|
+
console.error(`codemap serve: pid ${pid} not running (stale pid file removed)`);
|
|
71
|
+
}
|
|
72
|
+
try { unlinkSync(pidPath); } catch { /* already gone */ }
|
|
73
|
+
process.exit(0);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (!existsSync(join(root, "index.geml")) && !existsSync(join(root, "index.html"))) {
|
|
77
|
+
console.error(`error: ${root} has no index.geml — not a codemap directory? (build one: geml codemap build)`);
|
|
78
|
+
process.exit(1);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (background) {
|
|
82
|
+
// Already serving? Don't stack a second server on the port.
|
|
83
|
+
try {
|
|
84
|
+
const pre = await fetch(`http://127.0.0.1:${port}/`, { method: "HEAD" });
|
|
85
|
+
if (pre.status > 0) {
|
|
86
|
+
console.error(`codemap serve: port ${port} already answers — assuming it is up`);
|
|
87
|
+
console.error(` -> http://localhost:${port}/ (stop: geml codemap serve ${dir} --stop)`);
|
|
88
|
+
process.exit(0);
|
|
89
|
+
}
|
|
90
|
+
} catch { /* nothing there: start one */ }
|
|
91
|
+
// Detach fully: own process group, stdio to the log file — the child owes
|
|
92
|
+
// the launching session nothing. Report only once the port answers.
|
|
93
|
+
mkdirSync(runDir, { recursive: true });
|
|
94
|
+
const logFd = openSync(logPath, "a");
|
|
95
|
+
const child = spawn(process.execPath,
|
|
96
|
+
[process.argv[1], root, "--port", String(port), "--cache-mb", String(cacheMb), "--no-open", ...(noWarm ? ["--no-warm"] : []), ...(watchMode ? ["--watch"] : [])],
|
|
97
|
+
{ detached: true, stdio: ["ignore", logFd, logFd] });
|
|
98
|
+
child.unref();
|
|
99
|
+
const deadline = Date.now() + 8000;
|
|
100
|
+
let up = false, exited = false;
|
|
101
|
+
child.once("exit", () => { exited = true; });
|
|
102
|
+
while (Date.now() < deadline && !up && !exited) {
|
|
103
|
+
try {
|
|
104
|
+
const r = await fetch(`http://127.0.0.1:${port}/`, { method: "HEAD" });
|
|
105
|
+
up = r.status > 0;
|
|
106
|
+
} catch { await new Promise((r) => setTimeout(r, 250)); }
|
|
107
|
+
}
|
|
108
|
+
if (!up) {
|
|
109
|
+
let tail = "";
|
|
110
|
+
try { tail = readFileSync(logPath, "utf8").split("\n").slice(-4).join("\n "); } catch { /* no log */ }
|
|
111
|
+
console.error(`codemap serve: failed to start on port ${port}\n ${tail}`);
|
|
112
|
+
process.exit(1);
|
|
113
|
+
}
|
|
114
|
+
console.error(`codemap serve: running in background (pid ${child.pid}) — survives this session`);
|
|
115
|
+
console.error(` -> http://localhost:${port}/`);
|
|
116
|
+
console.error(` stop: geml codemap serve ${dir} --stop (log: ${logPath})`);
|
|
117
|
+
process.exit(0);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const MIME = {
|
|
121
|
+
".html": "text/html; charset=utf-8",
|
|
122
|
+
".geml": "text/plain; charset=utf-8",
|
|
123
|
+
".gemlhistory": "text/plain; charset=utf-8",
|
|
124
|
+
".json": "application/json; charset=utf-8",
|
|
125
|
+
".css": "text/css; charset=utf-8",
|
|
126
|
+
".js": "text/javascript; charset=utf-8",
|
|
127
|
+
".svg": "image/svg+xml",
|
|
128
|
+
};
|
|
129
|
+
const extOf = (p) => { const m = /\.[A-Za-z0-9]+$/.exec(p); return m ? m[0].toLowerCase() : ""; };
|
|
130
|
+
|
|
131
|
+
// Parsed-document cache. Pages still render on every request (never stale:
|
|
132
|
+
// entries are validated against mtime+size, so a rebuild is picked up on the
|
|
133
|
+
// next hit), but a click-walk revisits the same multi-MB documents constantly
|
|
134
|
+
// and re-parsing 7 MB of geml per request is pure waste. The LRU bound is a
|
|
135
|
+
// TEXT-BYTE budget, not a document count: one page's graph slice can cross
|
|
136
|
+
// hundreds of small documents (a count bound would thrash — evict and
|
|
137
|
+
// re-parse the whole working set on every request), while a handful of
|
|
138
|
+
// 7 MB documents is what actually threatens memory.
|
|
139
|
+
const DOC_CACHE_BUDGET = cacheMb * 1024 * 1024; // --cache-mb, default 256
|
|
140
|
+
const docCache = new Map(); // abs path -> { mtime, size, text, doc }
|
|
141
|
+
const parsedByText = new Map(); // text (same instance as in docCache) -> doc
|
|
142
|
+
let docCacheBytes = 0;
|
|
143
|
+
const evict = (abs, entry) => {
|
|
144
|
+
parsedByText.delete(entry.text);
|
|
145
|
+
docCache.delete(abs);
|
|
146
|
+
docCacheBytes -= entry.size;
|
|
147
|
+
};
|
|
148
|
+
const loadCached = (abs) => {
|
|
149
|
+
let st;
|
|
150
|
+
try { st = statSync(abs); } catch { return null; }
|
|
151
|
+
const hit = docCache.get(abs);
|
|
152
|
+
if (hit && hit.mtime === st.mtimeMs && hit.size === st.size) {
|
|
153
|
+
docCache.delete(abs); docCache.set(abs, hit); // LRU touch
|
|
154
|
+
return hit;
|
|
155
|
+
}
|
|
156
|
+
if (hit) evict(abs, hit);
|
|
157
|
+
let text;
|
|
158
|
+
try { text = readFileSync(abs, "utf8"); } catch { return null; }
|
|
159
|
+
const entry = { mtime: st.mtimeMs, size: st.size, text, doc: parse(text) };
|
|
160
|
+
docCache.set(abs, entry);
|
|
161
|
+
parsedByText.set(text, entry.doc);
|
|
162
|
+
docCacheBytes += entry.size;
|
|
163
|
+
while (docCacheBytes > DOC_CACHE_BUDGET && docCache.size > 1) {
|
|
164
|
+
const oldest = docCache.keys().next().value;
|
|
165
|
+
evict(oldest, docCache.get(oldest));
|
|
166
|
+
}
|
|
167
|
+
return entry;
|
|
168
|
+
};
|
|
169
|
+
const loadDoc = (rel) => {
|
|
170
|
+
const e = loadCached(join(root, rel));
|
|
171
|
+
return e ? e.text : null;
|
|
172
|
+
};
|
|
173
|
+
// loadDoc hands out the cached string instance, so the by-text lookup hits
|
|
174
|
+
// without re-hashing anything the render loop already loaded.
|
|
175
|
+
const parseDoc = (s) => parsedByText.get(s) ?? parse(s);
|
|
176
|
+
|
|
177
|
+
const server = createServer((req, res) => {
|
|
178
|
+
const send = (status, body, type) => {
|
|
179
|
+
// never-stale extends to the BROWSER: without this, heuristic caching
|
|
180
|
+
// keeps serving yesterday's pages and /_dist modules across restarts.
|
|
181
|
+
res.writeHead(status, { "content-type": type || "text/plain; charset=utf-8", "cache-control": "no-cache" });
|
|
182
|
+
res.end(req.method === "HEAD" ? undefined : body);
|
|
183
|
+
};
|
|
184
|
+
let urlPath;
|
|
185
|
+
try {
|
|
186
|
+
urlPath = decodeURIComponent(new URL(req.url, `http://127.0.0.1:${port}`).pathname);
|
|
187
|
+
} catch {
|
|
188
|
+
return send(400, "bad request");
|
|
189
|
+
}
|
|
190
|
+
if (urlPath.endsWith("/")) urlPath += "index.html";
|
|
191
|
+
|
|
192
|
+
const done = (status) => console.error(`${req.method} ${urlPath} ${status}`);
|
|
193
|
+
// Graph payloads as a sidecar: pages carry data-graph-src instead of a
|
|
194
|
+
// multi-MB inline attribute; the runtime fetches this route after first
|
|
195
|
+
// paint. Computed on demand from the SAME parse cache — never stale.
|
|
196
|
+
if (urlPath === "/_graph") {
|
|
197
|
+
let rel = "";
|
|
198
|
+
try { rel = new URL(req.url, `http://127.0.0.1:${port}`).searchParams.get("doc") || ""; } catch { /* fall through */ }
|
|
199
|
+
const target = resolve(join(root, "." + ("/" + rel).replace(/\//g, sep)));
|
|
200
|
+
if (!rel.endsWith(".geml") || (target !== root && !target.startsWith(root + sep)) || !existsSync(target)) {
|
|
201
|
+
done(404);
|
|
202
|
+
return send(404, JSON.stringify({ error: `no such document: ${rel}` }), "application/json; charset=utf-8");
|
|
203
|
+
}
|
|
204
|
+
try {
|
|
205
|
+
const r = buildCodeGraph(rel, { loadDoc, parseDoc });
|
|
206
|
+
done(200);
|
|
207
|
+
return send(200,
|
|
208
|
+
JSON.stringify(r.error !== undefined ? { error: r.error } : { data: r.data, truncated: !!r.truncated }),
|
|
209
|
+
"application/json; charset=utf-8");
|
|
210
|
+
} catch (e) {
|
|
211
|
+
done(500);
|
|
212
|
+
return send(500, JSON.stringify({ error: e.message }), "application/json; charset=utf-8");
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
// The parser's own ESM dist, for the live module script the pages load —
|
|
216
|
+
// clicks then swap views in place instead of navigating between pages.
|
|
217
|
+
if (urlPath.startsWith("/_dist/")) {
|
|
218
|
+
const sub = urlPath.slice("/_dist/".length);
|
|
219
|
+
// The import map in served pages sends every node:* builtin here, so the
|
|
220
|
+
// parser dist loads in a browser exactly like the bundled viewer does.
|
|
221
|
+
if (sub === "_node-stub.js") {
|
|
222
|
+
done(200);
|
|
223
|
+
return send(200, readFileSync(join(dirname(fileURLToPath(import.meta.url)), "browser-stub.mjs")), "text/javascript; charset=utf-8");
|
|
224
|
+
}
|
|
225
|
+
const distFile = resolve(join(DIST_DIR, "." + sep + sub.replace(/\//g, sep)));
|
|
226
|
+
if (!distFile.startsWith(DIST_DIR + sep) || !distFile.endsWith(".js") || !existsSync(distFile)) {
|
|
227
|
+
done(404);
|
|
228
|
+
return send(404, `not found: ${urlPath}`);
|
|
229
|
+
}
|
|
230
|
+
done(200);
|
|
231
|
+
return send(200, readFileSync(distFile), "text/javascript; charset=utf-8");
|
|
232
|
+
}
|
|
233
|
+
// Stay inside the codemap directory — a viewer, not a file server.
|
|
234
|
+
const file = resolve(join(root, "." + urlPath.replace(/\//g, sep)));
|
|
235
|
+
if (file !== root && !file.startsWith(root + sep)) return send(403, "forbidden");
|
|
236
|
+
// *.html: render the .geml source live when it exists.
|
|
237
|
+
if (urlPath.endsWith(".html")) {
|
|
238
|
+
const geml = file.replace(/\.html$/, ".geml");
|
|
239
|
+
if (existsSync(geml)) {
|
|
240
|
+
try {
|
|
241
|
+
const doc = loadCached(geml).doc;
|
|
242
|
+
const html = renderHtml(doc, {
|
|
243
|
+
source: basename(geml), loadDoc, parseDoc,
|
|
244
|
+
liveGraph: "/_dist/", graphSidecar: "/_graph?doc=",
|
|
245
|
+
});
|
|
246
|
+
done(200);
|
|
247
|
+
return send(200, html, MIME[".html"]);
|
|
248
|
+
} catch (e) {
|
|
249
|
+
done(500);
|
|
250
|
+
return send(500, `render error in ${basename(geml)}: ${e.message}`);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
if (existsSync(file) && statSync(file).isFile()) {
|
|
255
|
+
done(200);
|
|
256
|
+
return send(200, readFileSync(file), MIME[extOf(file)] || "application/octet-stream");
|
|
257
|
+
}
|
|
258
|
+
// Source files as a route: the graph's click-to-source fetches a method's
|
|
259
|
+
// src path (project-root-relative), which misses inside the codemap dir.
|
|
260
|
+
// Resolve the miss against the project root — read-only, indexed source
|
|
261
|
+
// extensions only, traversal-guarded. Still a viewer, not a file server.
|
|
262
|
+
if (isSourcePath(urlPath)) {
|
|
263
|
+
const srcFile = resolve(join(srcRoot, "." + urlPath.replace(/\//g, sep)));
|
|
264
|
+
if (srcFile.startsWith(srcRoot + sep) && existsSync(srcFile) && statSync(srcFile).isFile()) {
|
|
265
|
+
done(200);
|
|
266
|
+
return send(200, readFileSync(srcFile), "text/plain; charset=utf-8");
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
done(404);
|
|
270
|
+
return send(404, `not found: ${urlPath}`);
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
server.on("error", (e) => {
|
|
274
|
+
console.error(e && e.code === "EADDRINUSE"
|
|
275
|
+
? `error: port ${port} is in use — pick another with --port, or stop the old server (geml codemap serve ${dir} --stop)`
|
|
276
|
+
: `error: ${e.message}`);
|
|
277
|
+
process.exit(1);
|
|
278
|
+
});
|
|
279
|
+
// Background prewarm: the parse cache is lazy, so the FIRST click into a big
|
|
280
|
+
// container otherwise pays its whole cross-document working set (seconds at
|
|
281
|
+
// repo scale). Warm largest-first — the big documents are the long-tail
|
|
282
|
+
// first-clicks — ONE document per event-loop turn so requests arriving
|
|
283
|
+
// mid-warm are served normally (a tight synchronous loop would block them),
|
|
284
|
+
// and stop at 80% of the byte budget: warming past it only evicts what was
|
|
285
|
+
// just warmed. Requests still validate mtime+size, so a rebuild mid-warm is
|
|
286
|
+
// picked up as usual.
|
|
287
|
+
async function warmCache() {
|
|
288
|
+
let files = [];
|
|
289
|
+
try {
|
|
290
|
+
files = readdirSync(root)
|
|
291
|
+
.filter((f) => f.endsWith(".geml"))
|
|
292
|
+
.map((f) => {
|
|
293
|
+
const p = join(root, f);
|
|
294
|
+
try { return { p, size: statSync(p).size }; } catch { return null; }
|
|
295
|
+
})
|
|
296
|
+
.filter(Boolean)
|
|
297
|
+
.sort((a, b) => b.size - a.size);
|
|
298
|
+
} catch { return; }
|
|
299
|
+
const t0 = Date.now();
|
|
300
|
+
let n = 0;
|
|
301
|
+
// Brake on CUMULATIVE bytes pushed through the cache, not on current
|
|
302
|
+
// occupancy — the LRU evicts as it goes, so occupancy self-limits and
|
|
303
|
+
// would never stop the loop; past 80% of the budget every further load
|
|
304
|
+
// only evicts something just warmed.
|
|
305
|
+
let warmed = 0;
|
|
306
|
+
for (const { p, size } of files) {
|
|
307
|
+
if (warmed >= DOC_CACHE_BUDGET * 0.8) break;
|
|
308
|
+
if (loadCached(p)) { n++; warmed += size; }
|
|
309
|
+
await new Promise((r) => setImmediate(r));
|
|
310
|
+
}
|
|
311
|
+
console.error(`prewarm: ${n}/${files.length} document(s), ${(docCacheBytes / 1048576).toFixed(1)} MB cached, ${((Date.now() - t0) / 1000).toFixed(1)}s`);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// Open the graph in the default browser — ONLY when serving interactively in a
|
|
315
|
+
// real terminal (isTTY). A --background child (stdio -> log file) and piped/CI
|
|
316
|
+
// runs are non-TTY and never open; `--no-open` opts out explicitly. A missing
|
|
317
|
+
// opener is not an error — the URL is already printed.
|
|
318
|
+
const openBrowser = (url) => {
|
|
319
|
+
const argv = process.platform === "win32" ? ["cmd", "/c", "start", "", url]
|
|
320
|
+
: process.platform === "darwin" ? ["open", url]
|
|
321
|
+
: ["xdg-open", url];
|
|
322
|
+
try { spawn(argv[0], argv.slice(1), { stdio: "ignore", detached: true }).unref(); }
|
|
323
|
+
catch { /* no opener available: the printed URL is enough */ }
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
// --watch: editing-time sync. Watch the project's indexed source files and,
|
|
327
|
+
// after a quiet window, re-run the recorded recipe so the codemap follows the
|
|
328
|
+
// EDIT, not just the commit (the hook covers commits). --force is required:
|
|
329
|
+
// refresh's up-to-date check pins to git HEAD, which editing doesn't move.
|
|
330
|
+
// Single-flight — a change arriving mid-refresh queues exactly one more run.
|
|
331
|
+
// Pages render live from .geml, so when a run lands an F5 shows it.
|
|
332
|
+
const WATCH_QUIET = Number(process.env.GEML_WATCH_QUIET_MS) || 30_000;
|
|
333
|
+
function startWatch() {
|
|
334
|
+
if (!existsSync(join(runDir, "refresh.json"))) {
|
|
335
|
+
console.error("watch: no _index/refresh.json recipe recorded — --watch disabled (build once first)");
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
let timer = null, running = false, again = false;
|
|
339
|
+
const run = () => {
|
|
340
|
+
if (running) { again = true; return; }
|
|
341
|
+
running = true;
|
|
342
|
+
console.error("watch: sources changed — refreshing the codemap…");
|
|
343
|
+
const child = spawn(process.execPath,
|
|
344
|
+
[join(dirname(fileURLToPath(import.meta.url)), "refresh.mjs"), root, "--force"],
|
|
345
|
+
{ stdio: ["ignore", 2, 2] });
|
|
346
|
+
child.on("exit", (c) => {
|
|
347
|
+
running = false;
|
|
348
|
+
console.error(c === 0
|
|
349
|
+
? "watch: codemap refreshed — reload the browser to see it"
|
|
350
|
+
: `watch: refresh failed (exit ${c}) — see ${logPath.replace(/serve\.log$/, "refresh.log")}`);
|
|
351
|
+
if (again) { again = false; schedule(); }
|
|
352
|
+
});
|
|
353
|
+
};
|
|
354
|
+
const schedule = () => { clearTimeout(timer); timer = setTimeout(run, WATCH_QUIET); };
|
|
355
|
+
try {
|
|
356
|
+
watch(srcRoot, { recursive: true }, (_ev, rel) => {
|
|
357
|
+
if (!rel) return;
|
|
358
|
+
const parts = String(rel).split(/[\\/]/);
|
|
359
|
+
if (parts.some((p) => SKIP_DIRS.has(p) || p.startsWith("."))) return;
|
|
360
|
+
if (!isSourcePath(String(rel))) return;
|
|
361
|
+
schedule();
|
|
362
|
+
});
|
|
363
|
+
console.error(`watch: watching ${srcRoot} — a source change re-runs the recipe after ${WATCH_QUIET / 1000}s of quiet`);
|
|
364
|
+
} catch (e) {
|
|
365
|
+
console.error(`watch: recursive fs.watch unavailable here (${e.message}) — --watch disabled`);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
server.listen(port, "127.0.0.1", () => {
|
|
370
|
+
// Record the pid so `--stop` can find us (best effort — a read-only
|
|
371
|
+
// codemap dir just means no pid file).
|
|
372
|
+
try { mkdirSync(runDir, { recursive: true }); writeFileSync(pidPath, String(process.pid)); } catch { /* read-only */ }
|
|
373
|
+
console.error(`geml codemap serve: ${root}`);
|
|
374
|
+
console.error(` -> http://localhost:${port}/ (pages render live from .geml — rebuilds show on refresh)`);
|
|
375
|
+
if (process.stdout.isTTY && !noOpen) openBrowser(`http://localhost:${port}/`);
|
|
376
|
+
if (!noWarm) warmCache();
|
|
377
|
+
if (watchMode) startWatch();
|
|
378
|
+
});
|