@maxgfr/codeindex 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -0
- package/docs/MIGRATION.md +100 -0
- package/package.json +89 -0
- package/scripts/cli.mjs +10 -0
- package/scripts/engine.d.mts +618 -0
- package/scripts/engine.mjs +10655 -0
- package/scripts/grammars/bash.wasm +0 -0
- package/scripts/grammars/c.wasm +0 -0
- package/scripts/grammars/c_sharp.wasm +0 -0
- package/scripts/grammars/cpp.wasm +0 -0
- package/scripts/grammars/go.wasm +0 -0
- package/scripts/grammars/java.wasm +0 -0
- package/scripts/grammars/javascript.wasm +0 -0
- package/scripts/grammars/lua.wasm +0 -0
- package/scripts/grammars/php.wasm +0 -0
- package/scripts/grammars/python.wasm +0 -0
- package/scripts/grammars/ruby.wasm +0 -0
- package/scripts/grammars/rust.wasm +0 -0
- package/scripts/grammars/scala.wasm +0 -0
- package/scripts/grammars/tsx.wasm +0 -0
- package/scripts/grammars/typescript.wasm +0 -0
- package/scripts/grammars/web-tree-sitter.wasm +0 -0
- package/src/ast/extract.ts +713 -0
- package/src/ast/loader.ts +104 -0
- package/src/bm25.ts +156 -0
- package/src/callers.ts +0 -0
- package/src/calls.ts +131 -0
- package/src/categorize.ts +80 -0
- package/src/centrality.ts +148 -0
- package/src/classify.ts +53 -0
- package/src/cli-entry.ts +16 -0
- package/src/community.ts +345 -0
- package/src/complexity.ts +69 -0
- package/src/coupling.ts +0 -0
- package/src/deadcode.ts +46 -0
- package/src/edit.ts +93 -0
- package/src/engine-cli.ts +278 -0
- package/src/engine.ts +148 -0
- package/src/extract/code.ts +376 -0
- package/src/extract/markdown.ts +135 -0
- package/src/git.ts +205 -0
- package/src/glob.ts +56 -0
- package/src/graph.ts +0 -0
- package/src/grep.ts +115 -0
- package/src/hash.ts +13 -0
- package/src/ignore.ts +134 -0
- package/src/lang/c.ts +26 -0
- package/src/lang/common.ts +68 -0
- package/src/lang/csharp.ts +22 -0
- package/src/lang/elixir.ts +18 -0
- package/src/lang/go.ts +22 -0
- package/src/lang/java.ts +19 -0
- package/src/lang/js-ts.ts +112 -0
- package/src/lang/kotlin.ts +20 -0
- package/src/lang/lua.ts +18 -0
- package/src/lang/php.ts +24 -0
- package/src/lang/python.ts +22 -0
- package/src/lang/registry.ts +54 -0
- package/src/lang/ruby.ts +17 -0
- package/src/lang/rust.ts +22 -0
- package/src/lang/scala.ts +19 -0
- package/src/lang/shell.ts +17 -0
- package/src/lang/swift.ts +23 -0
- package/src/mcp.ts +512 -0
- package/src/memory.ts +75 -0
- package/src/modules.ts +128 -0
- package/src/pipeline.ts +59 -0
- package/src/query.ts +118 -0
- package/src/render/graph-json.ts +16 -0
- package/src/render/symbols-json.ts +79 -0
- package/src/repomap.ts +52 -0
- package/src/resolve.ts +950 -0
- package/src/rules.ts +249 -0
- package/src/scan.ts +172 -0
- package/src/sort.ts +12 -0
- package/src/surprise.ts +58 -0
- package/src/tests-map.ts +105 -0
- package/src/types.ts +201 -0
- package/src/util.ts +169 -0
- package/src/viz.ts +44 -0
- package/src/walk.ts +216 -0
- package/src/workspaces.ts +748 -0
package/src/rules.ts
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
// Architecture rules (issue #4): dependency-cruiser-style assertions validated
|
|
2
|
+
// against the built Graph — a deterministic CI gate with zero dependencies.
|
|
3
|
+
// Two rule shapes:
|
|
4
|
+
// forbidden edge {name, from, to, kind?, severity?, comment?} — no resolved
|
|
5
|
+
// file edge may go from a path matching `from` to one matching `to`
|
|
6
|
+
// (globs use src/glob.ts compileGlobs semantics; `kind` narrows to specific
|
|
7
|
+
// edge kinds, default all);
|
|
8
|
+
// builtin {name, builtin: "cycles" | "orphans", severity?, comment?}
|
|
9
|
+
// cycles — module-level import cycles (each strongly-connected component
|
|
10
|
+
// is reported once, as a canonical shortest cycle from its
|
|
11
|
+
// lexicographically smallest module);
|
|
12
|
+
// orphans — code files with no resolved in/out edges, excluding
|
|
13
|
+
// entrypoint-looking basenames (index/main/cli/…).
|
|
14
|
+
// Violations are sorted deterministically (rule, from, to, kind) so two runs on
|
|
15
|
+
// the same graph are byte-identical.
|
|
16
|
+
import type { EdgeKind, Graph } from "./types.js";
|
|
17
|
+
import { compileGlobs } from "./glob.js";
|
|
18
|
+
import { byStr } from "./sort.js";
|
|
19
|
+
|
|
20
|
+
export type RuleSeverity = "error" | "warn";
|
|
21
|
+
|
|
22
|
+
export interface ForbiddenEdgeRule {
|
|
23
|
+
name: string;
|
|
24
|
+
from: string | string[]; // glob(s) over the source file's repo-relative path
|
|
25
|
+
to: string | string[]; // glob(s) over the target file's repo-relative path
|
|
26
|
+
kind?: EdgeKind[]; // restrict to these edge kinds (default: all)
|
|
27
|
+
severity?: RuleSeverity; // default "error"
|
|
28
|
+
comment?: string; // rationale, echoed on each violation
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface BuiltinRule {
|
|
32
|
+
name: string;
|
|
33
|
+
builtin: "cycles" | "orphans";
|
|
34
|
+
severity?: RuleSeverity;
|
|
35
|
+
comment?: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export type ArchRule = ForbiddenEdgeRule | BuiltinRule;
|
|
39
|
+
|
|
40
|
+
export interface RuleViolation {
|
|
41
|
+
rule: string;
|
|
42
|
+
from: string;
|
|
43
|
+
to: string; // for a cycle: the full path, "a -> b -> a"
|
|
44
|
+
kind: EdgeKind | "cycle" | "orphan";
|
|
45
|
+
severity: RuleSeverity;
|
|
46
|
+
comment?: string;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const EDGE_KINDS = new Set<string>(["contains", "doc-link", "import", "call", "use", "mention"]);
|
|
50
|
+
const SEVERITIES = new Set<string>(["error", "warn"]);
|
|
51
|
+
const BUILTINS = new Set<string>(["cycles", "orphans"]);
|
|
52
|
+
|
|
53
|
+
// A basename that looks like an entrypoint — excluded from the orphans check,
|
|
54
|
+
// because nothing is EXPECTED to import a main/cli/server entry.
|
|
55
|
+
const ENTRYPOINT_STEMS = new Set([
|
|
56
|
+
"index",
|
|
57
|
+
"main",
|
|
58
|
+
"app",
|
|
59
|
+
"application",
|
|
60
|
+
"cli",
|
|
61
|
+
"server",
|
|
62
|
+
"entry",
|
|
63
|
+
"entrypoint",
|
|
64
|
+
"setup",
|
|
65
|
+
"conftest",
|
|
66
|
+
"__init__",
|
|
67
|
+
"__main__",
|
|
68
|
+
"mod",
|
|
69
|
+
"lib",
|
|
70
|
+
]);
|
|
71
|
+
|
|
72
|
+
function isEntrypointLike(rel: string): boolean {
|
|
73
|
+
const base = rel.split("/").pop()!;
|
|
74
|
+
const stem = base.split(".")[0]!.toLowerCase();
|
|
75
|
+
return ENTRYPOINT_STEMS.has(stem);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function toList(v: string | string[]): string[] {
|
|
79
|
+
return Array.isArray(v) ? v : [v];
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Validate an untrusted rules payload (CLI --config file, MCP inline JSON) into
|
|
83
|
+
// a typed rules array. Accepts either a bare array or a `{ rules: [...] }`
|
|
84
|
+
// wrapper. Throws a descriptive error on the first malformed entry.
|
|
85
|
+
export function parseRules(input: unknown): ArchRule[] {
|
|
86
|
+
const raw = Array.isArray(input) ? input : (input as { rules?: unknown } | null)?.rules;
|
|
87
|
+
if (!Array.isArray(raw)) throw new Error("rules config must be an array (or an object with a `rules` array)");
|
|
88
|
+
return raw.map((entry, i) => {
|
|
89
|
+
const at = `rules[${i}]`;
|
|
90
|
+
if (typeof entry !== "object" || entry === null) throw new Error(`${at}: must be an object`);
|
|
91
|
+
const r = entry as Record<string, unknown>;
|
|
92
|
+
if (typeof r.name !== "string" || !r.name) throw new Error(`${at}: \`name\` (non-empty string) is required`);
|
|
93
|
+
if (r.severity !== undefined && !SEVERITIES.has(r.severity as string))
|
|
94
|
+
throw new Error(`${at} (${r.name}): \`severity\` must be "error" or "warn"`);
|
|
95
|
+
if (r.comment !== undefined && typeof r.comment !== "string")
|
|
96
|
+
throw new Error(`${at} (${r.name}): \`comment\` must be a string`);
|
|
97
|
+
if (r.builtin !== undefined) {
|
|
98
|
+
if (!BUILTINS.has(r.builtin as string))
|
|
99
|
+
throw new Error(`${at} (${r.name}): \`builtin\` must be "cycles" or "orphans"`);
|
|
100
|
+
return { name: r.name, builtin: r.builtin, severity: r.severity, comment: r.comment } as BuiltinRule;
|
|
101
|
+
}
|
|
102
|
+
const glob = (field: "from" | "to"): string | string[] => {
|
|
103
|
+
const v = r[field];
|
|
104
|
+
const ok = typeof v === "string" ? v.length > 0 : Array.isArray(v) && v.length > 0 && v.every((g) => typeof g === "string" && g);
|
|
105
|
+
if (!ok) throw new Error(`${at} (${r.name}): \`${field}\` must be a glob or a non-empty array of globs`);
|
|
106
|
+
return v as string | string[];
|
|
107
|
+
};
|
|
108
|
+
const from = glob("from");
|
|
109
|
+
const to = glob("to");
|
|
110
|
+
if (r.kind !== undefined) {
|
|
111
|
+
const ok = Array.isArray(r.kind) && r.kind.every((k) => EDGE_KINDS.has(k as string));
|
|
112
|
+
if (!ok) throw new Error(`${at} (${r.name}): \`kind\` must be an array of edge kinds (${[...EDGE_KINDS].join(", ")})`);
|
|
113
|
+
}
|
|
114
|
+
return { name: r.name, from, to, kind: r.kind, severity: r.severity, comment: r.comment } as ForbiddenEdgeRule;
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Module-level import cycles. One violation per strongly-connected component
|
|
119
|
+
// (iterative Tarjan over the import-kind module edges), rendered as a canonical
|
|
120
|
+
// cycle: BFS from the SCC's smallest module along sorted adjacency, closing
|
|
121
|
+
// through the (nearest, then smallest) member with an edge back to the start.
|
|
122
|
+
function findImportCycles(graph: Graph): { start: string; path: string[] }[] {
|
|
123
|
+
const adj = new Map<string, string[]>();
|
|
124
|
+
for (const e of graph.moduleEdges) {
|
|
125
|
+
if (e.kind !== "import") continue;
|
|
126
|
+
let list = adj.get(e.from);
|
|
127
|
+
if (!list) adj.set(e.from, (list = []));
|
|
128
|
+
list.push(e.to);
|
|
129
|
+
}
|
|
130
|
+
for (const list of adj.values()) list.sort(byStr);
|
|
131
|
+
const nodes = [...adj.keys()].sort(byStr);
|
|
132
|
+
|
|
133
|
+
// Iterative Tarjan SCC.
|
|
134
|
+
const indexOf = new Map<string, number>();
|
|
135
|
+
const low = new Map<string, number>();
|
|
136
|
+
const onStack = new Set<string>();
|
|
137
|
+
const stack: string[] = [];
|
|
138
|
+
const sccs: string[][] = [];
|
|
139
|
+
let counter = 0;
|
|
140
|
+
for (const root of nodes) {
|
|
141
|
+
if (indexOf.has(root)) continue;
|
|
142
|
+
const work: { node: string; next: number }[] = [{ node: root, next: 0 }];
|
|
143
|
+
while (work.length) {
|
|
144
|
+
const frame = work[work.length - 1]!;
|
|
145
|
+
const v = frame.node;
|
|
146
|
+
if (frame.next === 0) {
|
|
147
|
+
indexOf.set(v, counter);
|
|
148
|
+
low.set(v, counter);
|
|
149
|
+
counter++;
|
|
150
|
+
stack.push(v);
|
|
151
|
+
onStack.add(v);
|
|
152
|
+
}
|
|
153
|
+
const targets = adj.get(v) ?? [];
|
|
154
|
+
if (frame.next < targets.length) {
|
|
155
|
+
const w = targets[frame.next]!;
|
|
156
|
+
frame.next++;
|
|
157
|
+
if (!indexOf.has(w)) work.push({ node: w, next: 0 });
|
|
158
|
+
else if (onStack.has(w)) low.set(v, Math.min(low.get(v)!, indexOf.get(w)!));
|
|
159
|
+
} else {
|
|
160
|
+
if (low.get(v) === indexOf.get(v)) {
|
|
161
|
+
const scc: string[] = [];
|
|
162
|
+
for (;;) {
|
|
163
|
+
const w = stack.pop()!;
|
|
164
|
+
onStack.delete(w);
|
|
165
|
+
scc.push(w);
|
|
166
|
+
if (w === v) break;
|
|
167
|
+
}
|
|
168
|
+
if (scc.length > 1) sccs.push(scc);
|
|
169
|
+
}
|
|
170
|
+
work.pop();
|
|
171
|
+
const parent = work[work.length - 1];
|
|
172
|
+
if (parent) low.set(parent.node, Math.min(low.get(parent.node)!, low.get(v)!));
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const cycles: { start: string; path: string[] }[] = [];
|
|
178
|
+
for (const scc of sccs) {
|
|
179
|
+
const members = new Set(scc);
|
|
180
|
+
const start = [...scc].sort(byStr)[0]!;
|
|
181
|
+
// BFS within the SCC from `start`; parent links reconstruct the path.
|
|
182
|
+
const parent = new Map<string, string | null>([[start, null]]);
|
|
183
|
+
const order: string[] = [start];
|
|
184
|
+
for (let i = 0; i < order.length; i++) {
|
|
185
|
+
const v = order[i]!;
|
|
186
|
+
for (const w of adj.get(v) ?? []) {
|
|
187
|
+
if (!members.has(w) || parent.has(w)) continue;
|
|
188
|
+
parent.set(w, v);
|
|
189
|
+
order.push(w);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
// The closing hop: the BFS-nearest (then smallest) member with an edge back
|
|
193
|
+
// to start. BFS `order` is deterministic, so `closer` is too.
|
|
194
|
+
const closer = order.find((v) => (adj.get(v) ?? []).includes(start) && v !== start) ??
|
|
195
|
+
// Degenerate (shouldn't happen in an SCC): fall back to start itself.
|
|
196
|
+
start;
|
|
197
|
+
const path: string[] = [];
|
|
198
|
+
for (let v: string | null = closer; v !== null; v = parent.get(v) ?? null) path.unshift(v);
|
|
199
|
+
path.push(start);
|
|
200
|
+
cycles.push({ start, path });
|
|
201
|
+
}
|
|
202
|
+
return cycles;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// Validate `rules` against the built graph. Pure and deterministic: violations
|
|
206
|
+
// are fully sorted; severity defaults to "error"; a rule's `comment` (when set)
|
|
207
|
+
// is echoed onto each of its violations.
|
|
208
|
+
export function checkRules(graph: Graph, rules: ArchRule[]): RuleViolation[] {
|
|
209
|
+
const out: RuleViolation[] = [];
|
|
210
|
+
const emit = (rule: ArchRule, v: Omit<RuleViolation, "rule" | "severity" | "comment">): void => {
|
|
211
|
+
out.push({
|
|
212
|
+
rule: rule.name,
|
|
213
|
+
...v,
|
|
214
|
+
severity: rule.severity ?? "error",
|
|
215
|
+
...(rule.comment !== undefined ? { comment: rule.comment } : {}),
|
|
216
|
+
});
|
|
217
|
+
};
|
|
218
|
+
const fileSet = new Set(graph.files.map((f) => f.rel));
|
|
219
|
+
|
|
220
|
+
for (const rule of rules) {
|
|
221
|
+
if ("builtin" in rule) {
|
|
222
|
+
if (rule.builtin === "cycles") {
|
|
223
|
+
for (const c of findImportCycles(graph)) {
|
|
224
|
+
emit(rule, { from: c.start, to: c.path.join(" -> "), kind: "cycle" });
|
|
225
|
+
}
|
|
226
|
+
} else {
|
|
227
|
+
for (const f of graph.files) {
|
|
228
|
+
if (f.fileKind !== "code" || f.degIn !== 0 || f.degOut !== 0) continue;
|
|
229
|
+
if (isEntrypointLike(f.rel)) continue;
|
|
230
|
+
emit(rule, { from: f.rel, to: f.rel, kind: "orphan" });
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
const fromMatch = compileGlobs(toList(rule.from));
|
|
236
|
+
const toMatch = compileGlobs(toList(rule.to));
|
|
237
|
+
if (!fromMatch || !toMatch) continue; // empty glob list — matches nothing
|
|
238
|
+
const kinds = rule.kind?.length ? new Set<string>(rule.kind) : null;
|
|
239
|
+
for (const e of graph.fileEdges) {
|
|
240
|
+
if (e.dangling || !fileSet.has(e.to)) continue;
|
|
241
|
+
if (kinds && !kinds.has(e.kind)) continue;
|
|
242
|
+
if (!fromMatch(e.from) || !toMatch(e.to)) continue;
|
|
243
|
+
emit(rule, { from: e.from, to: e.to, kind: e.kind });
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
out.sort((a, b) => byStr(a.rule, b.rule) || byStr(a.from, b.from) || byStr(a.to, b.to) || byStr(a.kind, b.kind));
|
|
248
|
+
return out;
|
|
249
|
+
}
|
package/src/scan.ts
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { basename } from "node:path";
|
|
2
|
+
import type { FileRecord } from "./types.js";
|
|
3
|
+
import { walk, readText } from "./walk.js";
|
|
4
|
+
import { headCommit } from "./git.js";
|
|
5
|
+
import { sha1 } from "./hash.js";
|
|
6
|
+
import { classify, MARKDOWN_EXT } from "./classify.js";
|
|
7
|
+
import { extToLang } from "./lang/registry.js";
|
|
8
|
+
import { compileGlobs } from "./glob.js";
|
|
9
|
+
import { byKey } from "./sort.js";
|
|
10
|
+
import { extractMarkdown } from "./extract/markdown.js";
|
|
11
|
+
import { extractCode } from "./extract/code.js";
|
|
12
|
+
|
|
13
|
+
export interface RepoScan {
|
|
14
|
+
root: string;
|
|
15
|
+
commit?: string;
|
|
16
|
+
files: FileRecord[];
|
|
17
|
+
languages: Record<string, number>;
|
|
18
|
+
// Raw content of doc files, kept so the graph's mention pass does not re-read
|
|
19
|
+
// them from disk (they were already read here). Docs only — bounding memory to
|
|
20
|
+
// prose, never the whole source tree.
|
|
21
|
+
docText: Map<string, string>;
|
|
22
|
+
// rel → last-modified ms for every kept file, so build.ts can persist the
|
|
23
|
+
// (size,mtime) fastpath key into cache.json for the next build.
|
|
24
|
+
mtimes: Map<string, number>;
|
|
25
|
+
capped: boolean; // the walk hit --max-files and the index is partial
|
|
26
|
+
// Files the walk saw and rejected (size/lockfile/binary/minified/gitignore
|
|
27
|
+
// rules — see WalkResult.excluded). Surfaced so a consumer can report how
|
|
28
|
+
// much of the tree was filtered out of the index.
|
|
29
|
+
excluded: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface ScanOptions {
|
|
33
|
+
include?: string[];
|
|
34
|
+
exclude?: string[];
|
|
35
|
+
// Sugar for include: ["<scope>/**"] — restrict the scan to one directory.
|
|
36
|
+
scope?: string;
|
|
37
|
+
// Honor .gitignore files (default true — see WalkOptions.gitignore).
|
|
38
|
+
gitignore?: boolean;
|
|
39
|
+
maxBytes?: number;
|
|
40
|
+
maxFiles?: number;
|
|
41
|
+
out?: string; // absolute output dir to exclude from the scan (self-index guard)
|
|
42
|
+
// Previous build's extraction cache (rel → {hash, record, size?, mtimeMs?}). A
|
|
43
|
+
// file whose (size,mtime) key matches skips read+hash entirely (the stat
|
|
44
|
+
// fastpath); one whose content hash is unchanged reuses its record and skips
|
|
45
|
+
// re-extraction.
|
|
46
|
+
cache?: Map<string, { hash: string; record: FileRecord; size?: number; mtimeMs?: number }>;
|
|
47
|
+
// Disable the stat fastpath: read and re-hash every file (see BuildOptions).
|
|
48
|
+
fullHash?: boolean;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function countLines(s: string): number {
|
|
52
|
+
if (!s) return 0;
|
|
53
|
+
let n = 1;
|
|
54
|
+
for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 10) n++;
|
|
55
|
+
return n;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Walk the repo once and turn every in-scope file into a FileRecord. Pure file
|
|
59
|
+
// I/O + deterministic extraction — never reads the repo into the model.
|
|
60
|
+
export function scanRepo(root: string, opts: ScanOptions = {}): RepoScan {
|
|
61
|
+
const scoped = opts.scope ? [...(opts.include ?? []), `${opts.scope.replace(/\/+$/, "")}/**`] : opts.include;
|
|
62
|
+
const include = compileGlobs(scoped);
|
|
63
|
+
const exclude = compileGlobs(opts.exclude);
|
|
64
|
+
const { files: walked, capped, excluded } = walk(root, {
|
|
65
|
+
maxFileBytes: opts.maxBytes,
|
|
66
|
+
maxFiles: opts.maxFiles,
|
|
67
|
+
gitignore: opts.gitignore,
|
|
68
|
+
});
|
|
69
|
+
// Never index our own output (e.g. a committed `docs/ultraindex/`), or builds
|
|
70
|
+
// would describe the encyclopedia instead of the code.
|
|
71
|
+
const outPrefix = opts.out ? opts.out.replace(/\/+$/, "") + "/" : null;
|
|
72
|
+
|
|
73
|
+
const files: FileRecord[] = [];
|
|
74
|
+
const languages: Record<string, number> = {};
|
|
75
|
+
const docText = new Map<string, string>();
|
|
76
|
+
const mtimes = new Map<string, number>();
|
|
77
|
+
|
|
78
|
+
for (const f of walked) {
|
|
79
|
+
if (outPrefix && (f.abs === opts.out || f.abs.startsWith(outPrefix))) continue;
|
|
80
|
+
if (include && !include(f.rel)) continue;
|
|
81
|
+
if (exclude && exclude(f.rel)) continue;
|
|
82
|
+
|
|
83
|
+
const kind = classify(f.rel, f.ext);
|
|
84
|
+
const lang = extToLang(f.ext);
|
|
85
|
+
languages[lang] = (languages[lang] ?? 0) + 1;
|
|
86
|
+
mtimes.set(f.rel, f.mtimeMs); // persist the fastpath key for the next build
|
|
87
|
+
|
|
88
|
+
const cached = opts.cache?.get(f.rel);
|
|
89
|
+
|
|
90
|
+
// Stat fastpath: a NON-DOC file whose size AND mtime both match its cache
|
|
91
|
+
// entry is treated as unchanged and reuses its record WITHOUT a read or hash.
|
|
92
|
+
// Docs are EXEMPT — the graph's mention pass needs their raw content (docText)
|
|
93
|
+
// every build, so a doc is always read regardless. --full-hash disables the
|
|
94
|
+
// fastpath. The (size,mtime) pair is the heuristic here (not the exact content
|
|
95
|
+
// hash below): a real editor bumps mtime on every save, and --full-hash /
|
|
96
|
+
// --no-cache are the escape hatches for the astronomically-unlikely edit that
|
|
97
|
+
// preserves both.
|
|
98
|
+
if (
|
|
99
|
+
kind !== "doc" &&
|
|
100
|
+
!opts.fullHash &&
|
|
101
|
+
cached &&
|
|
102
|
+
cached.size !== undefined &&
|
|
103
|
+
cached.mtimeMs !== undefined &&
|
|
104
|
+
cached.size === f.size &&
|
|
105
|
+
cached.mtimeMs === f.mtimeMs
|
|
106
|
+
) {
|
|
107
|
+
files.push(cached.record);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Read + hash (the staleness oracle stays exact); only EXTRACTION is cached. A
|
|
112
|
+
// hash hit reuses the previous record — content is byte-identical, so every
|
|
113
|
+
// derived field is too. classify()/extToLang() depend only on the path, so
|
|
114
|
+
// kind/lang are stable across the hit.
|
|
115
|
+
const content = readText(f.abs);
|
|
116
|
+
const hash = sha1(content);
|
|
117
|
+
if (cached && cached.hash === hash) {
|
|
118
|
+
files.push(cached.record);
|
|
119
|
+
if (kind === "doc" && content) docText.set(f.rel, content);
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const record: FileRecord = {
|
|
124
|
+
rel: f.rel,
|
|
125
|
+
ext: f.ext,
|
|
126
|
+
size: f.size,
|
|
127
|
+
lines: countLines(content),
|
|
128
|
+
hash,
|
|
129
|
+
kind,
|
|
130
|
+
lang,
|
|
131
|
+
headings: [],
|
|
132
|
+
symbols: [],
|
|
133
|
+
refs: [],
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
if (content) {
|
|
137
|
+
if (kind === "doc" && MARKDOWN_EXT.has(f.ext)) {
|
|
138
|
+
const md = extractMarkdown(content);
|
|
139
|
+
record.title = md.title ?? basename(f.rel);
|
|
140
|
+
record.summary = md.summary;
|
|
141
|
+
record.headings = md.headings;
|
|
142
|
+
record.refs = md.refs;
|
|
143
|
+
} else if (kind === "doc") {
|
|
144
|
+
// Non-markdown prose (.rst/.txt): title from basename, no link graph.
|
|
145
|
+
record.title = basename(f.rel);
|
|
146
|
+
} else if (kind === "code") {
|
|
147
|
+
const code = extractCode(f.rel, f.ext, content);
|
|
148
|
+
record.title = basename(f.rel);
|
|
149
|
+
record.summary = code.summary;
|
|
150
|
+
record.symbols = code.symbols;
|
|
151
|
+
record.refs = code.refs;
|
|
152
|
+
record.pkg = code.pkg;
|
|
153
|
+
record.idents = code.idents;
|
|
154
|
+
record.calls = code.calls;
|
|
155
|
+
record.importedNames = code.importedNames;
|
|
156
|
+
} else {
|
|
157
|
+
record.title = basename(f.rel);
|
|
158
|
+
}
|
|
159
|
+
} else {
|
|
160
|
+
record.title = basename(f.rel);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// Retain doc content for the graph's mention pass (docs only) so it is read
|
|
164
|
+
// once here, not a second time from disk.
|
|
165
|
+
if (kind === "doc" && content) docText.set(f.rel, content);
|
|
166
|
+
|
|
167
|
+
files.push(record);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
files.sort(byKey((f) => f.rel));
|
|
171
|
+
return { root, commit: headCommit(root), files, languages, docText, mtimes, capped, excluded };
|
|
172
|
+
}
|
package/src/sort.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// Locale-INDEPENDENT string comparator (UTF-16 code-unit order). Used for every
|
|
2
|
+
// sort that feeds the on-disk artifact, so two builds on different machines /
|
|
3
|
+
// locales produce byte-identical output. `localeCompare` must NOT be used for
|
|
4
|
+
// artifact ordering — its result varies with the host's ICU locale.
|
|
5
|
+
export function byStr(a: string, b: string): number {
|
|
6
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
// Sort by a string key, stably and locale-independently.
|
|
10
|
+
export function byKey<T>(keyOf: (x: T) => string): (a: T, b: T) => number {
|
|
11
|
+
return (a, b) => byStr(keyOf(a), keyOf(b));
|
|
12
|
+
}
|
package/src/surprise.ts
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { Graph, SurpriseEdge } from "./types.js";
|
|
2
|
+
import { byStr } from "./sort.js";
|
|
3
|
+
|
|
4
|
+
// Surprising cross-community coupling: a dependency edge that is one of at most
|
|
5
|
+
// two links between two otherwise-separate subsystems. These are the places
|
|
6
|
+
// where the architecture leaks — an agent reviewing a change touching one
|
|
7
|
+
// deserves a nudge. Deliberately conservative: doc-links/mentions crossing
|
|
8
|
+
// communities are normal, and everyone importing shared foundations (tier 0)
|
|
9
|
+
// across communities is expected.
|
|
10
|
+
export const SURPRISE_CAP = 24;
|
|
11
|
+
const MAX_PAIR_EDGES = 2;
|
|
12
|
+
|
|
13
|
+
const DEP_KINDS = new Set(["import", "call", "use"]);
|
|
14
|
+
|
|
15
|
+
// One O(E) pass over the module edges. Deterministic: candidates keep the
|
|
16
|
+
// stored edge order until the final (pairEdges asc, from, to) sort.
|
|
17
|
+
export function computeSurprises(graph: Graph): SurpriseEdge[] {
|
|
18
|
+
const commOf = new Map<string, number>();
|
|
19
|
+
const tierOf = new Map<string, number>();
|
|
20
|
+
for (const m of graph.modules) {
|
|
21
|
+
if (m.community !== undefined) commOf.set(m.slug, m.community);
|
|
22
|
+
tierOf.set(m.slug, m.tier);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const pairCount = new Map<string, number>();
|
|
26
|
+
const pairKey = (a: number, b: number): string => (a < b ? `${a}:${b}` : `${b}:${a}`);
|
|
27
|
+
const candidates: { edge: Graph["moduleEdges"][number]; comms: [number, number] }[] = [];
|
|
28
|
+
for (const e of graph.moduleEdges) {
|
|
29
|
+
if (e.dangling) continue;
|
|
30
|
+
const ca = commOf.get(e.from);
|
|
31
|
+
const cb = commOf.get(e.to);
|
|
32
|
+
if (ca === undefined || cb === undefined || ca === cb) continue;
|
|
33
|
+
pairCount.set(pairKey(ca, cb), (pairCount.get(pairKey(ca, cb)) ?? 0) + 1);
|
|
34
|
+
if (!DEP_KINDS.has(e.kind)) continue;
|
|
35
|
+
if (tierOf.get(e.to) === 0) continue;
|
|
36
|
+
candidates.push({ edge: e, comms: [ca, cb] });
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return candidates
|
|
40
|
+
.filter((c) => pairCount.get(pairKey(c.comms[0], c.comms[1]))! <= MAX_PAIR_EDGES)
|
|
41
|
+
.map((c) => ({
|
|
42
|
+
from: c.edge.from,
|
|
43
|
+
to: c.edge.to,
|
|
44
|
+
kind: c.edge.kind,
|
|
45
|
+
weight: c.edge.weight,
|
|
46
|
+
communities: c.comms,
|
|
47
|
+
pairEdges: pairCount.get(pairKey(c.comms[0], c.comms[1]))!,
|
|
48
|
+
}))
|
|
49
|
+
.sort((a, b) => a.pairEdges - b.pairEdges || byStr(a.from, b.from) || byStr(a.to, b.to))
|
|
50
|
+
.slice(0, SURPRISE_CAP);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Is the directed module edge from→to flagged surprising? Reads the stored
|
|
54
|
+
// build-time list when present, recomputes otherwise.
|
|
55
|
+
export function isSurprising(graph: Graph, from: string, to: string): boolean {
|
|
56
|
+
const list = graph.surprises ?? computeSurprises(graph);
|
|
57
|
+
return list.some((s) => s.from === from && s.to === to);
|
|
58
|
+
}
|
package/src/tests-map.ts
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import type { Graph, ModuleNode } from "./types.js";
|
|
2
|
+
import { isTestFile } from "./modules.js";
|
|
3
|
+
import { byStr } from "./sort.js";
|
|
4
|
+
|
|
5
|
+
// tests→code mapping, derived from the existing graph — NO new edge kind. The
|
|
6
|
+
// evidence is already there: a test file with a resolved import/call/use edge
|
|
7
|
+
// into a source file covers it (the same "depends on" kind set `impact` walks).
|
|
8
|
+
// Classifying test FILES and projecting the dependency edges through that
|
|
9
|
+
// classification keeps degree, Louvain, and the kind ranking untouched.
|
|
10
|
+
|
|
11
|
+
// Test-file detection by basename, per language convention. JS/TS reuses the
|
|
12
|
+
// tier logic's TEST_FILE regex via isTestFile; the rest are the conventional
|
|
13
|
+
// names: test_*.py / *_test.py, *_test.go, *Test(s).java|kt / *IT.java,
|
|
14
|
+
// *_spec.rb / *_test.rb, *Test.php, *Test(s).cs, *_test.exs.
|
|
15
|
+
const BASENAME_PATTERNS = [
|
|
16
|
+
/^test_.*\.py$/i,
|
|
17
|
+
/_test\.py$/i,
|
|
18
|
+
/_test\.go$/,
|
|
19
|
+
/(Test|Tests|IT)\.java$/,
|
|
20
|
+
/(Test|Tests)\.kt$/,
|
|
21
|
+
/_spec\.rb$/,
|
|
22
|
+
/_test\.rb$/,
|
|
23
|
+
/Test\.php$/,
|
|
24
|
+
/(Test|Tests)\.cs$/,
|
|
25
|
+
/_test\.exs$/,
|
|
26
|
+
];
|
|
27
|
+
|
|
28
|
+
// Directory rule: anything under a dedicated test dir is test material. This is
|
|
29
|
+
// deliberately NARROWER than the tier logic's TIER2_ANY — examples, docs,
|
|
30
|
+
// fixtures and benchmarks are tail, but they are not tests.
|
|
31
|
+
const TEST_DIR = /(^|\/)(tests?|__tests?__|spec|specs|e2e)(\/|$)/i;
|
|
32
|
+
|
|
33
|
+
// Is this repo-relative path a test file? Callers filter to code files; this
|
|
34
|
+
// only judges the path.
|
|
35
|
+
export function isTestPath(rel: string): boolean {
|
|
36
|
+
if (TEST_DIR.test(rel)) return true;
|
|
37
|
+
if (isTestFile(rel)) return true;
|
|
38
|
+
const base = rel.split("/").pop()!;
|
|
39
|
+
return BASENAME_PATTERNS.some((p) => p.test(base));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface TestMap {
|
|
43
|
+
testFiles: Set<string>; // rels of code files classified as tests
|
|
44
|
+
testedByFile: Map<string, string[]>; // source rel → sorted covering test rels
|
|
45
|
+
testedByModule: Map<string, string[]>; // module slug → sorted covering test rels
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Project the graph's dependency edges through the test classification. A
|
|
49
|
+
// test→test edge (helpers, shared setup) is not coverage; neither is a doc-link
|
|
50
|
+
// or mention, nor anything dangling. O(F + E), fully deterministic.
|
|
51
|
+
export function computeTestMap(graph: Graph): TestMap {
|
|
52
|
+
const testFiles = new Set<string>();
|
|
53
|
+
const moduleOf = new Map<string, string>();
|
|
54
|
+
for (const f of graph.files) {
|
|
55
|
+
moduleOf.set(f.rel, f.module);
|
|
56
|
+
if (f.fileKind === "code" && isTestPath(f.rel)) testFiles.add(f.rel);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const byFile = new Map<string, Set<string>>();
|
|
60
|
+
const byModule = new Map<string, Set<string>>();
|
|
61
|
+
for (const e of graph.fileEdges) {
|
|
62
|
+
if (e.dangling) continue;
|
|
63
|
+
if (e.kind !== "import" && e.kind !== "use" && e.kind !== "call") continue;
|
|
64
|
+
if (!testFiles.has(e.from) || testFiles.has(e.to)) continue;
|
|
65
|
+
let set = byFile.get(e.to);
|
|
66
|
+
if (!set) byFile.set(e.to, (set = new Set()));
|
|
67
|
+
set.add(e.from);
|
|
68
|
+
const slug = moduleOf.get(e.to);
|
|
69
|
+
if (slug !== undefined) {
|
|
70
|
+
let mset = byModule.get(slug);
|
|
71
|
+
if (!mset) byModule.set(slug, (mset = new Set()));
|
|
72
|
+
mset.add(e.from);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const sortSets = (m: Map<string, Set<string>>): Map<string, string[]> => {
|
|
77
|
+
const out = new Map<string, string[]>();
|
|
78
|
+
for (const key of [...m.keys()].sort(byStr)) out.set(key, [...m.get(key)!].sort(byStr));
|
|
79
|
+
return out;
|
|
80
|
+
};
|
|
81
|
+
return { testFiles, testedByFile: sortSets(byFile), testedByModule: sortSets(byModule) };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Covering tests for one module: the stored build-time field when present,
|
|
85
|
+
// recomputed from the graph otherwise (older graphs, hand-built test literals).
|
|
86
|
+
export function testsForModule(graph: Graph, slug: string): string[] {
|
|
87
|
+
const m = graph.modules.find((x) => x.slug === slug);
|
|
88
|
+
if (m?.testedBy) return m.testedBy;
|
|
89
|
+
return computeTestMap(graph).testedByModule.get(slug) ?? [];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// Modules that SHOULD have tests but don't: tier ≤ 1, at least one non-test
|
|
93
|
+
// code member, declared symbols, and no covering test. Doc-only and tail
|
|
94
|
+
// modules are out of scope by construction.
|
|
95
|
+
export function untestedModules(graph: Graph): ModuleNode[] {
|
|
96
|
+
const tm = computeTestMap(graph);
|
|
97
|
+
const codeMembers = new Map<string, number>();
|
|
98
|
+
for (const f of graph.files) {
|
|
99
|
+
if (f.fileKind !== "code" || tm.testFiles.has(f.rel)) continue;
|
|
100
|
+
codeMembers.set(f.module, (codeMembers.get(f.module) ?? 0) + 1);
|
|
101
|
+
}
|
|
102
|
+
return graph.modules.filter(
|
|
103
|
+
(m) => m.tier <= 1 && m.symbols > 0 && (codeMembers.get(m.slug) ?? 0) > 0 && !tm.testedByModule.has(m.slug),
|
|
104
|
+
);
|
|
105
|
+
}
|