@polycode-projects/seonix 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +77 -0
- package/bin/cli.mjs +248 -0
- package/package.json +54 -0
- package/roslyn/Program.cs +150 -0
- package/roslyn/RoslynExtract.csproj +13 -0
- package/src/codegraph.mjs +1313 -0
- package/src/config.mjs +28 -0
- package/src/cs_roslyn.mjs +38 -0
- package/src/cs_treesitter.mjs +140 -0
- package/src/extract.mjs +624 -0
- package/src/extract_ast.py +366 -0
- package/src/extract_lang.mjs +61 -0
- package/src/jsts_tsc.mjs +211 -0
- package/src/server.mjs +389 -0
- package/src/source.mjs +35 -0
- package/src/viz.mjs +218 -0
- package/src/walk.mjs +39 -0
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Deterministic, offline static extraction for seonix (Python).
|
|
3
|
+
|
|
4
|
+
Stdlib `ast` only — no third-party parser, no model calls. Walks every *.py file
|
|
5
|
+
under repo_path and emits ONE JSON document on stdout:
|
|
6
|
+
|
|
7
|
+
{"modules": [
|
|
8
|
+
{"path": "<repo-relative>", "dotted": "django.utils.text",
|
|
9
|
+
"imports": ["django.utils.functional", ...], # candidate dotted targets
|
|
10
|
+
"defines": [{"name": "slugify", "kind": "function", "lineno": 12,
|
|
11
|
+
"decorators": ["register.filter(is_safe=True)"]},
|
|
12
|
+
{"name": "Truncator", "kind": "class", "bases": ["object"], ...},
|
|
13
|
+
{"name": "Truncator.chars", "kind": "method", ...},
|
|
14
|
+
{"name": "Truncator.text", "kind": "attribute", ...}, ...],
|
|
15
|
+
"calls": ["str.strip", "re.sub", ...]}, # coarse callee names
|
|
16
|
+
...
|
|
17
|
+
]}
|
|
18
|
+
|
|
19
|
+
Resolution of import candidates and call targets to internal modules happens in
|
|
20
|
+
extract.mjs against the registry of discovered modules — this script stays a pure
|
|
21
|
+
per-file parser. Run: python3 extract_ast.py <repo_path>.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import ast
|
|
25
|
+
import json
|
|
26
|
+
import os
|
|
27
|
+
import sys
|
|
28
|
+
|
|
29
|
+
SKIP_DIRS = {".git", ".seonix", ".hg", ".svn", "node_modules", ".venv", "venv",
|
|
30
|
+
"__pycache__", ".tox", ".mypy_cache", ".pytest_cache", "build", "dist"}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def dotted_for(rel_path):
|
|
34
|
+
"""Repo-relative .py path -> dotted module + its package."""
|
|
35
|
+
parts = rel_path[:-3].split(os.sep) # drop ".py"
|
|
36
|
+
if parts and parts[-1] == "__init__":
|
|
37
|
+
parts = parts[:-1]
|
|
38
|
+
pkg = ".".join(parts)
|
|
39
|
+
return pkg, pkg # a package: dotted == its own package
|
|
40
|
+
dotted = ".".join(parts)
|
|
41
|
+
pkg = ".".join(parts[:-1])
|
|
42
|
+
return dotted, pkg
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def import_targets(node, pkg):
|
|
46
|
+
"""Candidate dotted module targets for one Import/ImportFrom node."""
|
|
47
|
+
out = []
|
|
48
|
+
if isinstance(node, ast.Import):
|
|
49
|
+
for alias in node.names:
|
|
50
|
+
out.append(alias.name) # import a.b.c -> "a.b.c"
|
|
51
|
+
elif isinstance(node, ast.ImportFrom):
|
|
52
|
+
if node.level: # relative: from . / from .mod import x
|
|
53
|
+
base_parts = pkg.split(".") if pkg else []
|
|
54
|
+
base_parts = base_parts[: len(base_parts) - (node.level - 1)] if node.level > 1 else base_parts
|
|
55
|
+
base = ".".join(base_parts)
|
|
56
|
+
mod = f"{base}.{node.module}" if node.module else base
|
|
57
|
+
else:
|
|
58
|
+
mod = node.module or ""
|
|
59
|
+
if mod:
|
|
60
|
+
out.append(mod)
|
|
61
|
+
# `from a.b import c` may import submodule a.b.c — record as a candidate too.
|
|
62
|
+
for alias in node.names:
|
|
63
|
+
if alias.name and alias.name != "*":
|
|
64
|
+
out.append(f"{mod}.{alias.name}")
|
|
65
|
+
return out
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def decorator_str(dec):
|
|
69
|
+
try:
|
|
70
|
+
return ast.unparse(dec)
|
|
71
|
+
except Exception:
|
|
72
|
+
return ""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def names_in_target(tgt):
|
|
76
|
+
"""Plain-name assignment targets (Name, or Name elements of a Tuple/List)."""
|
|
77
|
+
if isinstance(tgt, ast.Name):
|
|
78
|
+
return [tgt.id]
|
|
79
|
+
if isinstance(tgt, (ast.Tuple, ast.List)):
|
|
80
|
+
out = []
|
|
81
|
+
for el in tgt.elts:
|
|
82
|
+
out.extend(names_in_target(el))
|
|
83
|
+
return out
|
|
84
|
+
return []
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# --- mechanical enrichments (deterministic ast facts; no type inference) ----------
|
|
88
|
+
# Per PLAN_SEONIX_TUNING.md §5: params, return annotation, raises/catches, self-field
|
|
89
|
+
# access, static/abstract/visibility flags, and the first docstring line. All are
|
|
90
|
+
# free from `ast` alone; everything here stays honest (e.g. raises are the literal
|
|
91
|
+
# `raise` targets, not a resolved type).
|
|
92
|
+
|
|
93
|
+
def _unparse(node):
|
|
94
|
+
try:
|
|
95
|
+
return ast.unparse(node)
|
|
96
|
+
except Exception:
|
|
97
|
+
return ""
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _exc_name(node):
|
|
101
|
+
"""The named type of a raised/caught exception expression (drop the call args)."""
|
|
102
|
+
if isinstance(node, ast.Call):
|
|
103
|
+
node = node.func
|
|
104
|
+
return _unparse(node)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def raised_excs(fn):
|
|
108
|
+
out = []
|
|
109
|
+
for n in ast.walk(fn):
|
|
110
|
+
if isinstance(n, ast.Raise) and n.exc is not None:
|
|
111
|
+
nm = _exc_name(n.exc)
|
|
112
|
+
if nm:
|
|
113
|
+
out.append(nm)
|
|
114
|
+
return sorted(set(out))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def caught_excs(fn):
|
|
118
|
+
out = []
|
|
119
|
+
for n in ast.walk(fn):
|
|
120
|
+
if isinstance(n, ast.ExceptHandler) and n.type is not None:
|
|
121
|
+
t = n.type
|
|
122
|
+
elts = t.elts if isinstance(t, (ast.Tuple, ast.List)) else [t]
|
|
123
|
+
for el in elts:
|
|
124
|
+
nm = _exc_name(el)
|
|
125
|
+
if nm:
|
|
126
|
+
out.append(nm)
|
|
127
|
+
return sorted(set(out))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def self_field_names(fn):
|
|
131
|
+
"""`self.x` attribute names touched in a method body (read or write)."""
|
|
132
|
+
out = set()
|
|
133
|
+
for n in ast.walk(fn):
|
|
134
|
+
if isinstance(n, ast.Attribute) and isinstance(n.value, ast.Name) and n.value.id == "self":
|
|
135
|
+
out.add(n.attr)
|
|
136
|
+
return sorted(out)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def calls_in(fn):
|
|
140
|
+
"""Coarse callee names invoked WITHIN one function/method body (for the
|
|
141
|
+
symbol-granular call graph). Names are the unparsed call target (e.g.
|
|
142
|
+
'helper', 'self.foo', 're.sub'); resolution to a single in-repo symbol id —
|
|
143
|
+
and the unique-name discipline — happens in extract.mjs. Reuses the existing
|
|
144
|
+
per-function ast walk (no extra parse pass)."""
|
|
145
|
+
out = set()
|
|
146
|
+
for n in ast.walk(fn):
|
|
147
|
+
if isinstance(n, ast.Call):
|
|
148
|
+
try:
|
|
149
|
+
nm = ast.unparse(n.func)
|
|
150
|
+
except Exception:
|
|
151
|
+
nm = ""
|
|
152
|
+
if nm:
|
|
153
|
+
out.add(nm)
|
|
154
|
+
return sorted(out)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def first_doc_line(node):
|
|
158
|
+
try:
|
|
159
|
+
d = ast.get_docstring(node, clean=True)
|
|
160
|
+
except Exception:
|
|
161
|
+
d = None
|
|
162
|
+
if not d:
|
|
163
|
+
return ""
|
|
164
|
+
return d.strip().split("\n", 1)[0].strip()[:120]
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def visibility_of(name):
|
|
168
|
+
short = name.rsplit(".", 1)[-1]
|
|
169
|
+
if short.startswith("__") and not short.endswith("__"):
|
|
170
|
+
return "private"
|
|
171
|
+
if short.startswith("_") and not short.startswith("__"):
|
|
172
|
+
return "protected"
|
|
173
|
+
return "" # public is the default — omitted to keep the graph lean
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def func_extras(node, name, decorators, is_method):
|
|
177
|
+
"""Compact, only-when-present enrichment dict for a function/method define."""
|
|
178
|
+
extras = {}
|
|
179
|
+
sig = _unparse(node.args)
|
|
180
|
+
if sig:
|
|
181
|
+
extras["params"] = sig[:160]
|
|
182
|
+
if getattr(node, "returns", None) is not None:
|
|
183
|
+
r = _unparse(node.returns)
|
|
184
|
+
if r:
|
|
185
|
+
extras["returns"] = r[:80]
|
|
186
|
+
raises = raised_excs(node)
|
|
187
|
+
if raises:
|
|
188
|
+
extras["raises"] = raises[:12]
|
|
189
|
+
catches = caught_excs(node)
|
|
190
|
+
if catches:
|
|
191
|
+
extras["catches"] = catches[:12]
|
|
192
|
+
if is_method:
|
|
193
|
+
fields = self_field_names(node)
|
|
194
|
+
if fields:
|
|
195
|
+
extras["self_fields"] = fields[:24]
|
|
196
|
+
# per-function callee names — the raw material for the symbol-granular
|
|
197
|
+
# mgx:callsSymbol edge (caller fn → callee fn). Collected here so the subject
|
|
198
|
+
# (the enclosing def) is known; extract.mjs resolves names to symbol ids.
|
|
199
|
+
callees = calls_in(node)
|
|
200
|
+
if callees:
|
|
201
|
+
extras["calls"] = callees[:50]
|
|
202
|
+
decset = " ".join(decorators)
|
|
203
|
+
if "staticmethod" in decset or "classmethod" in decset:
|
|
204
|
+
extras["is_static"] = True
|
|
205
|
+
if "abstractmethod" in decset or "abstractproperty" in decset:
|
|
206
|
+
extras["is_abstract"] = True
|
|
207
|
+
vis = visibility_of(name)
|
|
208
|
+
if vis:
|
|
209
|
+
extras["visibility"] = vis
|
|
210
|
+
doc = first_doc_line(node)
|
|
211
|
+
if doc:
|
|
212
|
+
extras["doc"] = doc
|
|
213
|
+
return extras
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def parse_module(src, rel_path):
|
|
217
|
+
dotted, pkg = dotted_for(rel_path)
|
|
218
|
+
try:
|
|
219
|
+
tree = ast.parse(src, filename=rel_path)
|
|
220
|
+
except (SyntaxError, ValueError):
|
|
221
|
+
return None # skip unparseable files (py2 fixtures, templates, etc.)
|
|
222
|
+
|
|
223
|
+
imports = []
|
|
224
|
+
defines = []
|
|
225
|
+
calls = set()
|
|
226
|
+
globals_seen = set()
|
|
227
|
+
exports = [] # names in a literal __all__ (the module's declared public surface)
|
|
228
|
+
|
|
229
|
+
def string_list(v):
|
|
230
|
+
# names from a literal list/tuple of string constants (else [])
|
|
231
|
+
if not isinstance(v, (ast.List, ast.Tuple)):
|
|
232
|
+
return []
|
|
233
|
+
out = []
|
|
234
|
+
for el in v.elts:
|
|
235
|
+
if isinstance(el, ast.Constant) and isinstance(el.value, str):
|
|
236
|
+
out.append(el.value)
|
|
237
|
+
elif isinstance(el, ast.Str): # py<3.8 fallback
|
|
238
|
+
out.append(el.s)
|
|
239
|
+
return out
|
|
240
|
+
|
|
241
|
+
def end_of(n):
|
|
242
|
+
return getattr(n, "end_lineno", None) or n.lineno
|
|
243
|
+
|
|
244
|
+
def short_value(v):
|
|
245
|
+
try:
|
|
246
|
+
s = ast.unparse(v)
|
|
247
|
+
except Exception:
|
|
248
|
+
return ""
|
|
249
|
+
return s.replace("\n", " ")[:80]
|
|
250
|
+
|
|
251
|
+
def add_global(name, target_node, value_node):
|
|
252
|
+
# Module-level "live object" globals (RHS is a call, e.g. register =
|
|
253
|
+
# template.Library()) and ALL-CAPS constants — the registration anchors and
|
|
254
|
+
# config values a sibling-adding task must replicate. Skip noisy locals.
|
|
255
|
+
if name in globals_seen or value_node is None:
|
|
256
|
+
return
|
|
257
|
+
is_call = isinstance(value_node, ast.Call)
|
|
258
|
+
if not (is_call or name.isupper()):
|
|
259
|
+
return
|
|
260
|
+
globals_seen.add(name)
|
|
261
|
+
rec = {"name": name, "kind": "global", "lineno": target_node.lineno,
|
|
262
|
+
"end_lineno": end_of(target_node), "decorators": [],
|
|
263
|
+
"value": short_value(value_node)}
|
|
264
|
+
if name.isupper():
|
|
265
|
+
rec["is_constant"] = True
|
|
266
|
+
vis = visibility_of(name)
|
|
267
|
+
if vis:
|
|
268
|
+
rec["visibility"] = vis
|
|
269
|
+
defines.append(rec)
|
|
270
|
+
|
|
271
|
+
# Top-level defs/classes, plus one level of class methods (e.g. Truncator.chars).
|
|
272
|
+
for node in tree.body:
|
|
273
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
274
|
+
decs = [decorator_str(d) for d in node.decorator_list]
|
|
275
|
+
defines.append({"name": node.name, "kind": "function", "lineno": node.lineno,
|
|
276
|
+
"end_lineno": end_of(node), "decorators": decs,
|
|
277
|
+
**func_extras(node, node.name, decs, is_method=False)})
|
|
278
|
+
elif isinstance(node, ast.ClassDef):
|
|
279
|
+
cdoc = first_doc_line(node)
|
|
280
|
+
cvis = visibility_of(node.name)
|
|
281
|
+
cls_extra = {}
|
|
282
|
+
if cdoc:
|
|
283
|
+
cls_extra["doc"] = cdoc
|
|
284
|
+
if cvis:
|
|
285
|
+
cls_extra["visibility"] = cvis
|
|
286
|
+
defines.append({"name": node.name, "kind": "class", "lineno": node.lineno,
|
|
287
|
+
"end_lineno": end_of(node),
|
|
288
|
+
"bases": [decorator_str(b) for b in node.bases],
|
|
289
|
+
"decorators": [decorator_str(d) for d in node.decorator_list],
|
|
290
|
+
**cls_extra})
|
|
291
|
+
seen_attrs = set()
|
|
292
|
+
for sub in node.body:
|
|
293
|
+
if isinstance(sub, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
294
|
+
mdecs = [decorator_str(d) for d in sub.decorator_list]
|
|
295
|
+
defines.append({"name": f"{node.name}.{sub.name}", "kind": "method",
|
|
296
|
+
"lineno": sub.lineno, "end_lineno": end_of(sub),
|
|
297
|
+
"decorators": mdecs,
|
|
298
|
+
**func_extras(sub, f"{node.name}.{sub.name}", mdecs, is_method=True)})
|
|
299
|
+
elif isinstance(sub, ast.AnnAssign) and isinstance(sub.target, ast.Name):
|
|
300
|
+
if sub.target.id not in seen_attrs:
|
|
301
|
+
seen_attrs.add(sub.target.id)
|
|
302
|
+
defines.append({"name": f"{node.name}.{sub.target.id}", "kind": "attribute",
|
|
303
|
+
"lineno": sub.lineno, "end_lineno": end_of(sub), "decorators": []})
|
|
304
|
+
elif isinstance(sub, ast.Assign):
|
|
305
|
+
for tgt in sub.targets:
|
|
306
|
+
for nm in names_in_target(tgt):
|
|
307
|
+
if nm in seen_attrs:
|
|
308
|
+
continue
|
|
309
|
+
seen_attrs.add(nm)
|
|
310
|
+
defines.append({"name": f"{node.name}.{nm}", "kind": "attribute",
|
|
311
|
+
"lineno": sub.lineno, "end_lineno": end_of(sub), "decorators": []})
|
|
312
|
+
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
|
|
313
|
+
if node.target.id == "__all__" and node.value is not None:
|
|
314
|
+
exports = string_list(node.value)
|
|
315
|
+
else:
|
|
316
|
+
add_global(node.target.id, node, node.value)
|
|
317
|
+
elif isinstance(node, ast.Assign):
|
|
318
|
+
if any(isinstance(t, ast.Name) and t.id == "__all__" for t in node.targets):
|
|
319
|
+
exports = string_list(node.value)
|
|
320
|
+
else:
|
|
321
|
+
for tgt in node.targets:
|
|
322
|
+
for nm in names_in_target(tgt):
|
|
323
|
+
add_global(nm, node, node.value)
|
|
324
|
+
|
|
325
|
+
for node in ast.walk(tree):
|
|
326
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
327
|
+
imports.extend(import_targets(node, pkg))
|
|
328
|
+
elif isinstance(node, ast.Call):
|
|
329
|
+
try:
|
|
330
|
+
name = ast.unparse(node.func)
|
|
331
|
+
except Exception:
|
|
332
|
+
name = ""
|
|
333
|
+
if name:
|
|
334
|
+
calls.add(name)
|
|
335
|
+
|
|
336
|
+
return {"path": rel_path.replace(os.sep, "/"), "dotted": dotted,
|
|
337
|
+
"imports": sorted(set(imports)), "defines": defines, "calls": sorted(calls),
|
|
338
|
+
"exports": exports}
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def main():
|
|
342
|
+
if len(sys.argv) < 2:
|
|
343
|
+
sys.stderr.write("usage: extract_ast.py <repo_path>\n")
|
|
344
|
+
sys.exit(2)
|
|
345
|
+
root = os.path.abspath(sys.argv[1])
|
|
346
|
+
modules = []
|
|
347
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
348
|
+
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS and not d.startswith(".")]
|
|
349
|
+
for fn in filenames:
|
|
350
|
+
if not fn.endswith(".py"):
|
|
351
|
+
continue
|
|
352
|
+
abs_path = os.path.join(dirpath, fn)
|
|
353
|
+
rel_path = os.path.relpath(abs_path, root)
|
|
354
|
+
try:
|
|
355
|
+
with open(abs_path, "r", encoding="utf-8") as fh:
|
|
356
|
+
src = fh.read()
|
|
357
|
+
except (OSError, UnicodeDecodeError):
|
|
358
|
+
continue
|
|
359
|
+
mod = parse_module(src, rel_path)
|
|
360
|
+
if mod:
|
|
361
|
+
modules.append(mod)
|
|
362
|
+
json.dump({"modules": modules}, sys.stdout)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
if __name__ == "__main__":
|
|
366
|
+
main()
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// Multi-language front-end for the seonix index. Detects which non-Python
|
|
2
|
+
// languages are present under <root>, runs the PICKED extractor per language
|
|
3
|
+
// (each emitting the SAME `{modules:[{path,dotted,imports,defines,calls,exports}]}`
|
|
4
|
+
// contract extract_ast.py prints), and merges them. The product's extract.mjs owns
|
|
5
|
+
// buildEntities + git history; this module is parse-only and returns RAW modules.
|
|
6
|
+
//
|
|
7
|
+
// Offline, deterministic, ZERO-LLM (static parsing only). No source content leaves
|
|
8
|
+
// the machine. Python stays the responsibility of extract_ast.py (extract.mjs);
|
|
9
|
+
// this covers JS/TS (ts-morph) and C# (Roslyn, with a tree-sitter fallback).
|
|
10
|
+
import * as jstsTsc from "./jsts_tsc.mjs";
|
|
11
|
+
import * as csRoslyn from "./cs_roslyn.mjs";
|
|
12
|
+
import * as csTree from "./cs_treesitter.mjs";
|
|
13
|
+
|
|
14
|
+
// THE EXTRACTOR REGISTRY — the single place a new language plugs in. Keyed by
|
|
15
|
+
// language; `exts` is the file-extension surface it owns, `pick` the default
|
|
16
|
+
// extractor, `alt` the fallback. Everything downstream is language-agnostic.
|
|
17
|
+
export const REGISTRY = {
|
|
18
|
+
"js/ts": { exts: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"], pick: jstsTsc, alt: null },
|
|
19
|
+
"c#": { exts: [".cs"], pick: csRoslyn, alt: csTree },
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
/** The full set of file extensions this front-end indexes (sorted, lower-cased).
|
|
23
|
+
* extract.mjs unions this with `.py` to gate its git-log file filters. */
|
|
24
|
+
export const LANG_EXTS = [...new Set(Object.values(REGISTRY).flatMap((r) => r.exts))].sort();
|
|
25
|
+
|
|
26
|
+
/** Choose the C# extractor: Roslyn when its built tool is present, else tree-sitter. */
|
|
27
|
+
async function chooseCs(prefer) {
|
|
28
|
+
if (prefer === "treesitter") return csTree;
|
|
29
|
+
if (prefer === "roslyn" || prefer === undefined) {
|
|
30
|
+
if (await csRoslyn.toolAvailable()) return csRoslyn;
|
|
31
|
+
}
|
|
32
|
+
return csTree;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Parse every supported non-Python language under `root` and merge the results.
|
|
37
|
+
* @returns {Promise<{modules:Array, perLang:Object, totalFiles:number, failures:Array}>}
|
|
38
|
+
*/
|
|
39
|
+
export async function ingestRepo(root, { cs } = {}) {
|
|
40
|
+
const csExtractor = await chooseCs(cs);
|
|
41
|
+
const plan = [
|
|
42
|
+
{ lang: "js/ts", extractor: jstsTsc },
|
|
43
|
+
{ lang: "c#", extractor: csExtractor },
|
|
44
|
+
];
|
|
45
|
+
const allModules = [];
|
|
46
|
+
const perLang = {};
|
|
47
|
+
let totalFiles = 0;
|
|
48
|
+
const allFailures = [];
|
|
49
|
+
for (const { lang, extractor } of plan) {
|
|
50
|
+
const t0 = Date.now();
|
|
51
|
+
const { modules, failures, fileCount } = await extractor.ingest(root);
|
|
52
|
+
const ms = Date.now() - t0;
|
|
53
|
+
if (fileCount === 0) continue; // language not present
|
|
54
|
+
allModules.push(...modules);
|
|
55
|
+
totalFiles += fileCount;
|
|
56
|
+
allFailures.push(...failures);
|
|
57
|
+
const symbols = modules.reduce((n, m) => n + (m.defines?.length || 0), 0);
|
|
58
|
+
perLang[lang] = { lib: extractor.meta.lib, files: fileCount, modules: modules.length, symbols, failures: failures.length, ms };
|
|
59
|
+
}
|
|
60
|
+
return { modules: allModules, perLang, totalFiles, failures: allFailures };
|
|
61
|
+
}
|
package/src/jsts_tsc.mjs
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
// JS/TS extractor — TypeScript compiler API (via ts-morph's re-exported `ts`
|
|
2
|
+
// namespace; no separate `typescript` dep). PICKED candidate for JS/TS.
|
|
3
|
+
//
|
|
4
|
+
// Uses ts.createSourceFile per file (no Program / no type-checker) — the fast,
|
|
5
|
+
// deterministic, offline structural pass. Covers .ts/.tsx/.js/.jsx (allowJs is
|
|
6
|
+
// implicit: JS is just parsed with the JS script-kind). Emits the SAME
|
|
7
|
+
// `{path,dotted,imports,defines,calls,exports}` contract extract_ast.py prints,
|
|
8
|
+
// so extract.mjs/buildEntities + codegraph.mjs/digest consume it unchanged.
|
|
9
|
+
//
|
|
10
|
+
// Fidelity note (honest): params/returns are ANNOTATION strings (Group-A
|
|
11
|
+
// mechanical), exactly like the Python path's "returns = annotation only". A
|
|
12
|
+
// type-RESOLVED pass (real Program + checker → resolved call targets/types,
|
|
13
|
+
// promoting Group-B→A) is the documented ceiling, NOT run here.
|
|
14
|
+
import { readFile } from "node:fs/promises";
|
|
15
|
+
import { dirname, join } from "node:path";
|
|
16
|
+
import { ts } from "ts-morph";
|
|
17
|
+
import { walk, relPath } from "./walk.mjs";
|
|
18
|
+
|
|
19
|
+
const EXTS = [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"];
|
|
20
|
+
const stripExt = (p) => p.replace(/\.(tsx?|jsx?|mjs|cjs)$/i, "");
|
|
21
|
+
|
|
22
|
+
function scriptKind(path) {
|
|
23
|
+
if (/\.tsx$/i.test(path)) return ts.ScriptKind.TSX;
|
|
24
|
+
if (/\.ts$/i.test(path)) return ts.ScriptKind.TS;
|
|
25
|
+
if (/\.jsx$/i.test(path)) return ts.ScriptKind.JSX;
|
|
26
|
+
return ts.ScriptKind.JS;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const lineOf = (sf, node) => sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1;
|
|
30
|
+
const endLineOf = (sf, node) => sf.getLineAndCharacterOfPosition(node.getEnd()).line + 1;
|
|
31
|
+
|
|
32
|
+
/** "(a: string, b = 3)" → "a: string, b = 3" (annotation strings; not resolved). */
|
|
33
|
+
function paramsText(sf, node) {
|
|
34
|
+
const ps = node.parameters || [];
|
|
35
|
+
return ps.map((p) => p.getText(sf).replace(/\s+/g, " ").trim()).join(", ").slice(0, 160);
|
|
36
|
+
}
|
|
37
|
+
function returnText(sf, node) {
|
|
38
|
+
return node.type ? node.type.getText(sf).replace(/\s+/g, " ").trim().slice(0, 80) : "";
|
|
39
|
+
}
|
|
40
|
+
function firstDocLine(sf, node) {
|
|
41
|
+
const ranges = ts.getLeadingCommentRanges(sf.text, node.getFullStart()) || [];
|
|
42
|
+
for (const r of ranges) {
|
|
43
|
+
const raw = sf.text.slice(r.pos, r.end);
|
|
44
|
+
const m = raw.replace(/^\/\*\*?|\*\/$/g, "").split("\n").map((l) => l.replace(/^\s*\*?\s?/, "").trim()).find(Boolean);
|
|
45
|
+
if (m) return m.slice(0, 120);
|
|
46
|
+
}
|
|
47
|
+
return "";
|
|
48
|
+
}
|
|
49
|
+
const hasMod = (node, kind) => (node.modifiers || []).some((m) => m.kind === kind);
|
|
50
|
+
function visibilityOf(node) {
|
|
51
|
+
if (hasMod(node, ts.SyntaxKind.PrivateKeyword)) return "private";
|
|
52
|
+
if (hasMod(node, ts.SyntaxKind.ProtectedKeyword)) return "protected";
|
|
53
|
+
return "";
|
|
54
|
+
}
|
|
55
|
+
function decoratorsOf(sf, node) {
|
|
56
|
+
const ds = ts.canHaveDecorators?.(node) ? ts.getDecorators?.(node) : null;
|
|
57
|
+
return (ds || []).map((d) => d.expression.getText(sf).replace(/\s+/g, " ").slice(0, 80));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Collect callee names within a node body (coarse; unparsed call target). */
|
|
61
|
+
function callsIn(sf, node) {
|
|
62
|
+
const out = new Set();
|
|
63
|
+
const visit = (n) => {
|
|
64
|
+
if (ts.isCallExpression(n)) {
|
|
65
|
+
const t = n.expression;
|
|
66
|
+
let name = "";
|
|
67
|
+
if (ts.isIdentifier(t)) name = t.text;
|
|
68
|
+
else if (ts.isPropertyAccessExpression(t)) name = t.getText(sf);
|
|
69
|
+
if (name) out.add(name.slice(0, 80));
|
|
70
|
+
}
|
|
71
|
+
ts.forEachChild(n, visit);
|
|
72
|
+
};
|
|
73
|
+
ts.forEachChild(node, visit);
|
|
74
|
+
return [...out].sort();
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export async function extractFile(absPath, root) {
|
|
78
|
+
const text = await readFile(absPath, "utf8").catch(() => null);
|
|
79
|
+
if (text == null) return { failed: true, path: relPath(root, absPath) };
|
|
80
|
+
const path = relPath(root, absPath);
|
|
81
|
+
let sf;
|
|
82
|
+
try {
|
|
83
|
+
sf = ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true, scriptKind(path));
|
|
84
|
+
} catch {
|
|
85
|
+
return { failed: true, path };
|
|
86
|
+
}
|
|
87
|
+
const dir = dirname(path);
|
|
88
|
+
const imports = new Set();
|
|
89
|
+
const calls = new Set();
|
|
90
|
+
const defines = [];
|
|
91
|
+
const exports = new Set();
|
|
92
|
+
|
|
93
|
+
const resolveSpecifier = (spec) => {
|
|
94
|
+
if (!spec || !spec.startsWith(".")) { if (spec) imports.add(spec); return; } // bare/external
|
|
95
|
+
const joined = join(dir, spec).split(/[\\/]/).join("/");
|
|
96
|
+
const base = stripExt(joined);
|
|
97
|
+
imports.add(base);
|
|
98
|
+
imports.add(`${base}/index`); // dir-import fallback (./foo → ./foo/index)
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
const addFn = (name, node, kind, extra = {}) => {
|
|
102
|
+
defines.push({
|
|
103
|
+
name, kind, lineno: lineOf(sf, node), end_lineno: endLineOf(sf, node),
|
|
104
|
+
decorators: decoratorsOf(sf, node),
|
|
105
|
+
params: paramsText(sf, node), returns: returnText(sf, node),
|
|
106
|
+
calls: callsIn(sf, node), doc: firstDocLine(sf, node),
|
|
107
|
+
...(visibilityOf(node) ? { visibility: visibilityOf(node) } : {}),
|
|
108
|
+
...extra,
|
|
109
|
+
});
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
for (const stmt of sf.statements) {
|
|
113
|
+
// imports / re-exports
|
|
114
|
+
if (ts.isImportDeclaration(stmt) && stmt.moduleSpecifier && ts.isStringLiteral(stmt.moduleSpecifier)) {
|
|
115
|
+
resolveSpecifier(stmt.moduleSpecifier.text);
|
|
116
|
+
} else if (ts.isExportDeclaration(stmt) && stmt.moduleSpecifier && ts.isStringLiteral(stmt.moduleSpecifier)) {
|
|
117
|
+
resolveSpecifier(stmt.moduleSpecifier.text);
|
|
118
|
+
} else if (ts.isImportEqualsDeclaration?.(stmt)) {
|
|
119
|
+
// require()-style — best-effort skip of specifier
|
|
120
|
+
}
|
|
121
|
+
const exported = hasMod(stmt, ts.SyntaxKind.ExportKeyword);
|
|
122
|
+
|
|
123
|
+
if (ts.isFunctionDeclaration(stmt) && stmt.name) {
|
|
124
|
+
addFn(stmt.name.text, stmt, "function");
|
|
125
|
+
if (exported) exports.add(stmt.name.text);
|
|
126
|
+
} else if (ts.isClassDeclaration(stmt) && stmt.name) {
|
|
127
|
+
const cname = stmt.name.text;
|
|
128
|
+
const bases = [];
|
|
129
|
+
for (const h of stmt.heritageClauses || []) for (const t of h.types) bases.push(t.getText(sf).slice(0, 80));
|
|
130
|
+
defines.push({
|
|
131
|
+
name: cname, kind: "class", lineno: lineOf(sf, stmt), end_lineno: endLineOf(sf, stmt),
|
|
132
|
+
bases, decorators: decoratorsOf(sf, stmt), doc: firstDocLine(sf, stmt),
|
|
133
|
+
});
|
|
134
|
+
if (exported) exports.add(cname);
|
|
135
|
+
for (const mem of stmt.members) {
|
|
136
|
+
if ((ts.isMethodDeclaration(mem) || ts.isConstructorDeclaration(mem) ||
|
|
137
|
+
ts.isGetAccessor(mem) || ts.isSetAccessor(mem)) ) {
|
|
138
|
+
const mn = mem.name ? mem.name.getText(sf) : "constructor";
|
|
139
|
+
addFn(`${cname}.${mn}`, mem, "method");
|
|
140
|
+
} else if (ts.isPropertyDeclaration(mem) && mem.name) {
|
|
141
|
+
defines.push({
|
|
142
|
+
name: `${cname}.${mem.name.getText(sf)}`, kind: "attribute",
|
|
143
|
+
lineno: lineOf(sf, mem), end_lineno: endLineOf(sf, mem), decorators: decoratorsOf(sf, mem),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
} else if (ts.isInterfaceDeclaration(stmt) && stmt.name) {
|
|
148
|
+
// interfaces map to Class nodes (type surface) — useful for TS-heavy repos
|
|
149
|
+
defines.push({
|
|
150
|
+
name: stmt.name.text, kind: "class", lineno: lineOf(sf, stmt), end_lineno: endLineOf(sf, stmt),
|
|
151
|
+
bases: (stmt.heritageClauses || []).flatMap((h) => h.types.map((t) => t.getText(sf).slice(0, 80))),
|
|
152
|
+
decorators: [], doc: firstDocLine(sf, stmt),
|
|
153
|
+
});
|
|
154
|
+
if (exported) exports.add(stmt.name.text);
|
|
155
|
+
} else if (ts.isVariableStatement(stmt)) {
|
|
156
|
+
const isConst = (stmt.declarationList.flags & ts.NodeFlags.Const) !== 0;
|
|
157
|
+
for (const d of stmt.declarationList.declarations) {
|
|
158
|
+
if (!ts.isIdentifier(d.name)) continue;
|
|
159
|
+
const vn = d.name.text;
|
|
160
|
+
const init = d.initializer;
|
|
161
|
+
// arrow/function assigned to a const → treat as a function define
|
|
162
|
+
if (init && (ts.isArrowFunction(init) || ts.isFunctionExpression(init))) {
|
|
163
|
+
addFn(vn, init, "function");
|
|
164
|
+
} else if (isConst && /^[A-Z0-9_]+$/.test(vn)) {
|
|
165
|
+
defines.push({ name: vn, kind: "global", lineno: lineOf(sf, stmt), end_lineno: endLineOf(sf, stmt),
|
|
166
|
+
decorators: [], value: (init ? init.getText(sf) : "").replace(/\s+/g, " ").slice(0, 80), is_constant: true });
|
|
167
|
+
} else if (init && ts.isCallExpression(init)) {
|
|
168
|
+
// live-object global (e.g. const router = Router()) — a registration anchor
|
|
169
|
+
defines.push({ name: vn, kind: "global", lineno: lineOf(sf, stmt), end_lineno: endLineOf(sf, stmt),
|
|
170
|
+
decorators: [], value: init.getText(sf).replace(/\s+/g, " ").slice(0, 80) });
|
|
171
|
+
}
|
|
172
|
+
if (exported) exports.add(vn);
|
|
173
|
+
}
|
|
174
|
+
} else if (ts.isExportAssignment?.(stmt)) {
|
|
175
|
+
exports.add("default");
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// module-level coarse calls (whole file)
|
|
180
|
+
const visit = (n) => {
|
|
181
|
+
if (ts.isCallExpression(n)) {
|
|
182
|
+
const t = n.expression;
|
|
183
|
+
let name = "";
|
|
184
|
+
if (ts.isIdentifier(t)) name = t.text;
|
|
185
|
+
else if (ts.isPropertyAccessExpression(t)) name = t.getText(sf);
|
|
186
|
+
if (name) calls.add(name.slice(0, 80));
|
|
187
|
+
}
|
|
188
|
+
ts.forEachChild(n, visit);
|
|
189
|
+
};
|
|
190
|
+
ts.forEachChild(sf, visit);
|
|
191
|
+
|
|
192
|
+
return {
|
|
193
|
+
path, dotted: stripExt(path),
|
|
194
|
+
imports: [...imports].sort(), defines, calls: [...calls].sort(), exports: [...exports].sort(),
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Ingest a whole tree → {modules:[…]} contract + parse-failure list. */
|
|
199
|
+
export async function ingest(root) {
|
|
200
|
+
const files = await walk(root, EXTS);
|
|
201
|
+
const modules = [];
|
|
202
|
+
const failures = [];
|
|
203
|
+
for (const f of files) {
|
|
204
|
+
const mod = await extractFile(f, root);
|
|
205
|
+
if (mod.failed) failures.push(mod.path);
|
|
206
|
+
else modules.push(mod);
|
|
207
|
+
}
|
|
208
|
+
return { modules, failures, fileCount: files.length };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export const meta = { id: "tsc", language: "js/ts", lib: "typescript compiler API (via ts-morph)", exts: EXTS };
|