@polycode-projects/seonix 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -25
- package/bin/cli.mjs +78 -18
- package/package.json +5 -3
- package/roslyn/Program.cs +79 -11
- package/src/ask-nlp.mjs +73 -0
- package/src/ask-vocab.mjs +370 -5
- package/src/ask.mjs +1523 -83
- package/src/browser.mjs +99 -19
- package/src/chat.mjs +785 -0
- package/src/codegraph.mjs +213 -5
- package/src/cs_treesitter.mjs +57 -34
- package/src/embed.mjs +191 -0
- package/src/extract.mjs +220 -39
- package/src/java_treesitter.mjs +82 -55
- package/src/jsts_tsc.mjs +51 -4
- package/src/nlp-bundle.mjs +120 -0
- package/src/prose-nlp.mjs +52 -0
- package/src/prose.mjs +42 -0
- package/src/schema-docs.mjs +29 -0
- package/src/server.mjs +27 -4
- package/src/sessions.mjs +206 -0
- package/src/temporal.mjs +70 -0
- package/src/timeline.mjs +160 -0
- package/src/viz.mjs +273 -83
package/src/extract.mjs
CHANGED
|
@@ -18,14 +18,15 @@
|
|
|
18
18
|
// mgx:subclassOf Class → Class (inheritance; internal base resolved, else ext:)
|
|
19
19
|
|
|
20
20
|
import { spawn } from "node:child_process";
|
|
21
|
-
import { writeFile, mkdir } from "node:fs/promises";
|
|
22
|
-
import { dirname, join } from "node:path";
|
|
21
|
+
import { writeFile, mkdir, readdir, stat } from "node:fs/promises";
|
|
22
|
+
import { basename, dirname, join, parse, resolve, sep } from "node:path";
|
|
23
23
|
import { fileURLToPath } from "node:url";
|
|
24
24
|
import * as codegraph from "./codegraph.mjs"; // optional renderToolsCatalog (other agent owns this file)
|
|
25
25
|
import { ingestRepo, LANG_EXTS } from "./extract_lang.mjs";
|
|
26
26
|
import { loadIgnores } from "./walk.mjs";
|
|
27
27
|
import { attachProseTokens, buildProseIndex } from "./prose.mjs";
|
|
28
28
|
import { ingestSchemaDocs } from "./schema-docs.mjs";
|
|
29
|
+
import { foldInSessions, readSessionRecords } from "./sessions.mjs";
|
|
29
30
|
|
|
30
31
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
31
32
|
const AST_SCRIPT = join(here, "extract_ast.py");
|
|
@@ -194,16 +195,36 @@ export function buildEntities(modules, commits, { generatedAt = "", symbolHistor
|
|
|
194
195
|
for (const m of modules) {
|
|
195
196
|
for (const d of m.defines || []) {
|
|
196
197
|
if (d.kind === "method" || d.kind === "attribute" || d.kind === "global") continue; // not standalone call targets
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
if (
|
|
203
|
-
|
|
198
|
+
const register = (name) => {
|
|
199
|
+
if (!nameToPaths.has(name)) nameToPaths.set(name, new Set());
|
|
200
|
+
nameToPaths.get(name).add(m.path);
|
|
201
|
+
if (!nameToSymbolIds.has(name)) nameToSymbolIds.set(name, new Set());
|
|
202
|
+
nameToSymbolIds.get(name).add(fnId(m.path, d.name));
|
|
203
|
+
if (d.kind === "class") {
|
|
204
|
+
if (!classToPaths.has(name)) classToPaths.set(name, new Set());
|
|
205
|
+
classToPaths.get(name).add(m.path);
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
register(d.name);
|
|
209
|
+
// Nested types (Java/C#) define dotted names like Outer.Inner — ALSO register the
|
|
210
|
+
// simple name so `new Inner()` / `extends Inner` still resolve. Same Set semantics:
|
|
211
|
+
// a second definition of the simple name makes it ambiguous → dropped, honest.
|
|
212
|
+
if (d.kind === "class" && d.name.includes(".")) {
|
|
213
|
+
const simple = lastIdent(d.name);
|
|
214
|
+
if (simple && simple !== d.name) register(simple);
|
|
204
215
|
}
|
|
205
216
|
}
|
|
206
217
|
}
|
|
218
|
+
// Resolve a class SIMPLE name at a path to the id of its (possibly nested, dotted)
|
|
219
|
+
// define — nameToSymbolIds keeps full ids, so a unique in-path match wins; an exact
|
|
220
|
+
// plain define beats a nested one; else fall back to the literal id (pre-nesting shape).
|
|
221
|
+
const classIdAt = (path, ident) => {
|
|
222
|
+
const pre = `fn:${path}#`;
|
|
223
|
+
const exact = `${pre}${ident}`;
|
|
224
|
+
const ids = [...(nameToSymbolIds.get(ident) || [])].filter((i) => i.startsWith(pre));
|
|
225
|
+
if (ids.includes(exact) || ids.length !== 1) return exact;
|
|
226
|
+
return ids[0];
|
|
227
|
+
};
|
|
207
228
|
|
|
208
229
|
// resolve a module's import candidates to internal module paths
|
|
209
230
|
const internalImports = (m) => {
|
|
@@ -270,6 +291,7 @@ export function buildEntities(modules, commits, { generatedAt = "", symbolHistor
|
|
|
270
291
|
if (d.raises?.length) attrs.push({ prop: "seon:throwsException", key: "raises", value: list(d.raises) });
|
|
271
292
|
if (d.catches?.length) attrs.push({ prop: "seon:catchesException", key: "catches", value: list(d.catches) });
|
|
272
293
|
if (d.self_fields?.length) attrs.push({ prop: "seon:accessesField", key: "self_fields", value: list(d.self_fields) });
|
|
294
|
+
if (d.subkind) attrs.push({ prop: "seon:subKind", key: "subkind", value: String(d.subkind) });
|
|
273
295
|
if (d.is_static) attrs.push({ prop: "seon:isStatic", key: "isStatic", value: "true" });
|
|
274
296
|
if (d.is_abstract) attrs.push({ prop: "seon:isAbstract", key: "isAbstract", value: "true" });
|
|
275
297
|
if (d.is_constant) attrs.push({ prop: "seon:isConstant", key: "isConstant", value: "true" });
|
|
@@ -305,10 +327,10 @@ export function buildEntities(modules, commits, { generatedAt = "", symbolHistor
|
|
|
305
327
|
const defs = classToPaths.get(ident);
|
|
306
328
|
let object = `ext:${ident}`;
|
|
307
329
|
if (defs && defs.has(m.path)) {
|
|
308
|
-
object =
|
|
330
|
+
object = classIdAt(m.path, ident); // same-module base wins (local name scoping), even if the name is globally ambiguous
|
|
309
331
|
} else if (defs && defs.size === 1) {
|
|
310
332
|
const targetPath = [...defs][0];
|
|
311
|
-
if (imports.has(targetPath)) object =
|
|
333
|
+
if (imports.has(targetPath)) object = classIdAt(targetPath, ident);
|
|
312
334
|
}
|
|
313
335
|
const ikey = `${oid}>${object}`;
|
|
314
336
|
if (seenInherits.has(ikey) || object === oid) continue;
|
|
@@ -376,7 +398,9 @@ export function buildEntities(modules, commits, { generatedAt = "", symbolHistor
|
|
|
376
398
|
touchedBy.get(f).push(`git:${short}`);
|
|
377
399
|
touchedAny = true;
|
|
378
400
|
}
|
|
379
|
-
|
|
401
|
+
// !commitIds.has: a merged multi-repo commit list CAN repeat a sha (two clones of
|
|
402
|
+
// the same project indexed under different names) — one Commit individual per sha.
|
|
403
|
+
if (touchedAny && !commitIds.has(c.sha)) {
|
|
380
404
|
commitIds.add(c.sha);
|
|
381
405
|
commitIndividuals.push(commitInd(c.sha, short, c));
|
|
382
406
|
}
|
|
@@ -514,6 +538,7 @@ export function buildEntities(modules, commits, { generatedAt = "", symbolHistor
|
|
|
514
538
|
{ prop: "seon:isStatic", note: "@staticmethod/@classmethod" },
|
|
515
539
|
{ prop: "seon:isAbstract", note: "@abstractmethod/@abstractproperty" },
|
|
516
540
|
{ prop: "seon:isConstant", note: "ALL_CAPS module global" },
|
|
541
|
+
{ prop: "seon:subKind", note: "type flavour on a Class define when not a plain class (interface/enum/struct/record); kind stays class" },
|
|
517
542
|
{ prop: "seon:hasAccessModifier", note: "visibility from leading underscore (private/protected); public omitted" },
|
|
518
543
|
{ prop: "seon:hasDoc", note: "first docstring line (capped) — one-line purpose without the body" },
|
|
519
544
|
],
|
|
@@ -591,17 +616,13 @@ function localToolsCatalog(cliPath) {
|
|
|
591
616
|
return lines.join("\n");
|
|
592
617
|
}
|
|
593
618
|
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
*
|
|
598
|
-
*
|
|
599
|
-
|
|
600
|
-
export async function indexRepository(repoPath, { python = process.env.SEONIX_PYTHON || process.env.SEON_PYTHON || "python3", generatedAt = "", ignores = true } = {}) {
|
|
619
|
+
const DEFAULT_PYTHON = () => process.env.SEONIX_PYTHON || process.env.SEON_PYTHON || "python3";
|
|
620
|
+
|
|
621
|
+
/** One repo's raw extraction — parsers + git, NO graph assembly. Both index modes
|
|
622
|
+
* build on this; multi-repo runs it repo-by-repo so only one repo's raw source
|
|
623
|
+
* (parser/git-log output) is in memory at a time. */
|
|
624
|
+
async function extractRepo(repoPath, { python = DEFAULT_PYTHON(), ignores = true } = {}) {
|
|
601
625
|
const t0 = Date.now();
|
|
602
|
-
// Second pass (PLAN_PROSE_INDEX.md): on by default; SEONIX_PROSE_INDEX=0 disables it
|
|
603
|
-
// (mirrors SEONIX_HISTORY_SYMBOL_DEPTH=0's disable convention above).
|
|
604
|
-
const proseEnabled = process.env.SEONIX_PROSE_INDEX !== "0";
|
|
605
626
|
// `.seonixignore` (opt-out: `ignores:false`, the benchmark rig's flag). The matcher
|
|
606
627
|
// prunes the in-process walkers (extract_lang) at parse time; the module filter below
|
|
607
628
|
// is the AUTHORITATIVE cut — it also covers extractors that walk on their own
|
|
@@ -625,14 +646,24 @@ export async function indexRepository(repoPath, { python = process.env.SEONIX_PY
|
|
|
625
646
|
const tHist = Date.now();
|
|
626
647
|
const symbolHistory = await runGitLogHunks(repoPath, historySymbolDepth());
|
|
627
648
|
const historyMs = Date.now() - tHist;
|
|
649
|
+
return { modules, perLang: langResult.perLang, commits, symbolHistory, baseMs, historyMs };
|
|
650
|
+
}
|
|
628
651
|
|
|
652
|
+
/** Assemble entities from (possibly merged) extractions and write the artifacts
|
|
653
|
+
* (<rootDir>/.seonix/graph.json + TOOLS.md). Shared by both index modes. */
|
|
654
|
+
async function assembleAndWrite(rootDir, { modules, commits, symbolHistory, generatedAt, proseEnabled, sessions = [] }) {
|
|
629
655
|
const entities = buildEntities(modules, commits, { generatedAt, symbolHistory, prose: proseEnabled });
|
|
630
656
|
// Schema self-documentation (schema-docs.mjs): static, repo-independent — merges in
|
|
631
657
|
// SchemaClass/SchemaPredicate individuals + backfills entities.classes[].description
|
|
632
658
|
// and entities.vocabulary[].note, so "what does cochange mean" is answerable by the
|
|
633
659
|
// same graph traversal as any other question. Fixed-size, ~0ms marginal cost.
|
|
634
660
|
ingestSchemaDocs(entities);
|
|
635
|
-
|
|
661
|
+
// Chat sessions (sessions.mjs): sessions are runtime observations, not source
|
|
662
|
+
// derivations — they are re-attached AFTER the source-derived build, each recorded
|
|
663
|
+
// entity id re-resolved against the fresh graph (unresolvable → edge dropped and
|
|
664
|
+
// counted on the Session node, never guessed). No sessions → byte-identical output.
|
|
665
|
+
if (sessions.length) foldInSessions(entities, sessions);
|
|
666
|
+
const graphFile = join(rootDir, ".seonix", "graph.json");
|
|
636
667
|
await mkdir(dirname(graphFile), { recursive: true });
|
|
637
668
|
await writeFile(graphFile, JSON.stringify(entities));
|
|
638
669
|
|
|
@@ -641,27 +672,177 @@ export async function indexRepository(repoPath, { python = process.env.SEONIX_PY
|
|
|
641
672
|
const toolsMd = typeof codegraph.renderToolsCatalog === "function"
|
|
642
673
|
? codegraph.renderToolsCatalog(CLI_SCRIPT)
|
|
643
674
|
: localToolsCatalog(CLI_SCRIPT);
|
|
644
|
-
await writeFile(join(
|
|
675
|
+
await writeFile(join(rootDir, ".seonix", "TOOLS.md"), toolsMd);
|
|
676
|
+
return { entities, graphFile };
|
|
677
|
+
}
|
|
645
678
|
|
|
679
|
+
function buildCounts(entities, { modules, languages, commits, proseEnabled, baseMs, historyMs }) {
|
|
646
680
|
const propCount = (prop) => entities.objectProperties.find((g) => g.prop === prop)?.count ?? 0;
|
|
647
681
|
// history budget: the symbol pass should add ≤10% over base extraction time.
|
|
648
682
|
const historyPct = baseMs > 0 ? Math.round((historyMs / baseMs) * 1000) / 10 : 0;
|
|
683
|
+
return {
|
|
684
|
+
modules,
|
|
685
|
+
languages,
|
|
686
|
+
functions: entities.classes.find((c) => c.name === "Function")?.count ?? 0,
|
|
687
|
+
commits,
|
|
688
|
+
edges: entities.objectProperties.reduce((n, g) => n + g.count, 0),
|
|
689
|
+
callsSymbol: propCount("mgx:callsSymbol"),
|
|
690
|
+
touchesSymbol: propCount("mgx:touchesSymbol"),
|
|
691
|
+
historySymbolDepth: historySymbolDepth(),
|
|
692
|
+
proseEnabled,
|
|
693
|
+
proseWords: Object.keys(entities.proseIndex || {}).length,
|
|
694
|
+
baseMs,
|
|
695
|
+
historyMs,
|
|
696
|
+
historyPct,
|
|
697
|
+
};
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
/**
|
|
701
|
+
* Index a repository: parse (ast) + git log → entities → write <repo>/.seonix/graph.json
|
|
702
|
+
* (+ a TOOLS.md catalog of the cold tools). Times the base extraction vs the added
|
|
703
|
+
* symbol-history (line-range) pass so the latter can be kept within the ≤10% budget.
|
|
704
|
+
* @returns {Promise<{graphFile, counts}>}
|
|
705
|
+
*/
|
|
706
|
+
export async function indexRepository(repoPath, { python = DEFAULT_PYTHON(), generatedAt = "", ignores = true } = {}) {
|
|
707
|
+
// Second pass (PLAN_PROSE_INDEX.md): on by default; SEONIX_PROSE_INDEX=0 disables it
|
|
708
|
+
// (mirrors SEONIX_HISTORY_SYMBOL_DEPTH=0's disable convention above).
|
|
709
|
+
const proseEnabled = process.env.SEONIX_PROSE_INDEX !== "0";
|
|
710
|
+
const { modules, perLang, commits, symbolHistory, baseMs, historyMs } = await extractRepo(repoPath, { python, ignores });
|
|
711
|
+
const sessions = await readSessionRecords(repoPath); // recorded chat sessions (.seonix/sessions/*.jsonl), [] when none
|
|
712
|
+
const { entities, graphFile } = await assembleAndWrite(repoPath, { modules, commits, symbolHistory, generatedAt, proseEnabled, sessions });
|
|
649
713
|
return {
|
|
650
714
|
graphFile,
|
|
651
|
-
counts: {
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
715
|
+
counts: buildCounts(entities, { modules: modules.length, languages: perLang, commits: commits.length, proseEnabled, baseMs, historyMs }),
|
|
716
|
+
};
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
// ── multi-repository indexing ────────────────────────────────────────────────
|
|
720
|
+
// n repos → ONE merged graph at <out_root>/.seonix/. Single-path mode above is
|
|
721
|
+
// untouched (no prefix, artifacts in the repo — golden-compat guarded by
|
|
722
|
+
// test/multi-repo.test.mjs). In multi mode every module id/label/dotted gets the
|
|
723
|
+
// repo's directory basename as a leading component, so ids never collide and a
|
|
724
|
+
// reader can tell repos apart. Cross-repo callsSymbol edges arise only where a
|
|
725
|
+
// symbol name resolves uniquely across the whole merged registry (the existing
|
|
726
|
+
// Set-semantics drop ambiguous names — nothing cross-repo-special is done).
|
|
727
|
+
|
|
728
|
+
/** Deterministic repo-name prefixes for a set of (resolved, deduped) repo paths:
|
|
729
|
+
* the directory basename; when two repos share a basename the FIRST in path sort
|
|
730
|
+
* order keeps the bare name and later ones get `-2`, `-3`, … appended. */
|
|
731
|
+
export function assignRepoPrefixes(repoPaths) {
|
|
732
|
+
const map = new Map();
|
|
733
|
+
const used = new Set();
|
|
734
|
+
for (const rp of [...repoPaths].sort()) {
|
|
735
|
+
const base = basename(rp) || rp;
|
|
736
|
+
let name = base;
|
|
737
|
+
for (let n = 2; used.has(name); n += 1) name = `${base}-${n}`;
|
|
738
|
+
used.add(name);
|
|
739
|
+
map.set(rp, name);
|
|
740
|
+
}
|
|
741
|
+
return map;
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
/** Prefix one repo's raw extraction in place: module path/dotted/imports, commit
|
|
745
|
+
* file lists and symbol-history range keys all gain the repo name as a leading
|
|
746
|
+
* component (`mod:<repoName>/<relpath>`, dotted `<repoName>.<dotted>`). Imports
|
|
747
|
+
* are prefixed IDENTICALLY to dotted names so intra-repo import resolution is
|
|
748
|
+
* unchanged and two repos' equal dotted names can never cross-resolve. Commit
|
|
749
|
+
* ids (shas) stay as-is — history edges point at the prefixed module ids. */
|
|
750
|
+
export function applyRepoPrefix({ modules, commits, symbolHistory }, prefix) {
|
|
751
|
+
for (const m of modules) {
|
|
752
|
+
m.path = `${prefix}/${m.path}`;
|
|
753
|
+
if (m.dotted) m.dotted = `${prefix}.${m.dotted}`;
|
|
754
|
+
if (m.imports?.length) m.imports = m.imports.map((i) => `${prefix}.${i}`);
|
|
755
|
+
}
|
|
756
|
+
for (const c of commits) c.files = (c.files || []).map((f) => `${prefix}/${f}`);
|
|
757
|
+
for (const c of symbolHistory) {
|
|
758
|
+
c.ranges = Object.fromEntries(Object.entries(c.ranges || {}).map(([p, r]) => [`${prefix}/${p}`, r]));
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/** Default merged-artifact root: the deepest common ancestor DIRECTORY of the repo
|
|
763
|
+
* paths (segment-wise, so /x/foo and /x/foobar meet at /x, not /x/foo). Falls back
|
|
764
|
+
* to `cwd` when the ancestor is the filesystem root. */
|
|
765
|
+
export function defaultOutRoot(repoPaths, cwd = process.cwd()) {
|
|
766
|
+
const segs = repoPaths.map((p) => resolve(p).split(sep));
|
|
767
|
+
const first = segs[0];
|
|
768
|
+
let depth = Math.min(...segs.map((s) => s.length));
|
|
769
|
+
let i = 0;
|
|
770
|
+
while (i < depth && segs.every((s) => s[i] === first[i])) i += 1;
|
|
771
|
+
const ancestor = first.slice(0, i).join(sep) || sep;
|
|
772
|
+
return ancestor === parse(ancestor).root ? cwd : ancestor;
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
/** Discovery for the estate case: every immediate child directory of `multiRoot`
|
|
776
|
+
* that carries a `.git` (dir OR file — worktrees/submodules have .git files) is a
|
|
777
|
+
* repo. Dot-dirs are ignored outright; plain child dirs without .git are returned
|
|
778
|
+
* as `skipped` so the caller can log them. */
|
|
779
|
+
export async function discoverRepos(multiRoot) {
|
|
780
|
+
const root = resolve(multiRoot);
|
|
781
|
+
const entries = await readdir(root, { withFileTypes: true });
|
|
782
|
+
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
783
|
+
const repos = [];
|
|
784
|
+
const skipped = [];
|
|
785
|
+
for (const e of entries) {
|
|
786
|
+
if (!e.isDirectory() || e.name.startsWith(".")) continue;
|
|
787
|
+
try {
|
|
788
|
+
await stat(join(root, e.name, ".git"));
|
|
789
|
+
repos.push(join(root, e.name));
|
|
790
|
+
} catch {
|
|
791
|
+
skipped.push(e.name);
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
return { repos, skipped };
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
/**
|
|
798
|
+
* Index n repositories into ONE merged graph at <outRoot>/.seonix/. Repos are
|
|
799
|
+
* extracted sequentially (only one repo's raw parser/git output held at a time;
|
|
800
|
+
* the accumulated PARSED module list is small) and merged through a single
|
|
801
|
+
* buildEntities pass, so the unique-name registry spans all repos.
|
|
802
|
+
* `log` (optional) receives one progress line per repo.
|
|
803
|
+
* @returns {Promise<{graphFile, outRoot, repos, counts}>}
|
|
804
|
+
*/
|
|
805
|
+
export async function indexRepositories(repoPaths, { python = DEFAULT_PYTHON(), generatedAt = "", ignores = true, outRoot = "", log = () => {} } = {}) {
|
|
806
|
+
const paths = [...new Set((repoPaths || []).map((p) => resolve(p)))];
|
|
807
|
+
if (!paths.length) throw new Error("indexRepositories requires at least one repo path");
|
|
808
|
+
const proseEnabled = process.env.SEONIX_PROSE_INDEX !== "0";
|
|
809
|
+
const prefixes = assignRepoPrefixes(paths);
|
|
810
|
+
const root = outRoot ? resolve(outRoot) : defaultOutRoot(paths);
|
|
811
|
+
|
|
812
|
+
const allModules = [];
|
|
813
|
+
const allCommits = [];
|
|
814
|
+
const allSymbolHistory = [];
|
|
815
|
+
const languages = {};
|
|
816
|
+
const repos = [];
|
|
817
|
+
let baseMs = 0;
|
|
818
|
+
let historyMs = 0;
|
|
819
|
+
for (const rp of paths) {
|
|
820
|
+
const prefix = prefixes.get(rp);
|
|
821
|
+
const r = await extractRepo(rp, { python, ignores });
|
|
822
|
+
applyRepoPrefix(r, prefix);
|
|
823
|
+
allModules.push(...r.modules);
|
|
824
|
+
allCommits.push(...r.commits);
|
|
825
|
+
allSymbolHistory.push(...r.symbolHistory);
|
|
826
|
+
baseMs += r.baseMs;
|
|
827
|
+
historyMs += r.historyMs;
|
|
828
|
+
for (const [lang, s] of Object.entries(r.perLang)) {
|
|
829
|
+
const agg = languages[lang] || (languages[lang] = { lib: s.lib, files: 0, modules: 0, symbols: 0, failures: 0, ms: 0 });
|
|
830
|
+
agg.files += s.files; agg.modules += s.modules; agg.symbols += s.symbols; agg.failures += s.failures; agg.ms += s.ms;
|
|
831
|
+
}
|
|
832
|
+
repos.push({ path: rp, prefix, modules: r.modules.length, commits: r.commits.length });
|
|
833
|
+
log(`indexed ${prefix} (${rp}): ${r.modules.length} modules, ${r.commits.length} commits in ${r.baseMs + r.historyMs}ms`);
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
// TODO(sessions): multi-repo merges don't fold in the member repos' .seonix/sessions
|
|
837
|
+
// yet — their recorded ids would need the same repo-name prefixing as modules to
|
|
838
|
+
// re-resolve against the merged graph. Deferred; single-path fold-in is the contract.
|
|
839
|
+
const { entities, graphFile } = await assembleAndWrite(root, {
|
|
840
|
+
modules: allModules, commits: allCommits, symbolHistory: allSymbolHistory, generatedAt, proseEnabled,
|
|
841
|
+
});
|
|
842
|
+
return {
|
|
843
|
+
graphFile,
|
|
844
|
+
outRoot: root,
|
|
845
|
+
repos,
|
|
846
|
+
counts: buildCounts(entities, { modules: allModules.length, languages, commits: allCommits.length, proseEnabled, baseMs, historyMs }),
|
|
666
847
|
};
|
|
667
848
|
}
|
package/src/java_treesitter.mjs
CHANGED
|
@@ -36,10 +36,12 @@ const clip = (s, n) => (s.length <= n ? s : s.slice(0, n));
|
|
|
36
36
|
const fieldText = (n, f) => n.childForFieldName(f)?.text || "";
|
|
37
37
|
|
|
38
38
|
const TYPE_DECLS = new Set(["class_declaration", "interface_declaration", "enum_declaration", "record_declaration"]);
|
|
39
|
+
const SUBKIND = { interface_declaration: "interface", enum_declaration: "enum", record_declaration: "record" };
|
|
39
40
|
const BASE_WRAPPERS = new Set(["superclass", "super_interfaces", "extends_interfaces", "interfaces"]);
|
|
40
41
|
const TYPE_NODES = new Set(["type_identifier", "generic_type", "scoped_type_identifier"]);
|
|
41
42
|
|
|
42
|
-
/** Callee names within a node (syntax-level; unresolved).
|
|
43
|
+
/** Callee names within a node (syntax-level; unresolved). Object creations count as
|
|
44
|
+
* call edges to the created type: `new X(...)` → bare "X" (generics/scope stripped). */
|
|
43
45
|
function collectCalls(node) {
|
|
44
46
|
const out = new Set();
|
|
45
47
|
const stack = [node];
|
|
@@ -48,6 +50,10 @@ function collectCalls(node) {
|
|
|
48
50
|
if (n.type === "method_invocation") {
|
|
49
51
|
const nm = fieldText(n, "name");
|
|
50
52
|
if (nm) out.add(clip(nm, 80));
|
|
53
|
+
} else if (n.type === "object_creation_expression") {
|
|
54
|
+
const t = n.childForFieldName("type");
|
|
55
|
+
const nm = t ? oneLine(t.text).replace(/<.*$/s, "").split(".").pop().trim() : "";
|
|
56
|
+
if (nm) out.add(clip(nm, 80));
|
|
51
57
|
}
|
|
52
58
|
for (let i = 0; i < n.namedChildCount; i++) stack.push(n.namedChild(i));
|
|
53
59
|
}
|
|
@@ -104,6 +110,79 @@ function bodyContainer(typeNode) {
|
|
|
104
110
|
|| null;
|
|
105
111
|
}
|
|
106
112
|
|
|
113
|
+
/** Emit one type declaration (and, recursively, its nested types as Outer.Inner —
|
|
114
|
+
* matching the JavaParser backend's qualified names). kind stays "class"; the
|
|
115
|
+
* flavour (interface/enum/record) lands in `subkind`. */
|
|
116
|
+
function emitType(n, prefix, defines, exports) {
|
|
117
|
+
const simple = fieldText(n, "name");
|
|
118
|
+
if (!simple) return;
|
|
119
|
+
const cname = prefix ? `${prefix}.${simple}` : simple;
|
|
120
|
+
const mods = modifiersNode(n);
|
|
121
|
+
defines.push({
|
|
122
|
+
name: cname, kind: "class", lineno: line(n), end_lineno: endLine(n),
|
|
123
|
+
bases: basesOf(n), decorators: decoratorsFrom(mods),
|
|
124
|
+
...(SUBKIND[n.type] ? { subkind: SUBKIND[n.type] } : {}),
|
|
125
|
+
...(visFrom(mods) ? { visibility: visFrom(mods) } : {}),
|
|
126
|
+
});
|
|
127
|
+
if (isPublic(mods)) exports.add(cname);
|
|
128
|
+
|
|
129
|
+
// record components (formal params) surface as attributes (match JavaParser)
|
|
130
|
+
if (n.type === "record_declaration") {
|
|
131
|
+
const fp = n.childForFieldName("parameters");
|
|
132
|
+
if (fp) for (let i = 0; i < fp.namedChildCount; i++) {
|
|
133
|
+
const fpc = fp.namedChild(i);
|
|
134
|
+
if (fpc.type !== "formal_parameter") continue;
|
|
135
|
+
const pn = fieldText(fpc, "name");
|
|
136
|
+
if (pn) defines.push({ name: `${cname}.${pn}`, kind: "attribute", lineno: line(fpc), end_lineno: endLine(fpc), decorators: [] });
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const body = bodyContainer(n);
|
|
141
|
+
if (!body) return;
|
|
142
|
+
// enum bodies keep constants at the top and wrap the rest in enum_body_declarations
|
|
143
|
+
const members = [];
|
|
144
|
+
for (let i = 0; i < body.namedChildCount; i++) {
|
|
145
|
+
const mem = body.namedChild(i);
|
|
146
|
+
if (mem.type === "enum_body_declarations") {
|
|
147
|
+
for (let j = 0; j < mem.namedChildCount; j++) members.push(mem.namedChild(j));
|
|
148
|
+
} else {
|
|
149
|
+
members.push(mem);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
for (const mem of members) {
|
|
153
|
+
if (TYPE_DECLS.has(mem.type)) {
|
|
154
|
+
emitType(mem, cname, defines, exports); // nested type → Outer.Inner
|
|
155
|
+
} else if (mem.type === "enum_constant") {
|
|
156
|
+
const en = fieldText(mem, "name");
|
|
157
|
+
if (en) defines.push({ name: `${cname}.${en}`, kind: "attribute", lineno: line(mem), end_lineno: endLine(mem), decorators: [], is_constant: true });
|
|
158
|
+
} else if (mem.type === "method_declaration" || mem.type === "constructor_declaration") {
|
|
159
|
+
const mn = fieldText(mem, "name") || cname;
|
|
160
|
+
const mmods = modifiersNode(mem);
|
|
161
|
+
const returns = mem.type === "method_declaration"
|
|
162
|
+
? clip(oneLine(mem.childForFieldName("type")?.text || ""), 80) : "";
|
|
163
|
+
defines.push({
|
|
164
|
+
name: `${cname}.${mn}`, kind: "method", lineno: line(mem), end_lineno: endLine(mem),
|
|
165
|
+
decorators: decoratorsFrom(mmods), params: paramsText(mem),
|
|
166
|
+
...(returns ? { returns } : {}), calls: collectCalls(mem),
|
|
167
|
+
...(isStatic(mmods) ? { is_static: true } : {}),
|
|
168
|
+
...(visFrom(mmods) ? { visibility: visFrom(mmods) } : {}),
|
|
169
|
+
});
|
|
170
|
+
} else if (mem.type === "field_declaration") {
|
|
171
|
+
const fmods = modifiersNode(mem);
|
|
172
|
+
for (let j = 0; j < mem.namedChildCount; j++) {
|
|
173
|
+
const vd = mem.namedChild(j);
|
|
174
|
+
if (vd.type !== "variable_declarator") continue;
|
|
175
|
+
const fn = fieldText(vd, "name");
|
|
176
|
+
if (fn) defines.push({
|
|
177
|
+
name: `${cname}.${fn}`, kind: "attribute", lineno: line(mem), end_lineno: endLine(mem),
|
|
178
|
+
decorators: decoratorsFrom(fmods),
|
|
179
|
+
...(visFrom(fmods) ? { visibility: visFrom(fmods) } : {}),
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
107
186
|
export async function extractFile(absPath, root) {
|
|
108
187
|
const text = await readFile(absPath, "utf8").catch(() => null);
|
|
109
188
|
const path = relPath(root, absPath);
|
|
@@ -132,61 +211,9 @@ export async function extractFile(absPath, root) {
|
|
|
132
211
|
if (nm) imports.add(oneLine(nm));
|
|
133
212
|
}
|
|
134
213
|
} else if (TYPE_DECLS.has(n.type)) {
|
|
135
|
-
|
|
136
|
-
if (cname) {
|
|
137
|
-
const mods = modifiersNode(n);
|
|
138
|
-
defines.push({
|
|
139
|
-
name: cname, kind: "class", lineno: line(n), end_lineno: endLine(n),
|
|
140
|
-
bases: basesOf(n), decorators: decoratorsFrom(mods),
|
|
141
|
-
...(visFrom(mods) ? { visibility: visFrom(mods) } : {}),
|
|
142
|
-
});
|
|
143
|
-
if (isPublic(mods)) exports.add(cname);
|
|
144
|
-
|
|
145
|
-
// record components (formal params) surface as attributes (match JavaParser)
|
|
146
|
-
if (n.type === "record_declaration") {
|
|
147
|
-
const fp = n.childForFieldName("parameters");
|
|
148
|
-
if (fp) for (let i = 0; i < fp.namedChildCount; i++) {
|
|
149
|
-
const fpc = fp.namedChild(i);
|
|
150
|
-
if (fpc.type !== "formal_parameter") continue;
|
|
151
|
-
const pn = fieldText(fpc, "name");
|
|
152
|
-
if (pn) defines.push({ name: `${cname}.${pn}`, kind: "attribute", lineno: line(fpc), end_lineno: endLine(fpc), decorators: [] });
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
const body = bodyContainer(n);
|
|
157
|
-
if (body) {
|
|
158
|
-
for (let i = 0; i < body.namedChildCount; i++) {
|
|
159
|
-
const mem = body.namedChild(i);
|
|
160
|
-
if (mem.type === "method_declaration" || mem.type === "constructor_declaration") {
|
|
161
|
-
const mn = fieldText(mem, "name") || cname;
|
|
162
|
-
const mmods = modifiersNode(mem);
|
|
163
|
-
const returns = mem.type === "method_declaration"
|
|
164
|
-
? clip(oneLine(mem.childForFieldName("type")?.text || ""), 80) : "";
|
|
165
|
-
defines.push({
|
|
166
|
-
name: `${cname}.${mn}`, kind: "method", lineno: line(mem), end_lineno: endLine(mem),
|
|
167
|
-
decorators: decoratorsFrom(mmods), params: paramsText(mem),
|
|
168
|
-
...(returns ? { returns } : {}), calls: collectCalls(mem),
|
|
169
|
-
...(isStatic(mmods) ? { is_static: true } : {}),
|
|
170
|
-
...(visFrom(mmods) ? { visibility: visFrom(mmods) } : {}),
|
|
171
|
-
});
|
|
172
|
-
} else if (mem.type === "field_declaration") {
|
|
173
|
-
const fmods = modifiersNode(mem);
|
|
174
|
-
for (let j = 0; j < mem.namedChildCount; j++) {
|
|
175
|
-
const vd = mem.namedChild(j);
|
|
176
|
-
if (vd.type !== "variable_declarator") continue;
|
|
177
|
-
const fn = fieldText(vd, "name");
|
|
178
|
-
if (fn) defines.push({
|
|
179
|
-
name: `${cname}.${fn}`, kind: "attribute", lineno: line(mem), end_lineno: endLine(mem),
|
|
180
|
-
decorators: decoratorsFrom(fmods),
|
|
181
|
-
...(visFrom(fmods) ? { visibility: visFrom(fmods) } : {}),
|
|
182
|
-
});
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
}
|
|
214
|
+
emitType(n, "", defines, exports);
|
|
188
215
|
}
|
|
189
|
-
// descend, but not back into a type body (
|
|
216
|
+
// descend, but not back into a type body (emitType recurses for nested types)
|
|
190
217
|
if (!TYPE_DECLS.has(n.type)) for (let i = 0; i < n.namedChildCount; i++) stack.push(n.namedChild(i));
|
|
191
218
|
}
|
|
192
219
|
|
package/src/jsts_tsc.mjs
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
|
-
// JS/TS extractor — TypeScript compiler API (
|
|
2
|
-
//
|
|
1
|
+
// JS/TS extractor — TypeScript compiler API (direct `typescript` dep; the
|
|
2
|
+
// 2026-07-02 library review swapped ts-morph out — we only ever used its
|
|
3
|
+
// re-exported `ts` namespace, so importing the compiler directly is byte-identical
|
|
4
|
+
// output for −12.4 MB of install). PICKED candidate for JS/TS.
|
|
3
5
|
//
|
|
4
6
|
// Uses ts.createSourceFile per file (no Program / no type-checker) — the fast,
|
|
5
7
|
// deterministic, offline structural pass. Covers .ts/.tsx/.js/.jsx (allowJs is
|
|
6
8
|
// implicit: JS is just parsed with the JS script-kind). Emits the SAME
|
|
7
9
|
// `{path,dotted,imports,defines,calls,exports}` contract extract_ast.py prints,
|
|
8
10
|
// so extract.mjs/buildEntities + codegraph.mjs/digest consume it unchanged.
|
|
11
|
+
// Both module systems are read: ESM import/export declarations AND a CommonJS
|
|
12
|
+
// pass (top-level require() with literal specifiers → imports; module.exports /
|
|
13
|
+
// exports.name assignments → exports) — express-style repos are CJS-only and
|
|
14
|
+
// were producing edge-empty graphs before the 2026-07-02 pass.
|
|
9
15
|
//
|
|
10
16
|
// Fidelity note (honest): params/returns are ANNOTATION strings (Group-A
|
|
11
17
|
// mechanical), exactly like the Python path's "returns = annotation only". A
|
|
@@ -13,7 +19,7 @@
|
|
|
13
19
|
// promoting Group-B→A) is the documented ceiling, NOT run here.
|
|
14
20
|
import { readFile } from "node:fs/promises";
|
|
15
21
|
import { dirname, join } from "node:path";
|
|
16
|
-
import
|
|
22
|
+
import ts from "typescript";
|
|
17
23
|
import { walk, relPath } from "./walk.mjs";
|
|
18
24
|
|
|
19
25
|
const EXTS = [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"];
|
|
@@ -98,6 +104,27 @@ export async function extractFile(absPath, root) {
|
|
|
98
104
|
imports.add(`${base}/index`); // dir-import fallback (./foo → ./foo/index)
|
|
99
105
|
};
|
|
100
106
|
|
|
107
|
+
// ---- CommonJS pass (2026-07-02): express-style repos are require()/module.exports only,
|
|
108
|
+
// which left their graphs edge-empty (js-express: imports 0 — five of six ask families
|
|
109
|
+
// structurally impossible). Top-level statements only, same resolveSpecifier as the ESM
|
|
110
|
+
// specifiers (relative paths resolve to modules, bare ones stay as-is), and STRICTLY
|
|
111
|
+
// literal arguments — a require(expr) is skipped, never guessed (house "no wrong edge").
|
|
112
|
+
/** `require("x")` / require(`x`) with a single literal arg → its specifier, else null. */
|
|
113
|
+
const requireSpecifier = (node) =>
|
|
114
|
+
node && ts.isCallExpression(node) && ts.isIdentifier(node.expression) && node.expression.text === "require" &&
|
|
115
|
+
node.arguments.length === 1 && ts.isStringLiteralLike(node.arguments[0])
|
|
116
|
+
? node.arguments[0].text
|
|
117
|
+
: null;
|
|
118
|
+
/** Peel `require("x").member` / parens down to the require call (never enters functions). */
|
|
119
|
+
const requireIn = (expr) => {
|
|
120
|
+
let e = expr;
|
|
121
|
+
while (e && (ts.isPropertyAccessExpression(e) || ts.isParenthesizedExpression(e) || ts.isNonNullExpression?.(e))) e = e.expression;
|
|
122
|
+
return requireSpecifier(e);
|
|
123
|
+
};
|
|
124
|
+
const isModuleExports = (e) =>
|
|
125
|
+
ts.isPropertyAccessExpression(e) && ts.isIdentifier(e.expression) && e.expression.text === "module" && e.name.text === "exports";
|
|
126
|
+
const isExportsRef = (e) => isModuleExports(e) || (ts.isIdentifier(e) && e.text === "exports");
|
|
127
|
+
|
|
101
128
|
const addFn = (name, node, kind, extra = {}) => {
|
|
102
129
|
defines.push({
|
|
103
130
|
name, kind, lineno: lineOf(sf, node), end_lineno: endLineOf(sf, node),
|
|
@@ -155,6 +182,11 @@ export async function extractFile(absPath, root) {
|
|
|
155
182
|
} else if (ts.isVariableStatement(stmt)) {
|
|
156
183
|
const isConst = (stmt.declarationList.flags & ts.NodeFlags.Const) !== 0;
|
|
157
184
|
for (const d of stmt.declarationList.declarations) {
|
|
185
|
+
// CJS import: const X = require('spec') / const {a, b} = require('spec') /
|
|
186
|
+
// const Y = require('spec').member — BEFORE the identifier-only gate so a
|
|
187
|
+
// destructuring binding still records the import edge.
|
|
188
|
+
const reqSpec = requireIn(d.initializer);
|
|
189
|
+
if (reqSpec != null) resolveSpecifier(reqSpec);
|
|
158
190
|
if (!ts.isIdentifier(d.name)) continue;
|
|
159
191
|
const vn = d.name.text;
|
|
160
192
|
const init = d.initializer;
|
|
@@ -173,6 +205,21 @@ export async function extractFile(absPath, root) {
|
|
|
173
205
|
}
|
|
174
206
|
} else if (ts.isExportAssignment?.(stmt)) {
|
|
175
207
|
exports.add("default");
|
|
208
|
+
} else if (ts.isExpressionStatement(stmt)) {
|
|
209
|
+
// CJS exports + side-effect/re-export requires. Walk `=` chains so
|
|
210
|
+
// `exports = module.exports = X` records one default export, and
|
|
211
|
+
// `module.exports = require('./lib')` records both the export and the import.
|
|
212
|
+
let expr = stmt.expression;
|
|
213
|
+
while (ts.isBinaryExpression(expr) && expr.operatorToken.kind === ts.SyntaxKind.EqualsToken) {
|
|
214
|
+
const lhs = expr.left;
|
|
215
|
+
if (isExportsRef(lhs)) exports.add("default"); // module.exports = X / exports = X
|
|
216
|
+
else if (ts.isPropertyAccessExpression(lhs) && isExportsRef(lhs.expression)) {
|
|
217
|
+
exports.add(lhs.name.text); // module.exports.name = / exports.name =
|
|
218
|
+
}
|
|
219
|
+
expr = expr.right;
|
|
220
|
+
}
|
|
221
|
+
const reqSpec = requireIn(expr); // covers bare `require('./side-effect')` too
|
|
222
|
+
if (reqSpec != null) resolveSpecifier(reqSpec);
|
|
176
223
|
}
|
|
177
224
|
}
|
|
178
225
|
|
|
@@ -208,4 +255,4 @@ export async function ingest(root, { ignore = null } = {}) {
|
|
|
208
255
|
return { modules, failures, fileCount: files.length };
|
|
209
256
|
}
|
|
210
257
|
|
|
211
|
-
export const meta = { id: "tsc", language: "js/ts", lib: "typescript compiler API
|
|
258
|
+
export const meta = { id: "tsc", language: "js/ts", lib: "typescript compiler API", exts: EXTS };
|