gitnexus 1.6.9-rc.25 → 1.6.9-rc.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/core/group/extractors/http-patterns/python.js +50 -12
- package/dist/core/ingestion/parsing-processor.d.ts +2 -1
- package/dist/core/ingestion/parsing-processor.js +5 -0
- package/dist/core/ingestion/pipeline-phases/parse-impl.js +40 -4
- package/dist/core/ingestion/route-extractors/fastapi-router-bindings.d.ts +5 -1
- package/dist/core/ingestion/route-extractors/fastapi-router-bindings.js +52 -1
- package/dist/core/ingestion/workers/parse-worker.d.ts +2 -1
- package/dist/core/ingestion/workers/parse-worker.js +4 -1
- package/dist/core/ingestion/workers/result-merge.js +4 -0
- package/dist/core/lbug/lbug-adapter.d.ts +6 -0
- package/dist/core/lbug/lbug-adapter.js +8 -2
- package/dist/core/run-analyze.js +5 -1
- package/dist/core/search/fts-indexes.d.ts +14 -0
- package/dist/core/search/fts-indexes.js +67 -2
- package/dist/storage/parse-cache.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -363,6 +363,7 @@ Configure the behavior with two environment variables:
|
|
|
363
363
|
| -------------------------------------------- | ---------------------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
364
364
|
| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded INSTALL if LOAD fails. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. |
|
|
365
365
|
| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process `INSTALL` child before it is killed. |
|
|
366
|
+
| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. |
|
|
366
367
|
| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. |
|
|
367
368
|
|
|
368
369
|
```bash
|
|
@@ -371,6 +372,9 @@ GITNEXUS_LBUG_EXTENSION_INSTALL=load-only npx gitnexus analyze
|
|
|
371
372
|
|
|
372
373
|
# Slow network: give extension downloads more time
|
|
373
374
|
GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS=30000 npx gitnexus analyze
|
|
375
|
+
|
|
376
|
+
# CJK-heavy codebase: rebuild keyword indexes without English stemming
|
|
377
|
+
GITNEXUS_FTS_STEMMER=none npx gitnexus analyze --repair-fts
|
|
374
378
|
```
|
|
375
379
|
|
|
376
380
|
### Analysis runs out of memory
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import Python from 'tree-sitter-python';
|
|
2
2
|
import { compilePatterns, runCompiledPatterns, unquoteLiteral, } from '../tree-sitter-scanner.js';
|
|
3
|
+
import { normalizeExtractedRoutePath } from '../../../ingestion/route-extractors/route-path.js';
|
|
3
4
|
/**
|
|
4
5
|
* Python HTTP plugin. Handles:
|
|
5
6
|
* - FastAPI `@app.get("/path")` provider decorators
|
|
@@ -146,6 +147,25 @@ const INCLUDE_ROUTER_NAME_PATTERNS = compilePatterns({
|
|
|
146
147
|
},
|
|
147
148
|
],
|
|
148
149
|
});
|
|
150
|
+
const API_ROUTER_PREFIX_PATTERNS = compilePatterns({
|
|
151
|
+
name: 'python-fastapi-apirouter-prefix',
|
|
152
|
+
language: Python,
|
|
153
|
+
patterns: [
|
|
154
|
+
{
|
|
155
|
+
meta: {},
|
|
156
|
+
query: `
|
|
157
|
+
(assignment
|
|
158
|
+
left: (identifier) @router_name (#eq? @router_name "router")
|
|
159
|
+
right: (call
|
|
160
|
+
function: (identifier) @factory (#eq? @factory "APIRouter")
|
|
161
|
+
arguments: (argument_list
|
|
162
|
+
(keyword_argument
|
|
163
|
+
name: (identifier) @kw (#eq? @kw "prefix")
|
|
164
|
+
value: (string) @prefix))))
|
|
165
|
+
`,
|
|
166
|
+
},
|
|
167
|
+
],
|
|
168
|
+
});
|
|
149
169
|
// `from .api.assistant import router` style — used together with
|
|
150
170
|
// INCLUDE_ROUTER_NAME so we can map a local name back to its module
|
|
151
171
|
// path, then back to the file the router was declared in.
|
|
@@ -813,10 +833,10 @@ function recordPrefix(target, key, prefix) {
|
|
|
813
833
|
function buildPythonRepoContext(files, parser, readFile, parseSource) {
|
|
814
834
|
const prefixesByLongKey = new Map();
|
|
815
835
|
const prefixesByShortKey = new Map();
|
|
816
|
-
//
|
|
817
|
-
//
|
|
818
|
-
// is
|
|
819
|
-
//
|
|
836
|
+
// Cross-file pre-pass: only `include_router` sites need it — they bind a
|
|
837
|
+
// prefix declared in one file to a router defined in another. Same-file
|
|
838
|
+
// `APIRouter(prefix=...)` is resolved in scan() from the file's own tree, so
|
|
839
|
+
// APIRouter-only files are left out here and never parsed twice.
|
|
820
840
|
for (const rel of files) {
|
|
821
841
|
if (!rel.endsWith('.py'))
|
|
822
842
|
continue;
|
|
@@ -912,14 +932,16 @@ function buildPythonRepoContext(files, parser, readFile, parseSource) {
|
|
|
912
932
|
}
|
|
913
933
|
}
|
|
914
934
|
}
|
|
915
|
-
return {
|
|
935
|
+
return {
|
|
936
|
+
prefixesByLongKey,
|
|
937
|
+
prefixesByShortKey,
|
|
938
|
+
};
|
|
916
939
|
}
|
|
917
940
|
function joinPrefix(prefix, route) {
|
|
918
|
-
//
|
|
919
|
-
//
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
return `${p}${r}`;
|
|
941
|
+
// Delegate to the shared route-path normalizer so the group contract and the
|
|
942
|
+
// ingestion Route node join prefixes identically — one helper, no
|
|
943
|
+
// trailing-slash drift on empty routes (`APIRouter(prefix="/x")` + `@get("")`).
|
|
944
|
+
return normalizeExtractedRoutePath(route, prefix);
|
|
923
945
|
}
|
|
924
946
|
export const PYTHON_HTTP_PLUGIN = {
|
|
925
947
|
name: 'python-http',
|
|
@@ -979,6 +1001,19 @@ export const PYTHON_HTTP_PLUGIN = {
|
|
|
979
1001
|
// Django providers come from the graph Route nodes (includes composed by
|
|
980
1002
|
// the ingestion route extractor), not a per-file source scan — see the note
|
|
981
1003
|
// at the top of this file.
|
|
1004
|
+
// Same-file `router = APIRouter(prefix="/x")` (router-only). Read from this
|
|
1005
|
+
// file's own tree, so there is no cross-file map and no prefix bleed across
|
|
1006
|
+
// same-stem files; it stacks under any include_router(prefix=...) below.
|
|
1007
|
+
let constructorPrefix;
|
|
1008
|
+
for (const m of runCompiledPatterns(API_ROUTER_PREFIX_PATTERNS, tree)) {
|
|
1009
|
+
const prefixNode = m.captures.prefix;
|
|
1010
|
+
if (!prefixNode)
|
|
1011
|
+
continue;
|
|
1012
|
+
const p = unquoteLiteral(prefixNode.text);
|
|
1013
|
+
if (p === null)
|
|
1014
|
+
continue;
|
|
1015
|
+
constructorPrefix = p;
|
|
1016
|
+
}
|
|
982
1017
|
// Providers: FastAPI @router.<verb>("/path") — must be joined
|
|
983
1018
|
// with the prefix(es) declared at the include_router site. When
|
|
984
1019
|
// no prefix is found we still emit the unprefixed path so this
|
|
@@ -1004,9 +1039,12 @@ export const PYTHON_HTTP_PLUGIN = {
|
|
|
1004
1039
|
const shortKey = fileRel ? fileShortKey(fileRel) : '';
|
|
1005
1040
|
const shortPrefixes = longPrefixes || !shortKey ? undefined : ctx?.prefixesByShortKey.get(shortKey);
|
|
1006
1041
|
const prefixSet = longPrefixes ?? shortPrefixes;
|
|
1042
|
+
// Stack the same-file APIRouter(prefix=...) under any cross-file
|
|
1043
|
+
// include_router prefix.
|
|
1044
|
+
const localPath = constructorPrefix ? joinPrefix(constructorPrefix, rawPath) : rawPath;
|
|
1007
1045
|
const paths = prefixSet && prefixSet.size > 0
|
|
1008
|
-
? [...prefixSet].map((p) => joinPrefix(p,
|
|
1009
|
-
: [
|
|
1046
|
+
? [...prefixSet].map((p) => joinPrefix(p, localPath))
|
|
1047
|
+
: [localPath];
|
|
1010
1048
|
for (const p of paths) {
|
|
1011
1049
|
out.push({
|
|
1012
1050
|
role: 'provider',
|
|
@@ -4,7 +4,7 @@ import { type ExportedTypeMap } from './call-processor.js';
|
|
|
4
4
|
import type { ParsedFile } from '../../_shared/index.js';
|
|
5
5
|
import { WorkerPool } from './workers/worker-pool.js';
|
|
6
6
|
import type { ParseWorkerResult, ExtractedRoute, ExtractedFetchCall, ExtractedDecoratorRoute, ExtractedToolDef, FileScopeBindings, ExtractedORMQuery, FetchWrapperDef } from './workers/parse-worker.js';
|
|
7
|
-
import type { ExtractedRouterImport, ExtractedRouterInclude, ExtractedRouterModuleAlias } from './route-extractors/fastapi-router-bindings.js';
|
|
7
|
+
import type { ExtractedRouterConstructorPrefix, ExtractedRouterImport, ExtractedRouterInclude, ExtractedRouterModuleAlias } from './route-extractors/fastapi-router-bindings.js';
|
|
8
8
|
import type { SharedSpringType } from './route-extractors/spring-shared.js';
|
|
9
9
|
export type FileProgressCallback = (current: number, total: number, filePath: string) => void;
|
|
10
10
|
export interface WorkerExtractedData {
|
|
@@ -14,6 +14,7 @@ export interface WorkerExtractedData {
|
|
|
14
14
|
decoratorRoutes: ExtractedDecoratorRoute[];
|
|
15
15
|
routerIncludes: ExtractedRouterInclude[];
|
|
16
16
|
routerImports: ExtractedRouterImport[];
|
|
17
|
+
routerConstructorPrefixes: ExtractedRouterConstructorPrefix[];
|
|
17
18
|
routerModuleAliases: ExtractedRouterModuleAlias[];
|
|
18
19
|
toolDefs: ExtractedToolDef[];
|
|
19
20
|
ormQueries: ExtractedORMQuery[];
|
|
@@ -22,6 +22,7 @@ export const mergeChunkResults = (graph, symbolTable, chunkResults, exportedType
|
|
|
22
22
|
const allDecoratorRoutes = [];
|
|
23
23
|
const allRouterIncludes = [];
|
|
24
24
|
const allRouterImports = [];
|
|
25
|
+
const allRouterConstructorPrefixes = [];
|
|
25
26
|
const allRouterModuleAliases = [];
|
|
26
27
|
const allSpringTypes = [];
|
|
27
28
|
const allToolDefs = [];
|
|
@@ -70,6 +71,9 @@ export const mergeChunkResults = (graph, symbolTable, chunkResults, exportedType
|
|
|
70
71
|
allRouterIncludes.push(item);
|
|
71
72
|
for (const item of result.routerImports ?? [])
|
|
72
73
|
allRouterImports.push(item);
|
|
74
|
+
for (const item of result.routerConstructorPrefixes ?? []) {
|
|
75
|
+
allRouterConstructorPrefixes.push(item);
|
|
76
|
+
}
|
|
73
77
|
for (const item of result.routerModuleAliases ?? [])
|
|
74
78
|
allRouterModuleAliases.push(item);
|
|
75
79
|
for (const item of result.springTypes ?? [])
|
|
@@ -93,6 +97,7 @@ export const mergeChunkResults = (graph, symbolTable, chunkResults, exportedType
|
|
|
93
97
|
decoratorRoutes: allDecoratorRoutes,
|
|
94
98
|
routerIncludes: allRouterIncludes,
|
|
95
99
|
routerImports: allRouterImports,
|
|
100
|
+
routerConstructorPrefixes: allRouterConstructorPrefixes,
|
|
96
101
|
routerModuleAliases: allRouterModuleAliases,
|
|
97
102
|
toolDefs: allToolDefs,
|
|
98
103
|
ormQueries: allORMQueries,
|
|
@@ -27,6 +27,7 @@ import { isLanguageAvailable, isGrammarRuntimeSkipped, createParserForLanguage,
|
|
|
27
27
|
import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
|
|
28
28
|
import { getProvider, providers } from '../languages/index.js';
|
|
29
29
|
import { createWorkerPool, workerPoolDisabledByEnv, resolveAutoPoolSize, WorkerPoolInitializationError, WorkerPoolDisabledError, } from '../workers/worker-pool.js';
|
|
30
|
+
import { normalizeExtractedRoutePath } from '../route-extractors/route-path.js';
|
|
30
31
|
import { resolveInheritedSpringRoutes, } from '../route-extractors/spring-shared.js';
|
|
31
32
|
import fs from 'node:fs';
|
|
32
33
|
import path from 'node:path';
|
|
@@ -461,6 +462,7 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
461
462
|
const allDecoratorRoutes = [];
|
|
462
463
|
const allRouterIncludes = [];
|
|
463
464
|
const allRouterImports = [];
|
|
465
|
+
const allRouterConstructorPrefixes = [];
|
|
464
466
|
const allRouterModuleAliases = [];
|
|
465
467
|
const allSpringTypes = [];
|
|
466
468
|
const allToolDefs = [];
|
|
@@ -599,6 +601,11 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
599
601
|
for (const item of chunkWorkerData.routerImports)
|
|
600
602
|
allRouterImports.push(item);
|
|
601
603
|
}
|
|
604
|
+
if (chunkWorkerData.routerConstructorPrefixes?.length) {
|
|
605
|
+
for (const item of chunkWorkerData.routerConstructorPrefixes) {
|
|
606
|
+
allRouterConstructorPrefixes.push(item);
|
|
607
|
+
}
|
|
608
|
+
}
|
|
602
609
|
if (chunkWorkerData.routerModuleAliases?.length) {
|
|
603
610
|
for (const item of chunkWorkerData.routerModuleAliases)
|
|
604
611
|
allRouterModuleAliases.push(item);
|
|
@@ -932,7 +939,8 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
932
939
|
// decorator inherits its file-basename's prefix. When a router is mounted
|
|
933
940
|
// under multiple prefixes we duplicate the route entry, mirroring FastAPI's
|
|
934
941
|
// runtime behaviour.
|
|
935
|
-
if (allRouterIncludes.length > 0
|
|
942
|
+
if ((allRouterIncludes.length > 0 || allRouterConstructorPrefixes.length > 0) &&
|
|
943
|
+
allDecoratorRoutes.length > 0) {
|
|
936
944
|
const importsByFile = new Map();
|
|
937
945
|
for (const imp of allRouterImports) {
|
|
938
946
|
let m = importsByFile.get(imp.filePath);
|
|
@@ -965,6 +973,11 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
965
973
|
// without a corresponding import statement).
|
|
966
974
|
const prefixesByLongKey = new Map();
|
|
967
975
|
const prefixesByShortKey = new Map();
|
|
976
|
+
// Constructor prefixes are `router`-only (the apply gate below and the
|
|
977
|
+
// group-layer tree-sitter both pin to the literal name `router`), so a
|
|
978
|
+
// flat file-key → prefix map suffices — mirrors the group layer's shape.
|
|
979
|
+
const constructorPrefixesByLongKey = new Map();
|
|
980
|
+
const constructorPrefixesByShortKey = new Map();
|
|
968
981
|
const recordPrefix = (target, key, prefix) => {
|
|
969
982
|
let set = target.get(key);
|
|
970
983
|
if (!set) {
|
|
@@ -1004,7 +1017,9 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
1004
1017
|
recordPrefix(prefixesByShortKey, localImp.moduleKey, inc.prefix);
|
|
1005
1018
|
}
|
|
1006
1019
|
}
|
|
1007
|
-
if (prefixesByLongKey.size > 0 ||
|
|
1020
|
+
if (prefixesByLongKey.size > 0 ||
|
|
1021
|
+
prefixesByShortKey.size > 0 ||
|
|
1022
|
+
allRouterConstructorPrefixes.length > 0) {
|
|
1008
1023
|
const fileLongKey = (rel) => {
|
|
1009
1024
|
// Strip `.py`, then take the last two path segments. `api/users.py`
|
|
1010
1025
|
// → `api/users`. Files at the repo root return the empty string,
|
|
@@ -1025,6 +1040,15 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
1025
1040
|
const file = slash >= 0 ? rel.slice(slash + 1) : rel;
|
|
1026
1041
|
return file.endsWith('.py') ? file.slice(0, -3) : file;
|
|
1027
1042
|
};
|
|
1043
|
+
for (const ctor of allRouterConstructorPrefixes) {
|
|
1044
|
+
const longKey = fileLongKey(ctor.filePath);
|
|
1045
|
+
if (longKey) {
|
|
1046
|
+
constructorPrefixesByLongKey.set(longKey, ctor.prefix);
|
|
1047
|
+
}
|
|
1048
|
+
else {
|
|
1049
|
+
constructorPrefixesByShortKey.set(fileShortKey(ctor.filePath), ctor.prefix);
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1028
1052
|
const expanded = [];
|
|
1029
1053
|
for (const dr of allDecoratorRoutes) {
|
|
1030
1054
|
if (dr.decoratorReceiver !== 'router' || !dr.filePath.endsWith('.py')) {
|
|
@@ -1040,12 +1064,24 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
1040
1064
|
? undefined
|
|
1041
1065
|
: prefixesByShortKey.get(fileShortKey(dr.filePath));
|
|
1042
1066
|
const prefixes = longPrefixes ?? shortPrefixes;
|
|
1067
|
+
// Constructor prefixes are keyed like include_router prefixes:
|
|
1068
|
+
// long-key entries are precise, while short-key entries are only
|
|
1069
|
+
// valid for repo-root/single-segment files where `fileLongKey`
|
|
1070
|
+
// returns ''. Do not fall back from a missing long-key match to the
|
|
1071
|
+
// short key or a root `users.py` prefix can leak onto
|
|
1072
|
+
// `admin/users.py`.
|
|
1073
|
+
const constructorPrefix = longKey
|
|
1074
|
+
? constructorPrefixesByLongKey.get(longKey)
|
|
1075
|
+
: constructorPrefixesByShortKey.get(fileShortKey(dr.filePath));
|
|
1076
|
+
const routePath = constructorPrefix
|
|
1077
|
+
? normalizeExtractedRoutePath(dr.routePath, constructorPrefix)
|
|
1078
|
+
: dr.routePath;
|
|
1043
1079
|
if (!prefixes || prefixes.size === 0) {
|
|
1044
|
-
expanded.push(dr);
|
|
1080
|
+
expanded.push(routePath === dr.routePath ? dr : { ...dr, routePath });
|
|
1045
1081
|
continue;
|
|
1046
1082
|
}
|
|
1047
1083
|
for (const prefix of prefixes) {
|
|
1048
|
-
expanded.push({ ...dr, prefix });
|
|
1084
|
+
expanded.push({ ...dr, routePath, prefix });
|
|
1049
1085
|
}
|
|
1050
1086
|
}
|
|
1051
1087
|
allDecoratorRoutes.length = 0;
|
|
@@ -101,6 +101,10 @@ export interface ExtractedRouterModuleAlias {
|
|
|
101
101
|
/** Long key (`<parent>/<stem>`) — non-empty for every emitted record. */
|
|
102
102
|
moduleKeyLong: string;
|
|
103
103
|
}
|
|
104
|
+
export interface ExtractedRouterConstructorPrefix {
|
|
105
|
+
filePath: string;
|
|
106
|
+
prefix: string;
|
|
107
|
+
}
|
|
104
108
|
/**
|
|
105
109
|
* Last `.`-separated segment of a (possibly relative) Python module
|
|
106
110
|
* path. Strips any leading dots first so `from .api.assistant import
|
|
@@ -131,4 +135,4 @@ export declare function lastTwoSegmentsAsPath(text: string): string;
|
|
|
131
135
|
* skips the alias collection — this keeps the function signature
|
|
132
136
|
* back-compat with older callers (and the parse-cache replay path).
|
|
133
137
|
*/
|
|
134
|
-
export declare function extractFastAPIRouterBindings(filePath: string, content: string, outIncludes: ExtractedRouterInclude[], outImports: ExtractedRouterImport[], outModuleAliases?: ExtractedRouterModuleAlias[]): void;
|
|
138
|
+
export declare function extractFastAPIRouterBindings(filePath: string, content: string, outIncludes: ExtractedRouterInclude[], outImports: ExtractedRouterImport[], outModuleAliases?: ExtractedRouterModuleAlias[], outConstructorPrefixes?: ExtractedRouterConstructorPrefix[]): void;
|
|
@@ -59,6 +59,8 @@ const INCLUDE_ROUTER_NAME_RE = /\b(?:[A-Za-z_][\w.]*)\.include_router\s*\(\s*([A
|
|
|
59
59
|
// The latter is the common case and the only one we can map back to
|
|
60
60
|
// a module stem.
|
|
61
61
|
const FROM_IMPORT_ROUTER_RE = /^\s*from\s+(\.+|\.*[A-Za-z_][\w.]*)\s+import\s+([^#\n]+)/gm;
|
|
62
|
+
const API_ROUTER_ASSIGN_RE = /\brouter\s*=\s*APIRouter\s*\(/g;
|
|
63
|
+
const API_ROUTER_PREFIX_ARG_RE = /\bprefix\s*=\s*(['"])([^'"]*)\1/;
|
|
62
64
|
/**
|
|
63
65
|
* Last `.`-separated segment of a (possibly relative) Python module
|
|
64
66
|
* path. Strips any leading dots first so `from .api.assistant import
|
|
@@ -94,6 +96,36 @@ export function lastTwoSegmentsAsPath(text) {
|
|
|
94
96
|
const parent = prev >= 0 ? beforeLast.slice(prev + 1) : beforeLast;
|
|
95
97
|
return `${parent}/${stem}`;
|
|
96
98
|
}
|
|
99
|
+
function findMatchingParen(content, openIndex) {
|
|
100
|
+
let depth = 0;
|
|
101
|
+
let quote = null;
|
|
102
|
+
for (let i = openIndex; i < content.length; i++) {
|
|
103
|
+
const ch = content[i];
|
|
104
|
+
if (quote) {
|
|
105
|
+
if (ch === '\\') {
|
|
106
|
+
i++;
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
if (ch === quote)
|
|
110
|
+
quote = null;
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
if (ch === '"' || ch === "'") {
|
|
114
|
+
quote = ch;
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
if (ch === '(' || ch === '[' || ch === '{') {
|
|
118
|
+
depth++;
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
if (ch === ')' || ch === ']' || ch === '}') {
|
|
122
|
+
depth--;
|
|
123
|
+
if (depth === 0 && ch === ')')
|
|
124
|
+
return i;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return -1;
|
|
128
|
+
}
|
|
97
129
|
/**
|
|
98
130
|
* Scan a single Python file's source text for FastAPI router
|
|
99
131
|
* `include_router` sites and `from <module> import router` imports,
|
|
@@ -107,7 +139,7 @@ export function lastTwoSegmentsAsPath(text) {
|
|
|
107
139
|
* skips the alias collection — this keeps the function signature
|
|
108
140
|
* back-compat with older callers (and the parse-cache replay path).
|
|
109
141
|
*/
|
|
110
|
-
export function extractFastAPIRouterBindings(filePath, content, outIncludes, outImports, outModuleAliases) {
|
|
142
|
+
export function extractFastAPIRouterBindings(filePath, content, outIncludes, outImports, outModuleAliases, outConstructorPrefixes) {
|
|
111
143
|
if (!content.includes('include_router') && !content.includes('router'))
|
|
112
144
|
return;
|
|
113
145
|
// `from <module> import router [as <alias>]`. We capture every name
|
|
@@ -172,6 +204,25 @@ export function extractFastAPIRouterBindings(filePath, content, outIncludes, out
|
|
|
172
204
|
}
|
|
173
205
|
}
|
|
174
206
|
}
|
|
207
|
+
if (outConstructorPrefixes && content.includes('APIRouter') && content.includes('prefix')) {
|
|
208
|
+
API_ROUTER_ASSIGN_RE.lastIndex = 0;
|
|
209
|
+
// Only `router = APIRouter(...)` is captured (the apply gate and the
|
|
210
|
+
// group-layer tree-sitter both pin to the literal name `router`).
|
|
211
|
+
while (API_ROUTER_ASSIGN_RE.exec(content) !== null) {
|
|
212
|
+
const openParen = API_ROUTER_ASSIGN_RE.lastIndex - 1;
|
|
213
|
+
const closeParen = findMatchingParen(content, openParen);
|
|
214
|
+
if (closeParen < 0)
|
|
215
|
+
continue;
|
|
216
|
+
const args = content.slice(openParen + 1, closeParen);
|
|
217
|
+
const prefixMatch = API_ROUTER_PREFIX_ARG_RE.exec(args);
|
|
218
|
+
if (!prefixMatch)
|
|
219
|
+
continue;
|
|
220
|
+
outConstructorPrefixes.push({
|
|
221
|
+
filePath,
|
|
222
|
+
prefix: prefixMatch[2],
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
}
|
|
175
226
|
if (!content.includes('include_router'))
|
|
176
227
|
return;
|
|
177
228
|
// Shape A: `<host>.include_router(<module>.router, prefix='/x')`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { SupportedLanguages } from '../../../_shared/index.js';
|
|
2
2
|
import type { SkippedPath } from './clone-safety.js';
|
|
3
|
-
import type { ExtractedRouterInclude, ExtractedRouterImport, ExtractedRouterModuleAlias } from '../route-extractors/fastapi-router-bindings.js';
|
|
3
|
+
import type { ExtractedRouterConstructorPrefix, ExtractedRouterInclude, ExtractedRouterImport, ExtractedRouterModuleAlias } from '../route-extractors/fastapi-router-bindings.js';
|
|
4
4
|
import { type MixedChainStep } from '../utils/call-analysis.js';
|
|
5
5
|
import type { ConstructorBinding } from '../type-env.js';
|
|
6
6
|
import type { NodeLabel, ParameterTypeClass } from '../../../_shared/index.js';
|
|
@@ -191,6 +191,7 @@ export interface ParseWorkerResult {
|
|
|
191
191
|
decoratorRoutes: ExtractedDecoratorRoute[];
|
|
192
192
|
routerIncludes: ExtractedRouterInclude[];
|
|
193
193
|
routerImports: ExtractedRouterImport[];
|
|
194
|
+
routerConstructorPrefixes?: ExtractedRouterConstructorPrefix[];
|
|
194
195
|
/**
|
|
195
196
|
* Optional. Project-wide `SharedSpringType` view of route-defining
|
|
196
197
|
* class/interface declarations, produced by the provider's
|
|
@@ -504,6 +504,7 @@ const processBatch = (files, onProgress) => {
|
|
|
504
504
|
decoratorRoutes: [],
|
|
505
505
|
routerIncludes: [],
|
|
506
506
|
routerImports: [],
|
|
507
|
+
routerConstructorPrefixes: [],
|
|
507
508
|
routerModuleAliases: [],
|
|
508
509
|
toolDefs: [],
|
|
509
510
|
ormQueries: [],
|
|
@@ -1821,7 +1822,7 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
1821
1822
|
// injects the resolved prefix onto each ExtractedDecoratorRoute that
|
|
1822
1823
|
// came from a `@router.<verb>` decorator. Python-only.
|
|
1823
1824
|
if (language === SupportedLanguages.Python) {
|
|
1824
|
-
extractFastAPIRouterBindings(file.path, parseContent, result.routerIncludes, result.routerImports, (result.routerModuleAliases ??= []));
|
|
1825
|
+
extractFastAPIRouterBindings(file.path, parseContent, result.routerIncludes, result.routerImports, (result.routerModuleAliases ??= []), (result.routerConstructorPrefixes ??= []));
|
|
1825
1826
|
}
|
|
1826
1827
|
// Language-specific decorator route extraction via provider hook.
|
|
1827
1828
|
// The provider's extractDecoratorRoutes walks the AST for framework-specific
|
|
@@ -1870,6 +1871,7 @@ let accumulated = {
|
|
|
1870
1871
|
decoratorRoutes: [],
|
|
1871
1872
|
routerIncludes: [],
|
|
1872
1873
|
routerImports: [],
|
|
1874
|
+
routerConstructorPrefixes: [],
|
|
1873
1875
|
routerModuleAliases: [],
|
|
1874
1876
|
toolDefs: [],
|
|
1875
1877
|
ormQueries: [],
|
|
@@ -1989,6 +1991,7 @@ parentPort.on('message', (msg) => {
|
|
|
1989
1991
|
decoratorRoutes: [],
|
|
1990
1992
|
routerIncludes: [],
|
|
1991
1993
|
routerImports: [],
|
|
1994
|
+
routerConstructorPrefixes: [],
|
|
1992
1995
|
routerModuleAliases: [],
|
|
1993
1996
|
toolDefs: [],
|
|
1994
1997
|
ormQueries: [],
|
|
@@ -24,6 +24,10 @@ export const mergeResult = (target, src) => {
|
|
|
24
24
|
appendAll(target.routerIncludes, src.routerIncludes);
|
|
25
25
|
if (src.routerImports)
|
|
26
26
|
appendAll(target.routerImports, src.routerImports);
|
|
27
|
+
if (src.routerConstructorPrefixes) {
|
|
28
|
+
target.routerConstructorPrefixes ??= [];
|
|
29
|
+
appendAll(target.routerConstructorPrefixes, src.routerConstructorPrefixes);
|
|
30
|
+
}
|
|
27
31
|
if (src.routerModuleAliases) {
|
|
28
32
|
target.routerModuleAliases ??= [];
|
|
29
33
|
appendAll(target.routerModuleAliases, src.routerModuleAliases);
|
|
@@ -286,6 +286,12 @@ export declare const loadFTSExtension: (targetConn?: lbug.Connection, opts?: Ext
|
|
|
286
286
|
* unavailable so semantic search can fall back to exact scan.
|
|
287
287
|
*/
|
|
288
288
|
export declare const loadVectorExtension: (targetConn?: lbug.Connection, opts?: ExtensionEnsureOptions) => Promise<boolean>;
|
|
289
|
+
/**
|
|
290
|
+
* Default stemmer for FTS indexes. Single source so the analyze path
|
|
291
|
+
* (`getSearchFTSStemmer`) and the read-only `createFTSIndex`/`ensureFTSIndex`
|
|
292
|
+
* defaults can never silently diverge.
|
|
293
|
+
*/
|
|
294
|
+
export declare const DEFAULT_FTS_STEMMER = "porter";
|
|
289
295
|
/**
|
|
290
296
|
* Create a full-text search index on a table
|
|
291
297
|
* @param tableName - The node table name (e.g., 'File', 'CodeSymbol')
|
|
@@ -2081,6 +2081,12 @@ export const loadVectorExtension = async (targetConn, opts = {}) => {
|
|
|
2081
2081
|
vectorExtensionLoaded = true;
|
|
2082
2082
|
return loaded;
|
|
2083
2083
|
};
|
|
2084
|
+
/**
|
|
2085
|
+
* Default stemmer for FTS indexes. Single source so the analyze path
|
|
2086
|
+
* (`getSearchFTSStemmer`) and the read-only `createFTSIndex`/`ensureFTSIndex`
|
|
2087
|
+
* defaults can never silently diverge.
|
|
2088
|
+
*/
|
|
2089
|
+
export const DEFAULT_FTS_STEMMER = 'porter';
|
|
2084
2090
|
/**
|
|
2085
2091
|
* Create a full-text search index on a table
|
|
2086
2092
|
* @param tableName - The node table name (e.g., 'File', 'CodeSymbol')
|
|
@@ -2088,7 +2094,7 @@ export const loadVectorExtension = async (targetConn, opts = {}) => {
|
|
|
2088
2094
|
* @param properties - List of properties to index (e.g., ['name', 'code'])
|
|
2089
2095
|
* @param stemmer - Stemming algorithm (default: 'porter')
|
|
2090
2096
|
*/
|
|
2091
|
-
export const createFTSIndex = async (tableName, indexName, properties, stemmer =
|
|
2097
|
+
export const createFTSIndex = async (tableName, indexName, properties, stemmer = DEFAULT_FTS_STEMMER) => {
|
|
2092
2098
|
if (!conn) {
|
|
2093
2099
|
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
2094
2100
|
}
|
|
@@ -2176,7 +2182,7 @@ export const createVectorIndex = async () => {
|
|
|
2176
2182
|
* the index is owned by `gitnexus analyze` (writable) and either already
|
|
2177
2183
|
* exists or will be created on the next analyze.
|
|
2178
2184
|
*/
|
|
2179
|
-
export const ensureFTSIndex = async (tableName, indexName, properties, stemmer =
|
|
2185
|
+
export const ensureFTSIndex = async (tableName, indexName, properties, stemmer = DEFAULT_FTS_STEMMER) => {
|
|
2180
2186
|
const key = ftsIndexKey(tableName, indexName);
|
|
2181
2187
|
if (ensuredFTSIndexes.has(key))
|
|
2182
2188
|
return;
|
package/dist/core/run-analyze.js
CHANGED
|
@@ -14,7 +14,7 @@ import { execFileSync } from 'child_process';
|
|
|
14
14
|
import { runPipelineFromRepo } from './ingestion/pipeline.js';
|
|
15
15
|
import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js';
|
|
16
16
|
import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReusedStatement, closeLbug, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFile, deleteAllCommunitiesAndProcesses, deleteAllInterprocTaintPaths, deleteAllCallSummaries, queryImporters, loadFTSExtension, } from './lbug/lbug-adapter.js';
|
|
17
|
-
import { createSearchFTSIndexes, verifySearchFTSIndexes } from './search/fts-indexes.js';
|
|
17
|
+
import { createSearchFTSIndexes, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js';
|
|
18
18
|
import { resolveAnalyzeInstallPolicy } from './lbug/extension-loader.js';
|
|
19
19
|
import { startWalCheckpointDriver, } from './lbug/wal-checkpoint-driver.js';
|
|
20
20
|
import { getStoragePaths, resolveBranchPlacement, saveMeta, loadMeta, ensureGitNexusIgnored, registerRepo, isRepoRegistered, cleanupOldKuzuFiles, INCREMENTAL_SCHEMA_VERSION, } from '../storage/repo-manager.js';
|
|
@@ -255,6 +255,10 @@ export const pdgModeMismatch = (recorded, options) => {
|
|
|
255
255
|
export async function runFullAnalysis(repoPath, options, callbacks) {
|
|
256
256
|
const log = (msg) => callbacks.onLog?.(msg);
|
|
257
257
|
const progress = (phase, percent, message) => callbacks.onProgress(phase, percent, message);
|
|
258
|
+
// Resolve + validate operator-provided FTS config once, before the expensive
|
|
259
|
+
// parse/load phases. A typo fails here in ms; createSearchFTSIndexes reuses
|
|
260
|
+
// the cached value via getSearchFTSStemmer.
|
|
261
|
+
initialiseSearchFTSStemmer();
|
|
258
262
|
// Scope the degraded-parse log throttle to this run. On a reused process
|
|
259
263
|
// (e.g. tests, or any host that calls runFullAnalysis more than once) the
|
|
260
264
|
// module-level counter would otherwise stay saturated and suppress every
|
|
@@ -2,5 +2,19 @@ export interface CreateSearchFTSIndexesOptions {
|
|
|
2
2
|
onIndexStart?: (table: string, indexName: string) => void;
|
|
3
3
|
onIndexReady?: (table: string, indexName: string) => void;
|
|
4
4
|
}
|
|
5
|
+
/**
|
|
6
|
+
* Resolve + validate `GITNEXUS_FTS_STEMMER` once, up front at analyze startup,
|
|
7
|
+
* and cache it. An invalid value throws here — in milliseconds — instead of
|
|
8
|
+
* ~85% into a run (after the expensive parse/scope-resolution work). The cached
|
|
9
|
+
* value is what {@link getSearchFTSStemmer} returns for the rest of the run, so
|
|
10
|
+
* config is read and validated in exactly one place.
|
|
11
|
+
*/
|
|
12
|
+
export declare function initialiseSearchFTSStemmer(): string;
|
|
13
|
+
/**
|
|
14
|
+
* Return the stemmer resolved by {@link initialiseSearchFTSStemmer}. Falls back
|
|
15
|
+
* to resolving on demand when init was never called (read-only hosts, unit
|
|
16
|
+
* tests) so validation always applies.
|
|
17
|
+
*/
|
|
18
|
+
export declare function getSearchFTSStemmer(): string;
|
|
5
19
|
export declare function createSearchFTSIndexes(options?: CreateSearchFTSIndexesOptions): Promise<void>;
|
|
6
20
|
export declare function verifySearchFTSIndexes(executeQuery: (cypher: string) => Promise<unknown[]>): Promise<string[]>;
|
|
@@ -1,6 +1,71 @@
|
|
|
1
|
-
import { createFTSIndex, dropFTSIndex } from '../lbug/lbug-adapter.js';
|
|
1
|
+
import { createFTSIndex, dropFTSIndex, DEFAULT_FTS_STEMMER } from '../lbug/lbug-adapter.js';
|
|
2
2
|
import { FTS_INDEXES } from './fts-schema.js';
|
|
3
|
+
// Stemmers shipped by the LadybugDB FTS extension. Mirrors the lowercase token
|
|
4
|
+
// set in the extension bundled with @ladybugdb/core 0.17.x (see package.json).
|
|
5
|
+
// Keep in sync on a LadybugDB minor bump — a value here that the installed
|
|
6
|
+
// extension rejects would pass validation but fail at CREATE_FTS_INDEX.
|
|
7
|
+
const SUPPORTED_FTS_STEMMERS = new Set([
|
|
8
|
+
'arabic',
|
|
9
|
+
'basque',
|
|
10
|
+
'catalan',
|
|
11
|
+
'danish',
|
|
12
|
+
'dutch',
|
|
13
|
+
'english',
|
|
14
|
+
'finnish',
|
|
15
|
+
'french',
|
|
16
|
+
'german',
|
|
17
|
+
'greek',
|
|
18
|
+
'hindi',
|
|
19
|
+
'hungarian',
|
|
20
|
+
'indonesian',
|
|
21
|
+
'irish',
|
|
22
|
+
'italian',
|
|
23
|
+
'lithuanian',
|
|
24
|
+
'nepali',
|
|
25
|
+
'norwegian',
|
|
26
|
+
'none',
|
|
27
|
+
'porter',
|
|
28
|
+
'portuguese',
|
|
29
|
+
'romanian',
|
|
30
|
+
'russian',
|
|
31
|
+
'serbian',
|
|
32
|
+
'spanish',
|
|
33
|
+
'swedish',
|
|
34
|
+
'tamil',
|
|
35
|
+
'turkish',
|
|
36
|
+
]);
|
|
37
|
+
let resolvedStemmer;
|
|
38
|
+
/** Read + validate `GITNEXUS_FTS_STEMMER`. Throws on an unsupported value. */
|
|
39
|
+
function resolveFTSStemmer() {
|
|
40
|
+
const raw = process.env.GITNEXUS_FTS_STEMMER?.trim().toLowerCase();
|
|
41
|
+
if (!raw)
|
|
42
|
+
return DEFAULT_FTS_STEMMER;
|
|
43
|
+
if (SUPPORTED_FTS_STEMMERS.has(raw))
|
|
44
|
+
return raw;
|
|
45
|
+
throw new Error(`Invalid GITNEXUS_FTS_STEMMER "${process.env.GITNEXUS_FTS_STEMMER}". ` +
|
|
46
|
+
`Expected one of: ${[...SUPPORTED_FTS_STEMMERS].sort().join(', ')}.`);
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Resolve + validate `GITNEXUS_FTS_STEMMER` once, up front at analyze startup,
|
|
50
|
+
* and cache it. An invalid value throws here — in milliseconds — instead of
|
|
51
|
+
* ~85% into a run (after the expensive parse/scope-resolution work). The cached
|
|
52
|
+
* value is what {@link getSearchFTSStemmer} returns for the rest of the run, so
|
|
53
|
+
* config is read and validated in exactly one place.
|
|
54
|
+
*/
|
|
55
|
+
export function initialiseSearchFTSStemmer() {
|
|
56
|
+
resolvedStemmer = resolveFTSStemmer();
|
|
57
|
+
return resolvedStemmer;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Return the stemmer resolved by {@link initialiseSearchFTSStemmer}. Falls back
|
|
61
|
+
* to resolving on demand when init was never called (read-only hosts, unit
|
|
62
|
+
* tests) so validation always applies.
|
|
63
|
+
*/
|
|
64
|
+
export function getSearchFTSStemmer() {
|
|
65
|
+
return resolvedStemmer ?? resolveFTSStemmer();
|
|
66
|
+
}
|
|
3
67
|
export async function createSearchFTSIndexes(options) {
|
|
68
|
+
const stemmer = getSearchFTSStemmer();
|
|
4
69
|
for (const { table, indexName, properties } of FTS_INDEXES) {
|
|
5
70
|
options?.onIndexStart?.(table, indexName);
|
|
6
71
|
// Drop first so the live `properties` always win. `createFTSIndex` is
|
|
@@ -15,7 +80,7 @@ export async function createSearchFTSIndexes(options) {
|
|
|
15
80
|
// runs inside the existing FTS phase. Gate on a stored schema fingerprint if
|
|
16
81
|
// this rebuild cost ever shows up in analyze profiles.
|
|
17
82
|
await dropFTSIndex(table, indexName);
|
|
18
|
-
await createFTSIndex(table, indexName, [...properties]);
|
|
83
|
+
await createFTSIndex(table, indexName, [...properties], stemmer);
|
|
19
84
|
options?.onIndexReady?.(table, indexName);
|
|
20
85
|
}
|
|
21
86
|
}
|
|
@@ -52,7 +52,7 @@ import { fileURLToPath } from 'url';
|
|
|
52
52
|
// the main thread (the #1983 OOM). Because the two stores share this version,
|
|
53
53
|
// any future change to the `ParsedFile` serialization shape MUST bump
|
|
54
54
|
// SCHEMA_BUMP so both invalidate in lockstep.
|
|
55
|
-
const SCHEMA_BUMP =
|
|
55
|
+
const SCHEMA_BUMP = 9; // #2312: ParseWorkerResult gained `routerConstructorPrefixes` for FastAPI APIRouter(prefix=...) replay
|
|
56
56
|
const GITNEXUS_PKG_VERSION = (() => {
|
|
57
57
|
try {
|
|
58
58
|
// package.json sits at gitnexus/package.json — two levels up from
|
package/package.json
CHANGED