gitnexus 1.6.10-rc.7 → 1.6.10-rc.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/group/extractors/http-patterns/python.js +246 -108
- package/dist/core/ingestion/parsing-processor.d.ts +3 -1
- package/dist/core/ingestion/parsing-processor.js +4 -0
- package/dist/core/ingestion/pipeline-phases/parse-impl.js +42 -0
- package/dist/core/ingestion/route-extractors/constant-resolver.d.ts +83 -0
- package/dist/core/ingestion/route-extractors/constant-resolver.js +131 -0
- package/dist/core/ingestion/route-extractors/python-const-resolver.d.ts +75 -0
- package/dist/core/ingestion/route-extractors/python-const-resolver.js +299 -0
- package/dist/core/ingestion/tree-sitter-queries.d.ts +1 -1
- package/dist/core/ingestion/tree-sitter-queries.js +12 -1
- package/dist/core/ingestion/workers/parse-worker.d.ts +34 -0
- package/dist/core/ingestion/workers/parse-worker.js +39 -4
- package/dist/core/ingestion/workers/result-merge.js +4 -0
- package/dist/storage/parse-cache.js +1 -1
- package/package.json +1 -1
- package/scripts/cross-platform-tests.ts +3 -0
- package/scripts/ensure-fts.ts +32 -0
- package/scripts/run-cross-platform.ts +37 -6
- package/scripts/shard-arg.ts +30 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import Python from 'tree-sitter-python';
|
|
2
2
|
import { compilePatterns, runCompiledPatterns, unquoteLiteral, } from '../tree-sitter-scanner.js';
|
|
3
3
|
import { normalizeExtractedRoutePath } from '../../../ingestion/route-extractors/route-path.js';
|
|
4
|
+
import { extractPythonModuleConstants, parseConstOperands, resolveOperands, } from '../../../ingestion/route-extractors/python-const-resolver.js';
|
|
4
5
|
/**
|
|
5
6
|
* Python HTTP plugin. Handles:
|
|
6
7
|
* - FastAPI `@app.get("/path")` provider decorators
|
|
@@ -67,6 +68,44 @@ const FASTAPI_ROUTER_PATTERNS = compilePatterns({
|
|
|
67
68
|
},
|
|
68
69
|
],
|
|
69
70
|
});
|
|
71
|
+
// #2391: `@router.<verb>` / `@app.<verb>` whose first argument is a non-literal
|
|
72
|
+
// path — a bare imported constant or a `+`-concatenation. The path is resolved
|
|
73
|
+
// against the repo-wide constant map (parity with the ingestion side) and, on
|
|
74
|
+
// failure, the route is skipped (no provider contract) exactly like ingestion.
|
|
75
|
+
const FASTAPI_ROUTER_EXPR_PATTERNS = compilePatterns({
|
|
76
|
+
name: 'python-fastapi-router-expr',
|
|
77
|
+
language: Python,
|
|
78
|
+
patterns: [
|
|
79
|
+
{
|
|
80
|
+
meta: {},
|
|
81
|
+
query: `
|
|
82
|
+
(decorator
|
|
83
|
+
(call
|
|
84
|
+
function: (attribute
|
|
85
|
+
object: (identifier) @obj (#eq? @obj "router")
|
|
86
|
+
attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$"))
|
|
87
|
+
arguments: (argument_list . [(identifier) (binary_operator)] @path)))
|
|
88
|
+
`,
|
|
89
|
+
},
|
|
90
|
+
],
|
|
91
|
+
});
|
|
92
|
+
const FASTAPI_APP_EXPR_PATTERNS = compilePatterns({
|
|
93
|
+
name: 'python-fastapi-app-expr',
|
|
94
|
+
language: Python,
|
|
95
|
+
patterns: [
|
|
96
|
+
{
|
|
97
|
+
meta: {},
|
|
98
|
+
query: `
|
|
99
|
+
(decorator
|
|
100
|
+
(call
|
|
101
|
+
function: (attribute
|
|
102
|
+
object: (identifier) @obj (#eq? @obj "app")
|
|
103
|
+
attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$"))
|
|
104
|
+
arguments: (argument_list . [(identifier) (binary_operator)] @path)))
|
|
105
|
+
`,
|
|
106
|
+
},
|
|
107
|
+
],
|
|
108
|
+
});
|
|
70
109
|
// ─── Provider: Flask `app.add_url_rule('/path', view_func=handler)` ───
|
|
71
110
|
// The imperative Flask route registration: unlike `@app.route` (whose handler
|
|
72
111
|
// is the decorated function, same-file), `view_func` is frequently an IMPORTED
|
|
@@ -830,111 +869,153 @@ function recordPrefix(target, key, prefix) {
|
|
|
830
869
|
set.add(prefix);
|
|
831
870
|
target.set(key, set);
|
|
832
871
|
}
|
|
872
|
+
// Cheap cost-gate pre-filter: a `@router`/`@app.<verb>(` call whose first
|
|
873
|
+
// argument is non-literal — either it STARTS with an identifier (a bare constant
|
|
874
|
+
// or the head of `CONST + "/x"`), or it is a string-literal-LEADING concat
|
|
875
|
+
// (`"/api" + SUFFIX`) detected by a `+` before the decorator's closing paren
|
|
876
|
+
// (#2393). `[^)]*` spans the whole argument, including a Black-formatted concat
|
|
877
|
+
// that wraps across lines, but stays bounded by the decorator's own `)`. Gating
|
|
878
|
+
// the literal-leading case on the `+` (not merely a leading quote) keeps a plain
|
|
879
|
+
// string route `@router.get("/x")` OFF the gate, so a literal-only repo pays no
|
|
880
|
+
// parse pass. Deliberately loose — a false positive only costs a parse; a false
|
|
881
|
+
// negative would silently drop the feature.
|
|
882
|
+
const NONLITERAL_ROUTE_DECORATOR_RE = /@\s*(?:app|router)\s*\.\s*(?:get|post|put|delete|patch)\s*\(\s*(?:[A-Za-z_]|["'][^)]*\+)/;
|
|
833
883
|
function buildPythonRepoContext(files, parser, readFile, parseSource) {
|
|
834
884
|
const prefixesByLongKey = new Map();
|
|
835
885
|
const prefixesByShortKey = new Map();
|
|
836
|
-
//
|
|
837
|
-
//
|
|
838
|
-
//
|
|
839
|
-
//
|
|
886
|
+
// Single read pass (#2393): slurp every `.py` file's content ONCE. This used to
|
|
887
|
+
// be two passes — the include_router pre-pass below and the #2391 constant cost
|
|
888
|
+
// gate each re-read every `.py` file. The composed-route cost gate is computed
|
|
889
|
+
// in the same pass so a literal-only repo still does exactly one read and zero
|
|
890
|
+
// parses.
|
|
891
|
+
const pyContents = new Map();
|
|
892
|
+
let hasComposedRoute = false;
|
|
840
893
|
for (const rel of files) {
|
|
841
894
|
if (!rel.endsWith('.py'))
|
|
842
895
|
continue;
|
|
843
896
|
const src = readFile(rel);
|
|
844
897
|
if (!src)
|
|
845
898
|
continue;
|
|
846
|
-
|
|
899
|
+
pyContents.set(rel, src);
|
|
900
|
+
if (!hasComposedRoute && NONLITERAL_ROUTE_DECORATOR_RE.test(src))
|
|
901
|
+
hasComposedRoute = true;
|
|
902
|
+
}
|
|
903
|
+
// Single PARSE pass (#2391): parse each `.py` at most once and feed BOTH the
|
|
904
|
+
// include_router prefix pre-pass and the composed-constant map below. This used
|
|
905
|
+
// to be two loops, so an include_router file in a composed repo was parsed
|
|
906
|
+
// twice. A file that needs neither pass is not parsed at all (cost gates intact).
|
|
907
|
+
//
|
|
908
|
+
// Cross-file pre-pass: only `include_router` sites need it — they bind a prefix
|
|
909
|
+
// declared in one file to a router defined in another. Same-file
|
|
910
|
+
// `APIRouter(prefix=...)` is resolved in scan() from the file's own tree.
|
|
911
|
+
const constantsByFile = new Map();
|
|
912
|
+
for (const [rel, src] of pyContents) {
|
|
913
|
+
const needsRouter = src.includes('include_router');
|
|
914
|
+
if (!needsRouter && !hasComposedRoute)
|
|
847
915
|
continue;
|
|
848
916
|
parser.setLanguage(Python);
|
|
849
917
|
const tree = parseSource(parser, src);
|
|
850
918
|
if (!tree)
|
|
851
919
|
continue;
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
const
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
// Module-alias map: name imported from a multi-segment package →
|
|
867
|
-
// long key. Lets Shape A look up the precise file for `<name>.router`
|
|
868
|
-
// even when `<name>` collides with another package's basename.
|
|
869
|
-
const localNameToModuleAlias = new Map();
|
|
870
|
-
for (const m of runCompiledPatterns(FROM_IMPORT_MODULE_PATTERNS, tree)) {
|
|
871
|
-
const moduleNode = m.captures.module;
|
|
872
|
-
const importedNode = m.captures.imported;
|
|
873
|
-
const aliasNode = m.captures.alias;
|
|
874
|
-
if (!moduleNode || !importedNode)
|
|
875
|
-
continue;
|
|
876
|
-
// Skip the `router` shape — already handled by FROM_IMPORT_ROUTER_PATTERNS
|
|
877
|
-
// above and stored under its router-aware semantics.
|
|
878
|
-
if (importedNode.text === 'router')
|
|
879
|
-
continue;
|
|
880
|
-
const moduleLong = lastTwoSegmentsAsLongKey(`${moduleNode.text}.${importedNode.text}`);
|
|
881
|
-
if (!moduleLong)
|
|
882
|
-
continue;
|
|
883
|
-
const localName = aliasNode?.text ?? importedNode.text;
|
|
884
|
-
localNameToModuleAlias.set(localName, moduleLong);
|
|
885
|
-
}
|
|
886
|
-
// Shape A: `<host>.include_router(<module>.router, prefix='/x')`.
|
|
887
|
-
// The call site gives us only a short module name. We promote to a
|
|
888
|
-
// long key when the same file imports `<module>` via either
|
|
889
|
-
// `from <pkg> import <module>` (recorded in `localNameToModuleAlias`
|
|
890
|
-
// — the typical pattern) or, less commonly, a router-aware import
|
|
891
|
-
// statement. Only fall back to the basename short key when neither
|
|
892
|
-
// alias is available.
|
|
893
|
-
for (const m of runCompiledPatterns(INCLUDE_ROUTER_ATTR_PATTERNS, tree)) {
|
|
894
|
-
const modNode = m.captures.router_module;
|
|
895
|
-
const prefixNode = m.captures.prefix;
|
|
896
|
-
if (!modNode || !prefixNode)
|
|
897
|
-
continue;
|
|
898
|
-
const prefix = unquoteLiteral(prefixNode.text);
|
|
899
|
-
if (prefix === null)
|
|
900
|
-
continue;
|
|
901
|
-
const moduleShort = modNode.text;
|
|
902
|
-
const aliasLong = localNameToModuleAlias.get(moduleShort);
|
|
903
|
-
const sameFileImport = localNameToModule.get(moduleShort);
|
|
904
|
-
const longKey = aliasLong ?? sameFileImport?.moduleLong;
|
|
905
|
-
if (longKey) {
|
|
906
|
-
recordPrefix(prefixesByLongKey, longKey, prefix);
|
|
920
|
+
if (needsRouter) {
|
|
921
|
+
const localNameToModule = new Map();
|
|
922
|
+
for (const m of runCompiledPatterns(FROM_IMPORT_ROUTER_PATTERNS, tree)) {
|
|
923
|
+
const moduleNode = m.captures.module;
|
|
924
|
+
const aliasNode = m.captures.alias;
|
|
925
|
+
const importedNode = m.captures.imported;
|
|
926
|
+
if (!moduleNode || !importedNode)
|
|
927
|
+
continue;
|
|
928
|
+
const localName = aliasNode?.text ?? importedNode.text;
|
|
929
|
+
const moduleShort = lastSegmentOfDotted(moduleNode.text);
|
|
930
|
+
if (!moduleShort)
|
|
931
|
+
continue;
|
|
932
|
+
const moduleLong = lastTwoSegmentsAsLongKey(moduleNode.text);
|
|
933
|
+
localNameToModule.set(localName, { moduleShort, moduleLong });
|
|
907
934
|
}
|
|
908
|
-
|
|
909
|
-
|
|
935
|
+
// Module-alias map: name imported from a multi-segment package →
|
|
936
|
+
// long key. Lets Shape A look up the precise file for `<name>.router`
|
|
937
|
+
// even when `<name>` collides with another package's basename.
|
|
938
|
+
const localNameToModuleAlias = new Map();
|
|
939
|
+
for (const m of runCompiledPatterns(FROM_IMPORT_MODULE_PATTERNS, tree)) {
|
|
940
|
+
const moduleNode = m.captures.module;
|
|
941
|
+
const importedNode = m.captures.imported;
|
|
942
|
+
const aliasNode = m.captures.alias;
|
|
943
|
+
if (!moduleNode || !importedNode)
|
|
944
|
+
continue;
|
|
945
|
+
// Skip the `router` shape — already handled by FROM_IMPORT_ROUTER_PATTERNS
|
|
946
|
+
// above and stored under its router-aware semantics.
|
|
947
|
+
if (importedNode.text === 'router')
|
|
948
|
+
continue;
|
|
949
|
+
const moduleLong = lastTwoSegmentsAsLongKey(`${moduleNode.text}.${importedNode.text}`);
|
|
950
|
+
if (!moduleLong)
|
|
951
|
+
continue;
|
|
952
|
+
const localName = aliasNode?.text ?? importedNode.text;
|
|
953
|
+
localNameToModuleAlias.set(localName, moduleLong);
|
|
910
954
|
}
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
const
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
955
|
+
// Shape A: `<host>.include_router(<module>.router, prefix='/x')`.
|
|
956
|
+
// The call site gives us only a short module name. We promote to a
|
|
957
|
+
// long key when the same file imports `<module>` via either
|
|
958
|
+
// `from <pkg> import <module>` (recorded in `localNameToModuleAlias`
|
|
959
|
+
// — the typical pattern) or, less commonly, a router-aware import
|
|
960
|
+
// statement. Only fall back to the basename short key when neither
|
|
961
|
+
// alias is available.
|
|
962
|
+
for (const m of runCompiledPatterns(INCLUDE_ROUTER_ATTR_PATTERNS, tree)) {
|
|
963
|
+
const modNode = m.captures.router_module;
|
|
964
|
+
const prefixNode = m.captures.prefix;
|
|
965
|
+
if (!modNode || !prefixNode)
|
|
966
|
+
continue;
|
|
967
|
+
const prefix = unquoteLiteral(prefixNode.text);
|
|
968
|
+
if (prefix === null)
|
|
969
|
+
continue;
|
|
970
|
+
const moduleShort = modNode.text;
|
|
971
|
+
const aliasLong = localNameToModuleAlias.get(moduleShort);
|
|
972
|
+
const sameFileImport = localNameToModule.get(moduleShort);
|
|
973
|
+
const longKey = aliasLong ?? sameFileImport?.moduleLong;
|
|
974
|
+
if (longKey) {
|
|
975
|
+
recordPrefix(prefixesByLongKey, longKey, prefix);
|
|
976
|
+
}
|
|
977
|
+
else {
|
|
978
|
+
recordPrefix(prefixesByShortKey, moduleShort, prefix);
|
|
979
|
+
}
|
|
929
980
|
}
|
|
930
|
-
|
|
931
|
-
|
|
981
|
+
// Shape B: `<host>.include_router(my_router, prefix='/x')` — resolve
|
|
982
|
+
// `my_router` via the import map built above. Whenever the import
|
|
983
|
+
// statement supplied a multi-segment module path the long key is
|
|
984
|
+
// recorded, eliminating cross-package collisions.
|
|
985
|
+
for (const m of runCompiledPatterns(INCLUDE_ROUTER_NAME_PATTERNS, tree)) {
|
|
986
|
+
const nameNode = m.captures.router_name;
|
|
987
|
+
const prefixNode = m.captures.prefix;
|
|
988
|
+
if (!nameNode || !prefixNode)
|
|
989
|
+
continue;
|
|
990
|
+
const localImp = localNameToModule.get(nameNode.text);
|
|
991
|
+
if (!localImp)
|
|
992
|
+
continue;
|
|
993
|
+
const prefix = unquoteLiteral(prefixNode.text);
|
|
994
|
+
if (prefix === null)
|
|
995
|
+
continue;
|
|
996
|
+
if (localImp.moduleLong) {
|
|
997
|
+
recordPrefix(prefixesByLongKey, localImp.moduleLong, prefix);
|
|
998
|
+
}
|
|
999
|
+
else {
|
|
1000
|
+
recordPrefix(prefixesByShortKey, localImp.moduleShort, prefix);
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
// #2391: build the repo-wide constant map for resolving non-literal decorator
|
|
1005
|
+
// paths (KTD6 cost gate: only when `hasComposedRoute`). Parse EVERY `.py` so
|
|
1006
|
+
// the resolvable set matches the ingestion aggregate (R4 parity) — a narrower
|
|
1007
|
+
// set would return null where ingestion resolves.
|
|
1008
|
+
if (hasComposedRoute) {
|
|
1009
|
+
const mc = extractPythonModuleConstants(tree);
|
|
1010
|
+
if (mc.literals.size > 0 || mc.exprs.size > 0 || mc.imports.size > 0) {
|
|
1011
|
+
constantsByFile.set(rel, mc);
|
|
932
1012
|
}
|
|
933
1013
|
}
|
|
934
1014
|
}
|
|
935
1015
|
return {
|
|
936
1016
|
prefixesByLongKey,
|
|
937
1017
|
prefixesByShortKey,
|
|
1018
|
+
constantsByFile,
|
|
938
1019
|
};
|
|
939
1020
|
}
|
|
940
1021
|
function joinPrefix(prefix, route) {
|
|
@@ -971,32 +1052,67 @@ export const PYTHON_HTTP_PLUGIN = {
|
|
|
971
1052
|
// statements, so an imperatively-registered handler (Flask `view_func`) that
|
|
972
1053
|
// is an imported (possibly aliased) symbol resolves to its real definition.
|
|
973
1054
|
const importMap = buildPythonImportMap(tree);
|
|
974
|
-
//
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
1055
|
+
// #2391: fold a non-literal decorator argument (bare constant or
|
|
1056
|
+
// `+`-concatenation) to its literal path against the repo constant map, or
|
|
1057
|
+
// `null` → skip (the same floor the ingestion side applies, so provider
|
|
1058
|
+
// contracts and graph Route nodes agree on both resolved and dropped routes).
|
|
1059
|
+
const resolveExprArg = (argNode) => {
|
|
1060
|
+
const cbf = ctx?.constantsByFile;
|
|
1061
|
+
if (!cbf || !fileRel)
|
|
1062
|
+
return null;
|
|
1063
|
+
// Build an operand list and fold via `resolveOperands` — the SAME entry the
|
|
1064
|
+
// ingestion side uses (parse-impl folds `routePathOperands`). Using the
|
|
1065
|
+
// by-name `resolveConstant` here would enter `foldName` one depth shallower,
|
|
1066
|
+
// so at the MAX_RESOLVE_DEPTH boundary the group would resolve a chain
|
|
1067
|
+
// ingestion drops, breaking R4 parity (#2393).
|
|
1068
|
+
const operands = argNode.type === 'identifier'
|
|
1069
|
+
? [{ kind: 'ref', name: argNode.text }]
|
|
1070
|
+
: parseConstOperands(argNode);
|
|
1071
|
+
return operands ? resolveOperands(fileRel, operands, cbf) : null;
|
|
1072
|
+
};
|
|
1073
|
+
const emitAppProvider = (httpMethod, pathVal, line) => {
|
|
986
1074
|
out.push({
|
|
987
1075
|
role: 'provider',
|
|
988
1076
|
framework: 'fastapi',
|
|
989
1077
|
method: httpMethod,
|
|
990
|
-
path,
|
|
1078
|
+
path: pathVal,
|
|
991
1079
|
name: null,
|
|
992
1080
|
// The decorated handler has no captured name → resolve by line-span
|
|
993
1081
|
// containment. Best-effort fallback: FastAPI routes are graph-backed
|
|
994
1082
|
// (ingestion decorator routes) and the function span starts at `def`
|
|
995
1083
|
// (decorators excluded), so this lands the single-decorator case and
|
|
996
1084
|
// degrades to file-level for multi-decorator stacks.
|
|
997
|
-
line
|
|
1085
|
+
line,
|
|
998
1086
|
confidence: 0.8,
|
|
999
1087
|
});
|
|
1088
|
+
};
|
|
1089
|
+
// Providers: FastAPI @app.<verb>("/path") — already absolute path.
|
|
1090
|
+
for (const match of runCompiledPatterns(FASTAPI_APP_PATTERNS, tree)) {
|
|
1091
|
+
const methodNode = match.captures.method;
|
|
1092
|
+
const pathNode = match.captures.path;
|
|
1093
|
+
if (!methodNode || !pathNode)
|
|
1094
|
+
continue;
|
|
1095
|
+
const httpMethod = FASTAPI_VERBS[methodNode.text];
|
|
1096
|
+
if (!httpMethod)
|
|
1097
|
+
continue;
|
|
1098
|
+
const path = unquoteLiteral(pathNode.text);
|
|
1099
|
+
if (path === null)
|
|
1100
|
+
continue;
|
|
1101
|
+
emitAppProvider(httpMethod, path, pathNode.startPosition.row + 1);
|
|
1102
|
+
}
|
|
1103
|
+
// Providers: FastAPI @app.<verb>(CONST | A + "/x") — resolved composed path.
|
|
1104
|
+
for (const match of runCompiledPatterns(FASTAPI_APP_EXPR_PATTERNS, tree)) {
|
|
1105
|
+
const methodNode = match.captures.method;
|
|
1106
|
+
const pathNode = match.captures.path;
|
|
1107
|
+
if (!methodNode || !pathNode)
|
|
1108
|
+
continue;
|
|
1109
|
+
const httpMethod = FASTAPI_VERBS[methodNode.text];
|
|
1110
|
+
if (!httpMethod)
|
|
1111
|
+
continue;
|
|
1112
|
+
const resolved = resolveExprArg(pathNode);
|
|
1113
|
+
if (resolved === null)
|
|
1114
|
+
continue; // skip floor
|
|
1115
|
+
emitAppProvider(httpMethod, resolved, pathNode.startPosition.row + 1);
|
|
1000
1116
|
}
|
|
1001
1117
|
// Django providers come from the graph Route nodes (includes composed by
|
|
1002
1118
|
// the ingestion route extractor), not a per-file source scan — see the note
|
|
@@ -1020,17 +1136,11 @@ export const PYTHON_HTTP_PLUGIN = {
|
|
|
1020
1136
|
// change is strictly additive vs. the prior @app-only behaviour;
|
|
1021
1137
|
// when the same router is mounted under multiple prefixes we emit
|
|
1022
1138
|
// one detection per prefix.
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
const httpMethod = FASTAPI_VERBS[methodNode.text];
|
|
1029
|
-
if (!httpMethod)
|
|
1030
|
-
continue;
|
|
1031
|
-
const rawPath = unquoteLiteral(pathNode.text);
|
|
1032
|
-
if (rawPath === null)
|
|
1033
|
-
continue;
|
|
1139
|
+
// Join a `@router.<verb>` path with the include_router / APIRouter prefix(es)
|
|
1140
|
+
// that apply to this file and emit one provider detection per prefix. Shared
|
|
1141
|
+
// by the literal and the #2391 non-literal (resolved) router loops so both
|
|
1142
|
+
// stack prefixes identically.
|
|
1143
|
+
const emitRouterProvider = (httpMethod, rawPath, line) => {
|
|
1034
1144
|
// Long key first (precise, package-aware), short key as fallback.
|
|
1035
1145
|
// Mirrors the ingestion-side resolution in parse-impl.ts so the
|
|
1036
1146
|
// graph nodes and group contracts agree on which prefix applies.
|
|
@@ -1053,10 +1163,38 @@ export const PYTHON_HTTP_PLUGIN = {
|
|
|
1053
1163
|
path: p,
|
|
1054
1164
|
name: null,
|
|
1055
1165
|
// Best-effort containment fallback — see the @app provider note above.
|
|
1056
|
-
line
|
|
1166
|
+
line,
|
|
1057
1167
|
confidence: 0.8,
|
|
1058
1168
|
});
|
|
1059
1169
|
}
|
|
1170
|
+
};
|
|
1171
|
+
for (const match of runCompiledPatterns(FASTAPI_ROUTER_PATTERNS, tree)) {
|
|
1172
|
+
const methodNode = match.captures.method;
|
|
1173
|
+
const pathNode = match.captures.path;
|
|
1174
|
+
if (!methodNode || !pathNode)
|
|
1175
|
+
continue;
|
|
1176
|
+
const httpMethod = FASTAPI_VERBS[methodNode.text];
|
|
1177
|
+
if (!httpMethod)
|
|
1178
|
+
continue;
|
|
1179
|
+
const rawPath = unquoteLiteral(pathNode.text);
|
|
1180
|
+
if (rawPath === null)
|
|
1181
|
+
continue;
|
|
1182
|
+
emitRouterProvider(httpMethod, rawPath, pathNode.startPosition.row + 1);
|
|
1183
|
+
}
|
|
1184
|
+
// Providers: FastAPI @router.<verb>(CONST | A + "/x") — resolved composed path
|
|
1185
|
+
// (#2391). Null resolution → skip, so provider/graph parity holds.
|
|
1186
|
+
for (const match of runCompiledPatterns(FASTAPI_ROUTER_EXPR_PATTERNS, tree)) {
|
|
1187
|
+
const methodNode = match.captures.method;
|
|
1188
|
+
const pathNode = match.captures.path;
|
|
1189
|
+
if (!methodNode || !pathNode)
|
|
1190
|
+
continue;
|
|
1191
|
+
const httpMethod = FASTAPI_VERBS[methodNode.text];
|
|
1192
|
+
if (!httpMethod)
|
|
1193
|
+
continue;
|
|
1194
|
+
const resolved = resolveExprArg(pathNode);
|
|
1195
|
+
if (resolved === null)
|
|
1196
|
+
continue;
|
|
1197
|
+
emitRouterProvider(httpMethod, resolved, pathNode.startPosition.row + 1);
|
|
1060
1198
|
}
|
|
1061
1199
|
// Providers: Flask `app.add_url_rule('/path', view_func=handler, methods=[…])`.
|
|
1062
1200
|
// The handler is a `view_func` identifier, frequently an imported (possibly
|
|
@@ -3,7 +3,7 @@ import type { SymbolTableWriter } from './model/index.js';
|
|
|
3
3
|
import { type ExportedTypeMap } from './call-processor.js';
|
|
4
4
|
import type { ParsedFile } from '../../_shared/index.js';
|
|
5
5
|
import { WorkerPool } from './workers/worker-pool.js';
|
|
6
|
-
import type { ParseWorkerResult, ExtractedRoute, ExtractedFetchCall, ExtractedDecoratorRoute, ExtractedToolDef, FileScopeBindings, ExtractedORMQuery, FetchWrapperDef } from './workers/parse-worker.js';
|
|
6
|
+
import type { ParseWorkerResult, ExtractedRoute, ExtractedFetchCall, ExtractedDecoratorRoute, ExtractedModuleConstants, ExtractedToolDef, FileScopeBindings, ExtractedORMQuery, FetchWrapperDef } from './workers/parse-worker.js';
|
|
7
7
|
import type { ExtractedRouterConstructorPrefix, ExtractedRouterImport, ExtractedRouterInclude, ExtractedRouterModuleAlias } from './route-extractors/fastapi-router-bindings.js';
|
|
8
8
|
import type { SharedSpringType } from './route-extractors/spring-shared.js';
|
|
9
9
|
export type FileProgressCallback = (current: number, total: number, filePath: string) => void;
|
|
@@ -16,6 +16,8 @@ export interface WorkerExtractedData {
|
|
|
16
16
|
routerImports: ExtractedRouterImport[];
|
|
17
17
|
routerConstructorPrefixes: ExtractedRouterConstructorPrefix[];
|
|
18
18
|
routerModuleAliases: ExtractedRouterModuleAlias[];
|
|
19
|
+
/** Per-file Python module constants for cross-file route-path resolution (#2391). */
|
|
20
|
+
moduleConstants: ExtractedModuleConstants[];
|
|
19
21
|
toolDefs: ExtractedToolDef[];
|
|
20
22
|
ormQueries: ExtractedORMQuery[];
|
|
21
23
|
/** Project-wide Spring class/interface views for the #2288 inheritance pass. */
|
|
@@ -24,6 +24,7 @@ export const mergeChunkResults = (graph, symbolTable, chunkResults, exportedType
|
|
|
24
24
|
const allRouterImports = [];
|
|
25
25
|
const allRouterConstructorPrefixes = [];
|
|
26
26
|
const allRouterModuleAliases = [];
|
|
27
|
+
const allModuleConstants = [];
|
|
27
28
|
const allSpringTypes = [];
|
|
28
29
|
const allToolDefs = [];
|
|
29
30
|
const allORMQueries = [];
|
|
@@ -76,6 +77,8 @@ export const mergeChunkResults = (graph, symbolTable, chunkResults, exportedType
|
|
|
76
77
|
}
|
|
77
78
|
for (const item of result.routerModuleAliases ?? [])
|
|
78
79
|
allRouterModuleAliases.push(item);
|
|
80
|
+
for (const item of result.moduleConstants ?? [])
|
|
81
|
+
allModuleConstants.push(item);
|
|
79
82
|
for (const item of result.springTypes ?? [])
|
|
80
83
|
allSpringTypes.push(item);
|
|
81
84
|
for (const item of result.toolDefs)
|
|
@@ -99,6 +102,7 @@ export const mergeChunkResults = (graph, symbolTable, chunkResults, exportedType
|
|
|
99
102
|
routerImports: allRouterImports,
|
|
100
103
|
routerConstructorPrefixes: allRouterConstructorPrefixes,
|
|
101
104
|
routerModuleAliases: allRouterModuleAliases,
|
|
105
|
+
moduleConstants: allModuleConstants,
|
|
102
106
|
toolDefs: allToolDefs,
|
|
103
107
|
ormQueries: allORMQueries,
|
|
104
108
|
springTypes: allSpringTypes,
|
|
@@ -28,6 +28,7 @@ import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
|
|
|
28
28
|
import { getProvider, providers } from '../languages/index.js';
|
|
29
29
|
import { createWorkerPool, workerPoolDisabledByEnv, resolveAutoPoolSize, WorkerPoolInitializationError, WorkerPoolDisabledError, } from '../workers/worker-pool.js';
|
|
30
30
|
import { normalizeExtractedRoutePath } from '../route-extractors/route-path.js';
|
|
31
|
+
import { resolveOperands, } from '../route-extractors/python-const-resolver.js';
|
|
31
32
|
import { resolveInheritedSpringRoutes, } from '../route-extractors/spring-shared.js';
|
|
32
33
|
import fs from 'node:fs';
|
|
33
34
|
import path from 'node:path';
|
|
@@ -464,6 +465,9 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
464
465
|
const allRouterImports = [];
|
|
465
466
|
const allRouterConstructorPrefixes = [];
|
|
466
467
|
const allRouterModuleAliases = [];
|
|
468
|
+
// Per-file Python module constants (#2391); resolved into decorator route paths
|
|
469
|
+
// below, after cross-file aggregation, alongside the include_router prefix pass.
|
|
470
|
+
const allModuleConstants = [];
|
|
467
471
|
const allSpringTypes = [];
|
|
468
472
|
const allToolDefs = [];
|
|
469
473
|
const allORMQueries = [];
|
|
@@ -610,6 +614,10 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
610
614
|
for (const item of chunkWorkerData.routerModuleAliases)
|
|
611
615
|
allRouterModuleAliases.push(item);
|
|
612
616
|
}
|
|
617
|
+
if (chunkWorkerData.moduleConstants?.length) {
|
|
618
|
+
for (const item of chunkWorkerData.moduleConstants)
|
|
619
|
+
allModuleConstants.push(item);
|
|
620
|
+
}
|
|
613
621
|
if (chunkWorkerData.springTypes?.length) {
|
|
614
622
|
for (const item of chunkWorkerData.springTypes)
|
|
615
623
|
allSpringTypes.push(item);
|
|
@@ -925,6 +933,40 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
925
933
|
}
|
|
926
934
|
// FastAPI router-prefix resolution (cross-file).
|
|
927
935
|
//
|
|
936
|
+
// #2391: resolve non-literal FastAPI decorator route paths (imported/composed
|
|
937
|
+
// string constants) BEFORE the include_router/APIRouter prefix pass below, so a
|
|
938
|
+
// resolved path is then prefix-joined like any literal path. Each such route
|
|
939
|
+
// carries `routePathExpr`/`routePathOperands` and an empty `routePath`; we fold
|
|
940
|
+
// the operands against the repo-wide, file-path-keyed constant map. On failure
|
|
941
|
+
// we DROP the route (KTD5 skip floor) rather than emit a phantom `POST /`.
|
|
942
|
+
if (allDecoratorRoutes.some((dr) => dr.routePathExpr !== undefined)) {
|
|
943
|
+
const repoConstants = new Map();
|
|
944
|
+
for (const { filePath, constants } of allModuleConstants) {
|
|
945
|
+
repoConstants.set(filePath, constants);
|
|
946
|
+
}
|
|
947
|
+
const resolvedRoutes = [];
|
|
948
|
+
let skipped = 0;
|
|
949
|
+
for (const dr of allDecoratorRoutes) {
|
|
950
|
+
if (dr.routePathExpr === undefined) {
|
|
951
|
+
resolvedRoutes.push(dr);
|
|
952
|
+
continue;
|
|
953
|
+
}
|
|
954
|
+
const value = dr.routePathOperands
|
|
955
|
+
? resolveOperands(dr.filePath, dr.routePathOperands, repoConstants)
|
|
956
|
+
: null;
|
|
957
|
+
if (value === null) {
|
|
958
|
+
skipped++;
|
|
959
|
+
continue;
|
|
960
|
+
}
|
|
961
|
+
resolvedRoutes.push({ ...dr, routePath: value });
|
|
962
|
+
}
|
|
963
|
+
allDecoratorRoutes.length = 0;
|
|
964
|
+
for (const dr of resolvedRoutes)
|
|
965
|
+
allDecoratorRoutes.push(dr);
|
|
966
|
+
if (isDev && skipped > 0) {
|
|
967
|
+
logger.info(` 🧩 Resolved composed route constants; ${skipped} unresolved route(s) skipped`);
|
|
968
|
+
}
|
|
969
|
+
}
|
|
928
970
|
// Workers emit two kinds of records per Python file:
|
|
929
971
|
// • `routerIncludes` — every `app.include_router(<routerExpr>, prefix='/x')`
|
|
930
972
|
// site, where `routerExpr` is either `<module>.router` (Shape A) or a
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Language-agnostic string-constant folding for route-path resolution (#2391).
|
|
3
|
+
*
|
|
4
|
+
* Route decorators/annotations frequently build their path from a constant rather
|
|
5
|
+
* than a string literal — `@router.post(API_V1_WIDGETS_GET)` (Python),
|
|
6
|
+
* `@GetMapping(PathConstants.WIDGETS)` (Spring), and the Kotlin/C# equivalents are
|
|
7
|
+
* the same shape. This module folds such a constant — or an inline
|
|
8
|
+
* `+`-concatenation — to its literal value, following `+` operands and import
|
|
9
|
+
* chains across a repo-wide, file-keyed constant map.
|
|
10
|
+
*
|
|
11
|
+
* The FOLD is language-neutral: it walks {@link Operand} lists and
|
|
12
|
+
* {@link ModuleConstants} that ANY language's extractor can produce, and defers
|
|
13
|
+
* the one language-specific decision — mapping an import specifier to the file it
|
|
14
|
+
* refers to — to a caller-supplied {@link ImportResolver}. A language binding
|
|
15
|
+
* (e.g. `python-const-resolver.ts`) provides that resolver plus a tree →
|
|
16
|
+
* {@link ModuleConstants} extractor and, if wanted, thin pre-bound wrappers.
|
|
17
|
+
*
|
|
18
|
+
* This mirrors how `route-path.ts` (URL normalization) and `spring-shared.ts`
|
|
19
|
+
* (annotation primitives) are shared across the ingestion and group layers and
|
|
20
|
+
* across languages: the reusable core lives in one place; per-language semantics
|
|
21
|
+
* plug in. It deliberately does NOT reuse `ScopeResolver` (which resolves symbol
|
|
22
|
+
* IDENTITIES, not literal string VALUES) or the `--pdg` `REACHING_DEF` layer
|
|
23
|
+
* (intra-procedural, function-local, def→use reachability — not module-level
|
|
24
|
+
* cross-file value folding).
|
|
25
|
+
*/
|
|
26
|
+
/**
|
|
27
|
+
* One term of a constant's right-hand side. A `+`-concatenation
|
|
28
|
+
* (`A + "/b" + C`) becomes an ordered `Operand[]`; a bare literal is a
|
|
29
|
+
* single-element list.
|
|
30
|
+
*/
|
|
31
|
+
export type Operand = {
|
|
32
|
+
readonly kind: 'literal';
|
|
33
|
+
readonly value: string;
|
|
34
|
+
} | {
|
|
35
|
+
readonly kind: 'ref';
|
|
36
|
+
readonly name: string;
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* A `from <module> import <name> [as <local>]` (or the language's equivalent)
|
|
40
|
+
* binding. `module` is the import specifier as written (e.g. `.constants`,
|
|
41
|
+
* `..pkg.constants`, `api.constants`) so the {@link ImportResolver} can apply
|
|
42
|
+
* language-specific rules; `originalName` is the exported name in the target
|
|
43
|
+
* module (pre-alias). The map key is the local (in-file) name.
|
|
44
|
+
*/
|
|
45
|
+
export interface ImportBinding {
|
|
46
|
+
readonly module: string;
|
|
47
|
+
readonly originalName: string;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* String-valued module-level constants of one source file. `literals` are
|
|
51
|
+
* fully-resolved (`X = "/a"`); `exprs` are unresolved operand lists
|
|
52
|
+
* (`X = A + "/b"`); `imports` maps a local name to the module it was imported
|
|
53
|
+
* from. All string keys are the in-file (local) names.
|
|
54
|
+
*/
|
|
55
|
+
export interface ModuleConstants {
|
|
56
|
+
readonly literals: Map<string, string>;
|
|
57
|
+
readonly exprs: Map<string, readonly Operand[]>;
|
|
58
|
+
readonly imports: Map<string, ImportBinding>;
|
|
59
|
+
}
|
|
60
|
+
/** Repo-wide map: unique file key (e.g. `app/constants.py`) → that file's
|
|
61
|
+
* {@link ModuleConstants}. */
|
|
62
|
+
export type RepoConstants = ReadonlyMap<string, ModuleConstants>;
|
|
63
|
+
/**
|
|
64
|
+
* Resolve an import specifier (as written) from `importingFileKey` to the unique
|
|
65
|
+
* repo file key it refers to, or `null` when it cannot be pinned to exactly one
|
|
66
|
+
* file. This is the sole language-specific dependency of the fold: Python uses
|
|
67
|
+
* leading-dot relative imports + `.py`-suffix rules; a JVM binding would use
|
|
68
|
+
* package/classpath rules. Returning `null` on ambiguity keeps the fold honest —
|
|
69
|
+
* an unresolvable or ambiguous import floors to skip, never a wrong path.
|
|
70
|
+
*/
|
|
71
|
+
export type ImportResolver = (importingFileKey: string, moduleSpec: string, repoKeys: ReadonlySet<string>) => string | null;
|
|
72
|
+
/**
|
|
73
|
+
* Resolve a single named constant referenced in `fileKey` to its literal string
|
|
74
|
+
* value, folding `+` concatenation and following import chains via
|
|
75
|
+
* `resolveImport`, or `null` when it cannot be fully folded.
|
|
76
|
+
*/
|
|
77
|
+
export declare function resolveConstant(fileKey: string, name: string, repo: RepoConstants, resolveImport: ImportResolver): string | null;
|
|
78
|
+
/**
|
|
79
|
+
* Resolve an inline operand list (an unnamed `+`-expression captured directly at
|
|
80
|
+
* a decorator/annotation argument, e.g. `@router.get(API_V1 + "/widgets")`)
|
|
81
|
+
* against `fileKey`.
|
|
82
|
+
*/
|
|
83
|
+
export declare function resolveOperands(fileKey: string, operands: readonly Operand[], repo: RepoConstants, resolveImport: ImportResolver): string | null;
|