devlensio 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/dist/graph/edges/callEdges.js +77 -3
  2. package/dist/graph/edges/callEdges.test.js +1 -1
  3. package/dist/graph/edges/callOrder.test.d.ts +1 -0
  4. package/dist/graph/edges/callOrder.test.js +227 -0
  5. package/dist/graph/edges/propEdges.js +6 -1
  6. package/dist/index.d.ts +1 -0
  7. package/dist/index.js +1 -0
  8. package/dist/parser/callFlow.d.ts +15 -0
  9. package/dist/parser/callFlow.js +302 -0
  10. package/dist/parser/callOrder.test.d.ts +1 -0
  11. package/dist/parser/callOrder.test.js +407 -0
  12. package/dist/parser/extractors/components.js +18 -28
  13. package/dist/parser/extractors/functions.d.ts +12 -3
  14. package/dist/parser/extractors/functions.js +58 -36
  15. package/dist/parser/extractors/moduleScope.js +1 -1
  16. package/dist/parser/moduleScope.test.js +5 -3
  17. package/dist/summarizer/index.js +53 -4
  18. package/dist/summarizer/prompts.js +111 -50
  19. package/dist/summarizer/prompts.test.d.ts +1 -0
  20. package/dist/summarizer/prompts.test.js +112 -0
  21. package/dist/types.d.ts +63 -0
  22. package/dist/version.d.ts +1 -0
  23. package/dist/version.js +2 -0
  24. package/extractors/go/bin/darwin-amd64/devlens_go_extractor +0 -0
  25. package/extractors/go/bin/darwin-arm64/devlens_go_extractor +0 -0
  26. package/extractors/go/bin/linux-amd64/devlens_go_extractor +0 -0
  27. package/extractors/go/bin/linux-arm64/devlens_go_extractor +0 -0
  28. package/extractors/go/bin/windows-amd64/devlens_go_extractor.exe +0 -0
  29. package/extractors/go/calls.go +89 -10
  30. package/extractors/go/nodes.go +29 -11
  31. package/extractors/go/parser.go +311 -21
  32. package/extractors/java/devlens_java_extractor.jar +0 -0
  33. package/extractors/java/src/devlens/extractor/Extractor.java +36 -10
  34. package/extractors/java/src/devlens/extractor/Parser.java +387 -0
  35. package/extractors/java/src/devlens/extractor/edges/Calls.java +66 -10
  36. package/extractors/python/devlens_extractors_python/edges/calls.py +26 -3
  37. package/extractors/python/devlens_extractors_python/parser/__init__.py +4 -3
  38. package/extractors/python/devlens_extractors_python/parser/functions.py +199 -42
  39. package/extractors/rust/bin/darwin-amd64/devlens_rust_extractor +0 -0
  40. package/extractors/rust/bin/darwin-arm64/devlens_rust_extractor +0 -0
  41. package/extractors/rust/bin/linux-amd64/devlens_rust_extractor +0 -0
  42. package/extractors/rust/bin/linux-arm64/devlens_rust_extractor +0 -0
  43. package/extractors/rust/bin/windows-amd64/devlens_rust_extractor.exe +0 -0
  44. package/extractors/rust/src/calls.rs +82 -8
  45. package/extractors/rust/src/nodes.rs +29 -0
  46. package/extractors/rust/src/parser.rs +362 -27
  47. package/package.json +77 -73
@@ -8,10 +8,52 @@ RESOLVED later by edges/orm_edges.py via lookup maps — no re-parsing.
8
8
  """
9
9
  from __future__ import annotations
10
10
  import ast
11
+ from typing import NamedTuple
11
12
 
12
13
  from ..contract import code_node
13
14
 
14
15
  NESTED_SCOPES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)
16
+ COMPREHENSIONS = (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)
17
+
18
+ # ── callOrder/v1 scheduler tables (docs/CALL_ORDER.md §4.2) ─────────────
19
+ # Matched on the LAST name segment so `from threading import Timer` and
20
+ # `threading.Timer(...)` classify alike — the same name-based heuristic the rest
21
+ # of the extractor uses (no type info at parse time).
22
+ TIMER_CALLS = {"call_later", "call_at", "Timer"}
23
+ PROMISE_CALLS = {"create_task", "ensure_future", "gather"}
24
+ EVENT_CALLS = {"signal", "add_signal_handler"}
25
+ CONC_CALLS = {"Thread", "ThreadPoolExecutor", "ProcessPoolExecutor",
26
+ "to_thread", "run_in_executor"}
27
+
28
+
29
+ def _scheduler(name: str) -> tuple[str | None, bool, bool]:
30
+ """(defer, conc, defers_its_arguments) for a scheduler call, else
31
+ (None, False, False). `defers_its_arguments` is False for timers and event
32
+ registrations: their delay/signal argument is evaluated inline and the
33
+ deferred piece is a callback we never walk into."""
34
+ leaf = name.rsplit(".", 1)[-1]
35
+ if leaf in TIMER_CALLS:
36
+ return "timer", False, False
37
+ if leaf in PROMISE_CALLS:
38
+ return "promise", True, True
39
+ if leaf in EVENT_CALLS:
40
+ return "event", False, False
41
+ if leaf in CONC_CALLS:
42
+ return None, True, True
43
+ return None, False, False
44
+
45
+
46
+ class _SiteContext(NamedTuple):
47
+ """Execution-region facts accumulated on the way down the walk; a call site
48
+ only has to read the tuple it was reached with."""
49
+ scope: str = "inline"
50
+ enclosing_order: int | None = None
51
+ awaited: bool = False
52
+ defer: str | None = None
53
+ defer_detail: str | None = None
54
+ conc: bool = False
55
+ repeat: bool = False
56
+ maybe: bool = False
15
57
 
16
58
  # ── ORM patterns (parse-time facts; resolution lives in edges/orm_edges.py) ──
17
59
  DJANGO_QUERY_METHODS = {
@@ -86,52 +128,167 @@ def _infer_arg_type(arg: ast.AST) -> str:
86
128
  return "unknown"
87
129
 
88
130
 
89
- def extract_calls_with_sites(body: list[ast.stmt]) -> tuple[list[str], list[dict]]:
131
+ def _arg_shape(node: ast.Call) -> tuple[list[str], bool, int]:
132
+ """(argTypes, hasSpread, argCount). argCount counts positional args + keyword
133
+ args; *args/**kwargs set hasSpread (runtime arity unknowable)."""
134
+ arg_types: list[str] = []
135
+ has_spread = False
136
+ for a in node.args:
137
+ if isinstance(a, ast.Starred):
138
+ has_spread = True
139
+ arg_types.append("unknown")
140
+ else:
141
+ arg_types.append(_infer_arg_type(a))
142
+ for kw in node.keywords:
143
+ if kw.arg is None: # **kwargs
144
+ has_spread = True
145
+ arg_types.append("unknown")
146
+ else:
147
+ arg_types.append(_infer_arg_type(kw.value))
148
+ return arg_types, has_spread, len(node.args) + len(node.keywords)
149
+
150
+
151
+ def _comprehension_children(node: ast.AST, ctx: _SiteContext) -> list[tuple[ast.AST, _SiteContext]]:
152
+ """A comprehension body is an iteration callback (scope "callback",
153
+ repeat); only the first iterable is evaluated in the enclosing frame."""
154
+ callback = ctx._replace(scope="callback", repeat=True)
155
+ if isinstance(node, ast.DictComp):
156
+ children: list[tuple[ast.AST, _SiteContext]] = [
157
+ (node.key, callback), (node.value, callback)]
158
+ else:
159
+ children = [(node.elt, callback)]
160
+ for i, generator in enumerate(node.generators):
161
+ children.append((generator.target, callback))
162
+ children.append((generator.iter, ctx if i == 0 else callback))
163
+ children.extend((cond, callback) for cond in generator.ifs)
164
+ return children
165
+
166
+
167
+ def _children_with_context(node: ast.AST, ctx: _SiteContext) -> list[tuple[ast.AST, _SiteContext]]:
168
+ """(child, context) pairs in source order. This is where the callOrder flags
169
+ come from: loop bodies repeat, branch bodies may be skipped, except bodies
170
+ may be skipped, finally bodies run at frame exit, comprehension bodies are
171
+ iteration callbacks, an await operand is awaited."""
172
+ if isinstance(node, COMPREHENSIONS):
173
+ return _comprehension_children(node, ctx)
174
+ if isinstance(node, (ast.For, ast.AsyncFor)):
175
+ repeated = ctx._replace(repeat=True)
176
+ return ([(node.target, ctx), (node.iter, ctx)]
177
+ + [(s, repeated) for s in node.body]
178
+ + [(s, repeated) for s in node.orelse])
179
+ if isinstance(node, ast.While):
180
+ repeated = ctx._replace(repeat=True)
181
+ return ([(node.test, ctx)]
182
+ + [(s, repeated) for s in node.body]
183
+ + [(s, repeated) for s in node.orelse])
184
+ if isinstance(node, ast.If):
185
+ maybe = ctx._replace(maybe=True) # test always runs
186
+ return ([(node.test, ctx)]
187
+ + [(s, maybe) for s in node.body]
188
+ + [(s, maybe) for s in node.orelse])
189
+ if isinstance(node, ast.IfExp):
190
+ maybe = ctx._replace(maybe=True)
191
+ return [(node.test, ctx), (node.body, maybe), (node.orelse, maybe)]
192
+ if isinstance(node, ast.BoolOp):
193
+ # `and`/`or`: every operand after the first is short-circuitable
194
+ maybe = ctx._replace(maybe=True)
195
+ if not node.values:
196
+ return []
197
+ return [(node.values[0], ctx)] + [(v, maybe) for v in node.values[1:]]
198
+ if isinstance(node, (ast.Try, ast.TryStar)):
199
+ # try body runs (doc §2 earlyExit: it is the calls AFTER a throwing
200
+ # statement that may not run); except bodies may not run at all.
201
+ maybe = ctx._replace(maybe=True)
202
+ on_return = ctx._replace(defer=ctx.defer or "onReturn",
203
+ defer_detail=ctx.defer_detail or "finally")
204
+ return ([(s, ctx) for s in node.body]
205
+ + [(h, maybe) for h in node.handlers]
206
+ + [(s, ctx) for s in node.orelse]
207
+ + [(s, on_return) for s in node.finalbody])
208
+ if isinstance(node, ast.Match):
209
+ maybe = ctx._replace(maybe=True) # subject always evaluates
210
+ return [(node.subject, ctx)] + [(case, maybe) for case in node.cases]
211
+ if isinstance(node, ast.Await):
212
+ return [(node.value, ctx._replace(awaited=True))]
213
+ return [(child, ctx) for child in ast.iter_child_nodes(node)]
214
+
215
+
216
+ def _call_record(node: ast.Call, name: str, order: int, kind: str,
217
+ ctx: _SiteContext) -> tuple[dict, _SiteContext]:
218
+ """One call-site record for one occurrence + the context its own children
219
+ inherit."""
220
+ arg_types, has_spread, arg_count = _arg_shape(node)
221
+ site: dict = {"name": name, "argCount": arg_count, "argTypes": arg_types}
222
+ if has_spread:
223
+ site["hasSpread"] = True
224
+ site["kind"] = kind
225
+ site["order"] = order
226
+ site["line"] = node.lineno
227
+ site["col"] = node.col_offset
228
+ if ctx.enclosing_order is not None:
229
+ site["nested"] = ctx.enclosing_order
230
+ site["scope"] = ctx.scope
231
+ if ctx.awaited:
232
+ site["awaited"] = True
233
+
234
+ defer, conc, defers_arguments = _scheduler(name)
235
+ site_defer = ctx.defer or defer
236
+ site_conc = ctx.conc or conc
237
+ if site_defer:
238
+ site["defer"] = site_defer
239
+ if site_conc:
240
+ site["conc"] = True
241
+ detail = ctx.defer_detail or (name if (defer or conc) else None)
242
+ if detail and (site_defer or site_conc):
243
+ site["deferDetail"] = detail
244
+ if ctx.repeat:
245
+ site["repeat"] = True
246
+ if ctx.maybe:
247
+ site["maybe"] = True
248
+
249
+ children = ctx._replace(enclosing_order=order, awaited=False)
250
+ if defers_arguments and (defer or conc):
251
+ # the scheduler defers/parallelises this call's arguments — they inherit
252
+ # it (asyncio.gather(a(), b()) → both promise + conc).
253
+ children = children._replace(defer=ctx.defer or defer,
254
+ defer_detail=ctx.defer_detail or name,
255
+ conc=ctx.conc or conc)
256
+ return site, children
257
+
258
+
259
+ def extract_calls_with_sites(body: list[ast.stmt], module_scope: bool = False
260
+ ) -> tuple[list[str], list[dict]]:
90
261
  """ONE scope-guarded walk producing both outputs: the legacy deduped call
91
- name list and structured call sites {name, argCount, argTypes, hasSpread}.
92
- argCount counts positional args + keyword args; *args/**kwargs set
93
- hasSpread (runtime arity unknowable). One record per
94
- (name, argCount, argTypes) triple so same-arity different-type call sites
95
- both survive (each can hit a different overload sibling)."""
262
+ name list and one structured call-site record PER OCCURRENCE.
263
+
264
+ The recorded set is frozen (docs/CALL_ORDER.md §5): this walk descends into
265
+ exactly the same nodes `_walk_scope` does — never into nested defs, classes
266
+ or lambdas — and only attaches callOrder/v1 metadata (order, line, col,
267
+ nested, scope, awaited, defer, deferDetail, conc, repeat, maybe).
268
+ `module_scope` marks a FILE owner: every site then reads scope "module".
269
+ Hooks (use* callees) are recorded in the flow but excluded from the name
270
+ list, exactly as metadata.calls has always been (§2 kind)."""
96
271
  calls: list[str] = []
97
272
  sites: list[dict] = []
98
- seen_sites: set[str] = set()
99
-
100
- def visit(node: ast.AST) -> None:
101
- if not isinstance(node, ast.Call):
102
- return
103
- name = call_name(node.func)
104
- if not name:
105
- return
106
- if name not in calls:
107
- calls.append(name)
108
-
109
- arg_types: list[str] = []
110
- has_spread = False
111
- for a in node.args:
112
- if isinstance(a, ast.Starred):
113
- has_spread = True
114
- arg_types.append("unknown")
115
- else:
116
- arg_types.append(_infer_arg_type(a))
117
- for kw in node.keywords:
118
- if kw.arg is None: # **kwargs
119
- has_spread = True
120
- arg_types.append("unknown")
273
+ root = _SiteContext(scope="module" if module_scope else "inline")
274
+ stack: list[tuple[ast.AST, _SiteContext]] = [
275
+ (stmt, root) for stmt in reversed(body)]
276
+ while stack:
277
+ node, ctx = stack.pop()
278
+ if isinstance(node, NESTED_SCOPES):
279
+ continue
280
+ children_ctx = ctx
281
+ if isinstance(node, ast.Call):
282
+ name = call_name(node.func)
283
+ if name:
284
+ kind = "hook" if name.rsplit(".", 1)[-1].startswith("use") else "call"
285
+ site, children_ctx = _call_record(node, name, len(sites) + 1, kind, ctx)
286
+ sites.append(site)
287
+ if kind == "call" and name not in calls:
288
+ calls.append(name)
121
289
  else:
122
- arg_types.append(_infer_arg_type(kw.value))
123
-
124
- arg_count = len(node.args) + len(node.keywords)
125
- key = f"{name}/{arg_count}/{','.join(arg_types)}"
126
- if key in seen_sites:
127
- return
128
- seen_sites.add(key)
129
- site = {"name": name, "argCount": arg_count, "argTypes": arg_types}
130
- if has_spread:
131
- site["hasSpread"] = True
132
- sites.append(site)
133
-
134
- _walk_scope(body, visit)
290
+ children_ctx = ctx._replace(enclosing_order=None)
291
+ stack.extend(reversed(_children_with_context(node, children_ctx)))
135
292
  return calls, sites
136
293
 
137
294
 
@@ -26,6 +26,8 @@ use crate::module_map::ModuleMap;
26
26
  use crate::parser::ParsedItem;
27
27
  use crate::thirdparty::ThirdPartyRegistry;
28
28
 
29
+ use std::collections::HashMap;
30
+
29
31
  pub struct CallsOutput {
30
32
  pub edges: Vec<CodeEdge>,
31
33
  }
@@ -53,7 +55,11 @@ pub fn detect_calls(
53
55
  continue;
54
56
  }
55
57
  let base_module = crate::walker::module_path_for_file(&pf.rel_path);
56
- let mut resolved: Vec<String> = vec![];
58
+ // One metadata.resolvedCalls entry per target, shaped like ResolvedCall
59
+ // in src/types.ts. Node metadata only: no edge-metadata dependency.
60
+ let mut resolved: Vec<serde_json::Value> = vec![];
61
+ let mut resolved_index: HashMap<String, usize> = HashMap::new();
62
+ let mut orders_by_target: HashMap<String, Vec<usize>> = HashMap::new();
57
63
  for site in &item.call_sites {
58
64
  if let Some((target, matched_by)) = resolve_call(
59
65
  &site.name,
@@ -95,7 +101,77 @@ pub fn detect_calls(
95
101
  }
96
102
  }
97
103
  edges.push(edge_with_meta(&from, &target, "CALLS", meta));
98
- resolved.push(target);
104
+ if site.order > 0 {
105
+ orders_by_target
106
+ .entry(target.clone())
107
+ .or_default()
108
+ .push(site.order);
109
+ }
110
+ if !resolved_index.contains_key(&target) {
111
+ let mut rec = serde_json::Map::new();
112
+ let declared = lookup
113
+ .node_by_id
114
+ .get(&target)
115
+ .map(|n| n.name.clone())
116
+ .filter(|n| !n.is_empty())
117
+ .unwrap_or_else(|| site.name.clone());
118
+ rec.insert("name".to_string(), serde_json::Value::from(declared));
119
+ rec.insert("nodeId".to_string(), serde_json::Value::from(target.clone()));
120
+ if site.order > 0 {
121
+ rec.insert(
122
+ "firstOrder".to_string(),
123
+ serde_json::Value::from(site.order as u64),
124
+ );
125
+ rec.insert("line".to_string(), serde_json::Value::from(site.line as u64));
126
+ }
127
+ if let Some(defer) = site.defer {
128
+ rec.insert(
129
+ "defer".to_string(),
130
+ serde_json::Value::from(defer.as_str()),
131
+ );
132
+ }
133
+ if let Some(detail) = &site.defer_detail {
134
+ rec.insert(
135
+ "deferDetail".to_string(),
136
+ serde_json::Value::from(detail.clone()),
137
+ );
138
+ }
139
+ if site.awaited {
140
+ rec.insert("awaited".to_string(), serde_json::Value::Bool(true));
141
+ }
142
+ if site.conc {
143
+ rec.insert("conc".to_string(), serde_json::Value::Bool(true));
144
+ }
145
+ if site.repeat {
146
+ rec.insert("repeat".to_string(), serde_json::Value::Bool(true));
147
+ }
148
+ if site.maybe {
149
+ rec.insert("maybe".to_string(), serde_json::Value::Bool(true));
150
+ }
151
+ resolved_index.insert(target.clone(), resolved.len());
152
+ resolved.push(serde_json::Value::Object(rec));
153
+ }
154
+ }
155
+ }
156
+ // orders[] only when a target was reached more than once.
157
+ for (target, orders) in &orders_by_target {
158
+ if let Some(&idx) = resolved_index.get(target) {
159
+ let mut unique = orders.clone();
160
+ unique.sort_unstable();
161
+ unique.dedup();
162
+ if unique.len() > 1 {
163
+ if let serde_json::Value::Object(rec) = &mut resolved[idx] {
164
+ rec.insert(
165
+ "orders".to_string(),
166
+ serde_json::Value::Array(
167
+ unique
168
+ .into_iter()
169
+ .map(|o| serde_json::Value::from(o as u64))
170
+ .collect(),
171
+ ),
172
+ );
173
+ }
174
+ }
99
175
  }
100
176
  }
101
177
  if !resolved.is_empty() {
@@ -103,12 +179,10 @@ pub fn detect_calls(
103
179
  // registry; clone → mutate → reinsert)
104
180
  if let Some(node) = lookup.node_by_id.get(&from).cloned() {
105
181
  let mut n = node;
106
- let r: Vec<serde_json::Value> = resolved
107
- .iter()
108
- .map(|s| serde_json::Value::from(s.clone()))
109
- .collect();
110
- n.metadata
111
- .insert("resolvedCalls".to_string(), serde_json::Value::Array(r));
182
+ n.metadata.insert(
183
+ "resolvedCalls".to_string(),
184
+ serde_json::Value::Array(resolved),
185
+ );
112
186
  lookup.node_by_id.insert(from, n);
113
187
  }
114
188
  }
@@ -204,6 +204,35 @@ fn item_node(pf: &ParsedFile, item: &ParsedItem) -> CodeNode {
204
204
  m.insert("argCount".to_string(), Value::from(cs.arg_count as u64));
205
205
  let types: Vec<Value> = cs.arg_types.iter().map(|t| Value::from(t.clone())).collect();
206
206
  m.insert("argTypes".to_string(), Value::Array(types));
207
+ m.insert("kind".to_string(), Value::from("call"));
208
+ m.insert("order".to_string(), Value::from(cs.order as u64));
209
+ m.insert("line".to_string(), Value::from(cs.line as u64));
210
+ m.insert("col".to_string(), Value::from(cs.col as u64));
211
+ if let Some(nested) = cs.nested {
212
+ m.insert("nested".to_string(), Value::from(nested as u64));
213
+ }
214
+ m.insert("scope".to_string(), Value::from(cs.scope.as_str()));
215
+ if cs.awaited {
216
+ m.insert("awaited".to_string(), Value::Bool(true));
217
+ }
218
+ if let Some(defer) = cs.defer {
219
+ m.insert("defer".to_string(), Value::from(defer.as_str()));
220
+ }
221
+ if let Some(detail) = &cs.defer_detail {
222
+ m.insert("deferDetail".to_string(), Value::from(detail.clone()));
223
+ }
224
+ if cs.conc {
225
+ m.insert("conc".to_string(), Value::Bool(true));
226
+ }
227
+ if cs.repeat {
228
+ m.insert("repeat".to_string(), Value::Bool(true));
229
+ }
230
+ if cs.maybe {
231
+ m.insert("maybe".to_string(), Value::Bool(true));
232
+ }
233
+ if cs.early_exit {
234
+ m.insert("earlyExit".to_string(), Value::Bool(true));
235
+ }
207
236
  Value::Object(m)
208
237
  })
209
238
  .collect();