@allansantos-dev/smart-tool 0.9.79 → 0.9.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +132 -0
- package/README.md +82 -18
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/client_hooks.py +10 -8
- package/code_health.py +182 -0
- package/code_impact.py +14 -2
- package/config.py +76 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +71 -9
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +289 -18
- package/hooks/pretool_router.py +5 -4
- package/indexer.py +92 -3
- package/install.py +66 -27
- package/install_runtime.py +24 -10
- package/layer_check.py +57 -0
- package/link_check.py +69 -0
- package/local_embedder.py +115 -58
- package/locales/pt-BR.json +122 -18
- package/low_priority.py +41 -0
- package/model_client.py +1 -1
- package/model_health.py +24 -6
- package/notice_action.py +9 -0
- package/package.json +1 -1
- package/pr_comment.py +94 -0
- package/pr_impact.py +292 -8
- package/process_limits.py +108 -0
- package/project_admission.py +17 -15
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/removed_callers.py +146 -0
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +241 -41
- package/smart_tool_daemon.py +403 -91
- package/tray.py +29 -9
- package/update_check.py +18 -0
- package/update_splash.py +2 -2
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_document_graph.py +12 -1
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/project_admission.py
CHANGED
|
@@ -8,6 +8,7 @@ longer exist, are junk (`junk_reason`)."""
|
|
|
8
8
|
import os
|
|
9
9
|
import tempfile
|
|
10
10
|
|
|
11
|
+
import i18n
|
|
11
12
|
import project_identity
|
|
12
13
|
import project_store
|
|
13
14
|
import redirect_rule
|
|
@@ -81,25 +82,25 @@ def refusal(root):
|
|
|
81
82
|
norm = _norm(real)
|
|
82
83
|
scratch = _scratch_roots()
|
|
83
84
|
if os.path.dirname(real) == real:
|
|
84
|
-
return "a drive or filesystem root"
|
|
85
|
+
return i18n.N_("a drive or filesystem root")
|
|
85
86
|
if norm in _personal_folders():
|
|
86
|
-
return "a personal folder, not a project"
|
|
87
|
+
return i18n.N_("a personal folder, not a project")
|
|
87
88
|
if norm == scratch[0]:
|
|
88
|
-
return "the temporary folder"
|
|
89
|
+
return i18n.N_("the temporary folder")
|
|
89
90
|
home = _norm(os.path.expanduser("~"))
|
|
90
91
|
if _inside(norm, home) and norm != home and os.path.relpath(norm, home).split(os.sep)[0].startswith("."):
|
|
91
|
-
return "inside a tool's hidden folder in the home folder"
|
|
92
|
+
return i18n.N_("inside a tool's hidden folder in the home folder")
|
|
92
93
|
if any(_inside(norm, base) for base in _system_roots()):
|
|
93
|
-
return "a system or application folder"
|
|
94
|
+
return i18n.N_("a system or application folder")
|
|
94
95
|
if (any(_inside(norm, base) for base in _env_folders("APPDATA", "LOCALAPPDATA", "LOCALAPPDATA_LOW"))
|
|
95
96
|
and not any(_inside(norm, base) for base in scratch)):
|
|
96
|
-
return "application data"
|
|
97
|
+
return i18n.N_("application data")
|
|
97
98
|
folder = next((part for part in real.replace("\\", "/").split("/") if part.lower() in TOOL_FOLDERS), None)
|
|
98
99
|
if folder:
|
|
99
|
-
return
|
|
100
|
+
return i18n.N_("a build or dependency folder ({folder})").format(folder=folder)
|
|
100
101
|
java = _env_folders("JAVA_HOME")
|
|
101
102
|
if any(_inside(norm, base) for base in java) or any(_is_java_home(path) for path in _ancestors(real)):
|
|
102
|
-
return "a Java home"
|
|
103
|
+
return i18n.N_("a Java home")
|
|
103
104
|
return None
|
|
104
105
|
|
|
105
106
|
|
|
@@ -110,7 +111,7 @@ def temporary_refusal(root):
|
|
|
110
111
|
norm = _norm(project_identity.display_root(root))
|
|
111
112
|
if _inside(norm, _norm(project_identity.display_root(redirect_rule.system_temp()))) and not any(
|
|
112
113
|
_inside(norm, base) for base in _env_folders("SMART_TOOL_SCRATCH")):
|
|
113
|
-
return "inside the system temporary folder"
|
|
114
|
+
return i18n.N_("inside the system temporary folder")
|
|
114
115
|
return None
|
|
115
116
|
|
|
116
117
|
|
|
@@ -150,18 +151,19 @@ def admit(path, origin=None):
|
|
|
150
151
|
return root
|
|
151
152
|
reason = refusal(root) or temporary_refusal(root)
|
|
152
153
|
if reason:
|
|
153
|
-
raise ValueError(
|
|
154
|
-
|
|
154
|
+
raise ValueError(i18n.t("Not indexing {root}: it is {reason}. A project is the folder where the session started; "
|
|
155
|
+
"pass that folder as project_root.", root=root, reason=i18n.t(reason)))
|
|
155
156
|
if origin:
|
|
156
157
|
home, _held = resolve(origin)
|
|
157
158
|
home_reason = refusal(home)
|
|
158
159
|
if home_reason:
|
|
159
|
-
raise ValueError(
|
|
160
|
-
|
|
160
|
+
raise ValueError(i18n.t("Not indexing {root}: the session started in {home}, which is {home_reason}, not a "
|
|
161
|
+
"project. Start the session inside the project folder.",
|
|
162
|
+
root=root, home=home, home_reason=i18n.t(home_reason)))
|
|
161
163
|
if _inside(_norm(root), _norm(home)):
|
|
162
164
|
return home
|
|
163
|
-
raise ValueError(
|
|
164
|
-
|
|
165
|
+
raise ValueError(i18n.t("Not indexing {root}: the session started in the project {home}, and a new project is "
|
|
166
|
+
"indexed only from the session that started in it.", root=root, home=home))
|
|
165
167
|
return root
|
|
166
168
|
|
|
167
169
|
|
package/project_monitor.py
CHANGED
|
@@ -70,11 +70,18 @@ def snapshot(root, scope):
|
|
|
70
70
|
|
|
71
71
|
|
|
72
72
|
def index_model(current=None):
|
|
73
|
-
return local_embedder.resolve(config.load_config()
|
|
73
|
+
return local_embedder.resolve(config.load_config(), current)
|
|
74
74
|
|
|
75
75
|
|
|
76
76
|
def needs_model_rebuild(meta, configured_model):
|
|
77
|
-
|
|
77
|
+
"""Whether the index must be built again for `configured_model`; never while the target model is unknown (None:
|
|
78
|
+
the embeddings home is not running and the index keeps answering as it is)."""
|
|
79
|
+
return configured_model is not None and (meta.get("model_id") != configured_model or not meta.get("exists"))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def needs_model_switch(meta, configured_model):
|
|
83
|
+
"""Whether an existing index holds another model than `configured_model` and must be re-embedded."""
|
|
84
|
+
return bool(meta.get("chunks")) and needs_model_rebuild(meta, configured_model)
|
|
78
85
|
|
|
79
86
|
class ProjectMonitor:
|
|
80
87
|
def __init__(self, enqueue, watcher_factory=directory_watch.DirectoryWatcher):
|
|
@@ -92,6 +99,7 @@ class ProjectMonitor:
|
|
|
92
99
|
self._gap_level = {}
|
|
93
100
|
self._filters = {}
|
|
94
101
|
self._views = {}
|
|
102
|
+
self._switch = set()
|
|
95
103
|
self._lock = threading.RLock()
|
|
96
104
|
self._stop = threading.Event()
|
|
97
105
|
self._thread = None
|
|
@@ -238,6 +246,7 @@ class ProjectMonitor:
|
|
|
238
246
|
projects = project_store.all_projects()
|
|
239
247
|
self._sync_watchers(projects, now)
|
|
240
248
|
busy = project_store.projects_with_pending_job()
|
|
249
|
+
switching = bool(busy)
|
|
241
250
|
for project in projects:
|
|
242
251
|
key = project["id"]
|
|
243
252
|
if not monitored(project):
|
|
@@ -269,7 +278,13 @@ class ProjectMonitor:
|
|
|
269
278
|
if scope is not None:
|
|
270
279
|
changed |= snapshot(project["root"], scope) != indexer.manifest_snapshot(project["root"])
|
|
271
280
|
full = time.time() - meta.get("full_scan_at", 0) >= FULL_CHECK_S
|
|
272
|
-
|
|
281
|
+
target = index_model(meta.get("model_id"))
|
|
282
|
+
free = local_embedder.is_local(target) or target == indexer.LEXICAL_MODEL
|
|
283
|
+
if free and needs_model_switch(meta, target):
|
|
284
|
+
self._switch.add(key)
|
|
285
|
+
else:
|
|
286
|
+
self._switch.discard(key)
|
|
287
|
+
changed |= needs_model_rebuild(meta, target)
|
|
273
288
|
project_store.update(key, last_checked=time.time())
|
|
274
289
|
if changed or full or project.get("dirty_seq", 0) > project.get("indexed_seq", 0):
|
|
275
290
|
with self._lock:
|
|
@@ -281,8 +296,17 @@ class ProjectMonitor:
|
|
|
281
296
|
if first:
|
|
282
297
|
project_store.mark_dirty(key, "Reconciliation found a pending update.")
|
|
283
298
|
project_store.record_changes(key, full=full, scope=rescope)
|
|
284
|
-
#
|
|
285
|
-
#
|
|
299
|
+
# A switch to a model that costs nothing (the embeddings home, or lexical) runs in every update mode, one
|
|
300
|
+
# project at a time, so the old index keeps answering and searches keep a free worker. Outside eager mode
|
|
301
|
+
# any other change only stays pending for the next search, which pays the embeddings.
|
|
302
|
+
if key in self._switch:
|
|
303
|
+
if not switching:
|
|
304
|
+
switching = True
|
|
305
|
+
with self._lock:
|
|
306
|
+
self._pending.setdefault(key, {"paths": set(), "last": now - DEBOUNCE_S, "full": False,
|
|
307
|
+
"force_scope": False, "revision": 0})
|
|
308
|
+
self._dispatch(key, now)
|
|
309
|
+
continue
|
|
286
310
|
if project.get('update_mode', 'on_search') != 'eager':
|
|
287
311
|
with self._lock:
|
|
288
312
|
self._pending.pop(key, None)
|
|
@@ -314,7 +338,8 @@ class ProjectMonitor:
|
|
|
314
338
|
|
|
315
339
|
def _dispatch_quiet(self, now):
|
|
316
340
|
"""Between full passes: sends the projects whose debounce ended, without listing every project of the registry.
|
|
317
|
-
A project that is not eager, is busy or is no longer monitored is left to the next full
|
|
341
|
+
A project that is not eager, is busy, is switching model or is no longer monitored is left to the next full
|
|
342
|
+
pass."""
|
|
318
343
|
self._flush_notes(now)
|
|
319
344
|
with self._lock:
|
|
320
345
|
ready = [key for key, pending in self._pending.items()
|
|
@@ -324,7 +349,8 @@ class ProjectMonitor:
|
|
|
324
349
|
busy = project_store.projects_with_pending_job()
|
|
325
350
|
for key in ready:
|
|
326
351
|
project = project_store.get(key)
|
|
327
|
-
if (project and key not in busy and
|
|
352
|
+
if (project and key not in busy and key not in self._switch and monitored(project)
|
|
353
|
+
and project.get("update_mode", "on_search") == "eager"):
|
|
328
354
|
self._dispatch(key, now)
|
|
329
355
|
|
|
330
356
|
def _run(self):
|
package/project_scheduler.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""Fila justa entre raízes; jobs aguardando a mesma pasta não ocupam workers. Um job de prioridade maior passa à frente
|
|
2
|
-
dos de prioridade menor ainda na fila (o que já está rodando não é interrompido).
|
|
2
|
+
dos de prioridade menor ainda na fila (o que já está rodando não é interrompido). Pistas (`lane`) distintas da mesma pasta
|
|
3
|
+
rodam em paralelo. Com `reserved_lane`, as outras pistas nunca ocupam o último worker livre: uma prioridade só ordena
|
|
4
|
+
a fila e não libera worker, então sem a reserva uma busca espera o fim de um job longo."""
|
|
3
5
|
import collections
|
|
4
6
|
import concurrent.futures
|
|
5
7
|
import threading
|
|
@@ -8,16 +10,19 @@ import project_identity
|
|
|
8
10
|
|
|
9
11
|
|
|
10
12
|
class ProjectScheduler:
|
|
11
|
-
def __init__(self, workers=3):
|
|
13
|
+
def __init__(self, workers=3, reserved_lane=None):
|
|
12
14
|
self.workers = workers
|
|
15
|
+
self.reserved_lane = reserved_lane
|
|
13
16
|
self._pool = concurrent.futures.ThreadPoolExecutor(max_workers=workers, thread_name_prefix="project")
|
|
14
17
|
self._lock = threading.RLock()
|
|
15
18
|
self._queues = collections.OrderedDict()
|
|
16
19
|
self._running = set()
|
|
17
20
|
self._closed = False
|
|
18
21
|
|
|
19
|
-
def submit(self, root, function, *args, priority=0):
|
|
20
|
-
|
|
22
|
+
def submit(self, root, function, *args, priority=0, lane=None):
|
|
23
|
+
"""Queues `function`. Jobs of one folder run one at a time within a lane; jobs of one folder in different
|
|
24
|
+
lanes run side by side (the caller keeps them from writing the same data)."""
|
|
25
|
+
key = (project_identity.canonical_root(root), lane)
|
|
21
26
|
future = concurrent.futures.Future()
|
|
22
27
|
with self._lock:
|
|
23
28
|
if self._closed:
|
|
@@ -31,6 +36,9 @@ class ProjectScheduler:
|
|
|
31
36
|
def _dispatch(self):
|
|
32
37
|
while len(self._running) < self.workers:
|
|
33
38
|
waiting = [root for root in self._queues if root not in self._running]
|
|
39
|
+
if self.reserved_lane is not None and sum(
|
|
40
|
+
lane != self.reserved_lane for _root, lane in self._running) >= self.workers - 1:
|
|
41
|
+
waiting = [key for key in waiting if key[1] == self.reserved_lane]
|
|
34
42
|
if not waiting:
|
|
35
43
|
break
|
|
36
44
|
key = max(waiting, key=lambda root: self._queues[root][0][3])
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Callers that still name a function an edit removed or renamed, or whose signature it changed.
|
|
2
|
+
|
|
3
|
+
The functions the text before the edit defines and the text after lacks (`doc_check.functions`) are looked up in the code
|
|
4
|
+
graph: calls the graph resolved to them (a graph of the text before the edit, as in the edit hook), calls it left
|
|
5
|
+
unresolved under that name (a graph of the text after the edit, as in a diff already on disk), imports of the name, and
|
|
6
|
+
the calls still in the edited file. Names are compared by their last segment, so a name the file still defines
|
|
7
|
+
elsewhere (a method of another class) is not reported. For a function that stays with another signature the callers are
|
|
8
|
+
the calls from other files, resolved or unresolved by name; imports and calls in the edited file are left out.
|
|
9
|
+
"""
|
|
10
|
+
import posixpath
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
import code_graph
|
|
14
|
+
import code_impact
|
|
15
|
+
import doc_check
|
|
16
|
+
import doc_scope
|
|
17
|
+
|
|
18
|
+
MAX_CALLERS = 5
|
|
19
|
+
MAX_SITES = 200
|
|
20
|
+
UNRESOLVED_CAP = 1000
|
|
21
|
+
_IDENTIFIER = re.compile(r"[A-Za-z_$][\w$]*")
|
|
22
|
+
_COMMENT = re.compile(r"^\s*(#|//|\*|/\*)")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _short(name):
|
|
26
|
+
return name.rpartition(".")[2]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _gone(rel, before, after):
|
|
30
|
+
"""Functions of `before` whose last name segment the functions of `after` no longer carry: {short name: qualified name}."""
|
|
31
|
+
kept = {_short(function["name"]) for function in doc_check.functions(rel, after)}
|
|
32
|
+
old = {}
|
|
33
|
+
for function in doc_check.functions(rel, before):
|
|
34
|
+
short = _short(function["name"])
|
|
35
|
+
if short not in kept and not (short.startswith("__") and short.endswith("__")):
|
|
36
|
+
old.setdefault(short, function["name"])
|
|
37
|
+
return old
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _resigned(rel, before, after):
|
|
41
|
+
"""Qualified names of the functions both texts define once whose signature differs; a name a text defines more than once
|
|
42
|
+
(overloads) is not compared."""
|
|
43
|
+
def hashes(text):
|
|
44
|
+
found = {}
|
|
45
|
+
for function in doc_check.functions(rel, text):
|
|
46
|
+
found.setdefault(function["name"], []).append(doc_scope.signature_hash(function["signature"]))
|
|
47
|
+
return found
|
|
48
|
+
old, new = hashes(before), hashes(after)
|
|
49
|
+
return [name for name, signed in old.items()
|
|
50
|
+
if len(signed) == 1 and len(new.get(name, ())) == 1 and new[name] != signed
|
|
51
|
+
and not (_short(name).startswith("__") and _short(name).endswith("__"))]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _names(expression, short, stem, method):
|
|
55
|
+
"""Whether a call expression can mean the removed function: its last name is the function's, and for a plain function
|
|
56
|
+
it is bare or reached through the module (`stem.name`); a method is reached through any receiver."""
|
|
57
|
+
parts = _IDENTIFIER.findall(expression)
|
|
58
|
+
return bool(parts) and parts[-1] == short and (len(parts) == 1 or method or parts[-2] == stem)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _sites(root, data, rel, name, short, after, removed=True):
|
|
62
|
+
"""The (path, line) pairs that reference one function, from the graph; for a removed function also the imports of its
|
|
63
|
+
name and the calls left in the edited file (`after`)."""
|
|
64
|
+
method, stem = "." in name, posixpath.splitext(posixpath.basename(rel))[0]
|
|
65
|
+
sites = set()
|
|
66
|
+
try:
|
|
67
|
+
found = code_impact.impact(root, f"{rel}::{name}", limit=MAX_SITES)
|
|
68
|
+
sites.update((site["path"], site["line"]) for site in found["callers"] if site["path"] != rel)
|
|
69
|
+
except ValueError:
|
|
70
|
+
pass
|
|
71
|
+
sites.update((call["path"], call["line"]) for call in data.get("unresolved") or []
|
|
72
|
+
if call["path"] != rel and _names(call["expression"], short, stem, method))
|
|
73
|
+
if not removed:
|
|
74
|
+
return sorted(sites)
|
|
75
|
+
sites.update((dep["source"], dep["line"]) for dep in data.get("dependencies") or []
|
|
76
|
+
if dep.get("target") == rel and dep["source"] != rel and _names(dep.get("specifier") or "", short, stem, False))
|
|
77
|
+
called = re.compile(rf"\b{re.escape(short)}\s*\(")
|
|
78
|
+
sites.update((rel, number) for number, line in enumerate(after.splitlines(), 1)
|
|
79
|
+
if called.search(line) and not _COMMENT.match(line))
|
|
80
|
+
return sorted(sites)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _note(data):
|
|
84
|
+
parts = []
|
|
85
|
+
cut = code_graph.left_out_sentence(data)
|
|
86
|
+
if cut:
|
|
87
|
+
parts.append(f"{cut} Callers beyond the cut are not listed.")
|
|
88
|
+
if len(data.get("unresolved") or []) >= UNRESOLVED_CAP:
|
|
89
|
+
parts.append(f"The graph keeps at most {UNRESOLVED_CAP} unresolved calls and this one reached it, so calls by name "
|
|
90
|
+
"beyond them may be missing.")
|
|
91
|
+
if not data.get("symbols") and not data.get("dependencies"):
|
|
92
|
+
parts.append("The code graph is empty: index the project (project_manage action=index) to find the callers.")
|
|
93
|
+
return " ".join(parts)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def review(root, rel, before, after):
|
|
97
|
+
"""The callers that still reference the functions an edit removed or renamed in one file.
|
|
98
|
+
|
|
99
|
+
Parameters: root (str) project folder; rel (str) project-relative path of the edited file; before (str or None) text
|
|
100
|
+
before the edit; after (str) text after it.
|
|
101
|
+
Returns: dict {"removed": [qualified names], "findings": [{"name", "path", "callers": [{"path", "line"}] (up to
|
|
102
|
+
MAX_CALLERS), "total", "also_defined_in": [paths]}], "note": str}; `note` says what the graph cut, "" when it cut
|
|
103
|
+
nothing. The graph is read only when a function is gone; findings hold the removed functions something still calls.
|
|
104
|
+
Raises: RuntimeError when the graph cannot be built, ValueError when the project has no index.
|
|
105
|
+
"""
|
|
106
|
+
gone = _gone(rel, before, after) if before and doc_check.supported(rel) else {}
|
|
107
|
+
if not gone:
|
|
108
|
+
return {"removed": [], "findings": [], "note": ""}
|
|
109
|
+
return {"removed": list(gone.values()), **_findings(root, rel, list(gone.items()), after, True)}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _findings(root, rel, functions, after, removed):
|
|
113
|
+
"""The callers of each (short name, qualified name) pair, as `review` and `signature_review` return them."""
|
|
114
|
+
data = code_graph.build_current(root, None, shared=True)
|
|
115
|
+
defined = {}
|
|
116
|
+
for symbol in data.get("symbols") or []:
|
|
117
|
+
defined.setdefault(_short(symbol["name"]), set()).add(symbol["path"])
|
|
118
|
+
findings = []
|
|
119
|
+
for short, name in functions:
|
|
120
|
+
sites = _sites(root, data, rel, name, short, after, removed)
|
|
121
|
+
if sites:
|
|
122
|
+
findings.append({"name": name, "path": rel, "callers": [{"path": p, "line": n} for p, n in sites[:MAX_CALLERS]],
|
|
123
|
+
"total": len(sites), "also_defined_in": sorted(defined.get(short, set()) - {rel})})
|
|
124
|
+
return {"findings": findings, "note": _note(data)}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def signature_review(root, rel, before, after):
|
|
128
|
+
"""The callers in other files of the functions an edit left with another signature (parameters, order, defaults, return
|
|
129
|
+
annotation) in one file.
|
|
130
|
+
|
|
131
|
+
Parameters: as `review`.
|
|
132
|
+
Returns: dict {"changed": [qualified names], "findings": as `review`, "note": str}; the callers are those outside
|
|
133
|
+
`rel`. The graph is read only when a signature changed.
|
|
134
|
+
Raises: RuntimeError when the graph cannot be built, ValueError when the project has no index.
|
|
135
|
+
"""
|
|
136
|
+
changed = _resigned(rel, before, after) if before and doc_check.supported(rel) else []
|
|
137
|
+
if not changed:
|
|
138
|
+
return {"changed": [], "findings": [], "note": ""}
|
|
139
|
+
return {"changed": changed, **_findings(root, rel, [(_short(name), name) for name in changed], after, False)}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def sites_text(finding):
|
|
143
|
+
"""`app/api.py:6, app/cli.py:3 (+2 more)` for one finding."""
|
|
144
|
+
shown = ", ".join(f"{site['path']}:{site['line']}" for site in finding["callers"])
|
|
145
|
+
more = f" (+{finding['total'] - len(finding['callers'])} more)" if finding["total"] > len(finding["callers"]) else ""
|
|
146
|
+
return shown + more
|
package/scout.py
CHANGED
|
@@ -102,7 +102,8 @@ def is_intent_question(question):
|
|
|
102
102
|
|
|
103
103
|
def disk_lines(root, path, start, end):
|
|
104
104
|
"""`[(line_number, text)]` of lines `start..end` of the file as it is on disk now, or None when the file cannot be
|
|
105
|
-
read by the index rules (secret, dotfile, too large, binary, gone) or holds no line text (docx, pdf, pptx, xlsx
|
|
105
|
+
read by the index rules (secret, dotfile, too large, binary, gone) or holds no line text (docx, pdf, pptx, xlsx;
|
|
106
|
+
`indexer.indexed_lines` serves those)."""
|
|
106
107
|
if document_text.location_kind(path) != "source_line":
|
|
107
108
|
return None
|
|
108
109
|
data, encoding = indexer.read_indexable(root, path)
|
|
@@ -152,7 +153,8 @@ def evidence(root, rows, terms, max_files, window_lines):
|
|
|
152
153
|
"""Evidence items `{id, path, start, end, lines, symbols}` for the first `max_files` files of `rows` (search rows
|
|
153
154
|
`{path, lines: "a-b", snippet, symbols}`), at most WINDOWS_PER_FILE non-overlapping windows per file, each the
|
|
154
155
|
`window_lines` lines of the chunk with most `terms`, read from disk now. A file that cannot be read from disk keeps
|
|
155
|
-
the indexed snippet with its chunk range
|
|
156
|
+
the indexed snippet with its chunk range, except a PDF or Office file, whose window is chosen from the lines the
|
|
157
|
+
index stored (it never extracts the document again)."""
|
|
156
158
|
items, files = [], []
|
|
157
159
|
for row in rows:
|
|
158
160
|
path = row["path"]
|
|
@@ -168,6 +170,8 @@ def evidence(root, rows, terms, max_files, window_lines):
|
|
|
168
170
|
if len(mine) >= WINDOWS_PER_FILE or any(item["start"] <= end and start <= item["end"] for item in mine):
|
|
169
171
|
continue
|
|
170
172
|
lines = disk_lines(root, path, start, end)
|
|
173
|
+
if lines is None and document_text.is_extracted(path):
|
|
174
|
+
lines = indexer.indexed_lines(root, path, start, end)
|
|
171
175
|
if lines:
|
|
172
176
|
window = best_window(lines, terms, window_lines)
|
|
173
177
|
if not window:
|
package/setup_audit.py
CHANGED
|
@@ -30,6 +30,7 @@ import urllib.request
|
|
|
30
30
|
import yaml
|
|
31
31
|
|
|
32
32
|
import atomic_io
|
|
33
|
+
import dependency_notice
|
|
33
34
|
import client_hooks
|
|
34
35
|
import endpoint_sync
|
|
35
36
|
import paths
|
|
@@ -56,7 +57,8 @@ DOCS = {"claude_hooks": "https://code.claude.com/docs/en/hooks.md", "codex_hooks
|
|
|
56
57
|
"claude_memory": "https://code.claude.com/docs/en/memory.md",
|
|
57
58
|
"codex_agents_md": "https://developers.openai.com/codex/guides/agents-md",
|
|
58
59
|
"claude_mcp": "https://code.claude.com/docs/en/mcp.md", "mcp_lifecycle":
|
|
59
|
-
"https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle"
|
|
60
|
+
"https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle",
|
|
61
|
+
"osv": "https://google.github.io/osv.dev/post-v1-querybatch/"}
|
|
60
62
|
_SCRIPT_RE = re.compile(r"""["']?([^\s"'|&;<>]+\.(?:js|cjs|mjs|py|ps1|sh|bat|cmd|exe))["']?""", re.IGNORECASE)
|
|
61
63
|
# The quoted program text after -e / -c / -p / -Command / --eval: code, not a file the hook runs.
|
|
62
64
|
_INLINE_CODE_RE = re.compile(r"""\s(?:-e|-c|-p|-Command|--eval)\s+(?:"(?:[^"\\]|\\.)*"|'[^']*')""", re.IGNORECASE)
|
|
@@ -848,6 +850,36 @@ def _public(section):
|
|
|
848
850
|
return out
|
|
849
851
|
|
|
850
852
|
|
|
853
|
+
def check_dependencies(project):
|
|
854
|
+
"""Known advisories of the dependencies the manifests of the project declare, one warning per manifest. Names and
|
|
855
|
+
versions go to OSV; the shared 6-hour cache keeps the 30-minute schedule from asking again.
|
|
856
|
+
|
|
857
|
+
Parameters: project (str) project folder.
|
|
858
|
+
Returns: list of findings; one `info` finding when the check could not run, none for a project without manifests or
|
|
859
|
+
without advisories.
|
|
860
|
+
"""
|
|
861
|
+
try:
|
|
862
|
+
report = dependency_notice.audit(project)
|
|
863
|
+
except dependency_notice.DependencyCheckError as exc:
|
|
864
|
+
return [_finding("dependency_advisories_unchecked", "info", "project", "dependency advisories",
|
|
865
|
+
f"The dependency advisory check could not run: {exc}", DOCS["osv"],
|
|
866
|
+
"Fix the cause named in the detail (OSV unreachable, or the advisory cache file), then run the "
|
|
867
|
+
"audit again.")]
|
|
868
|
+
findings = []
|
|
869
|
+
for manifest in report["manifests"]:
|
|
870
|
+
if not manifest["vulnerable"]:
|
|
871
|
+
continue
|
|
872
|
+
shown = "; ".join(f"{item['package']}@{item['version']}: {', '.join(item['advisories'][:3])}"
|
|
873
|
+
for item in manifest["vulnerable"][:5])
|
|
874
|
+
more = len(manifest["vulnerable"]) - 5
|
|
875
|
+
findings.append(_finding("dependency_advisories", "warn", "project", manifest["path"],
|
|
876
|
+
f"{len(manifest['vulnerable'])} of {manifest['packages']} dependencies have advisories: "
|
|
877
|
+
f"{shown}" + (f"; {more} more" if more > 0 else "") + ".", DOCS["osv"],
|
|
878
|
+
f"In {manifest['path']}, move each listed package to a version past its advisories; "
|
|
879
|
+
"project_manage action=deps lists every advisory ID.", [manifest["path"]]))
|
|
880
|
+
return findings
|
|
881
|
+
|
|
882
|
+
|
|
851
883
|
def run(project, probe=True, save=True, probe_max_age=0, trigger="manual"):
|
|
852
884
|
"""Collects both harnesses for project, runs every check and returns the report; with save, stores it and
|
|
853
885
|
marks findings new or resolved against the previous report of the same project. probe_max_age reuses recent
|
|
@@ -867,6 +899,7 @@ def run(project, probe=True, save=True, probe_max_age=0, trigger="manual"):
|
|
|
867
899
|
findings += check_agents(claude)
|
|
868
900
|
findings += check_instructions(project, claude, codex)
|
|
869
901
|
findings += check_codex_config(codex)
|
|
902
|
+
findings += check_dependencies(project)
|
|
870
903
|
unverified = list(snapshot.unverified)
|
|
871
904
|
if not probe:
|
|
872
905
|
unverified.append("MCP servers and HTTP hook endpoints were not contacted (probe off)")
|