@allansantos-dev/smart-tool 0.9.79 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +65 -18
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_health.py +180 -0
- package/code_impact.py +13 -1
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +111 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/pr_impact.py
CHANGED
|
@@ -1,26 +1,105 @@
|
|
|
1
1
|
"""Impact report of a change set, in Markdown, for the description or a comment of a pull request.
|
|
2
2
|
|
|
3
3
|
It joins what Smart Tool already computes: the functions the diff touches and the tests to run (affected_tests), the
|
|
4
|
-
callers of each touched function (code_impact)
|
|
4
|
+
callers of each touched function and the tests that reach it (code_impact), the documentation anchors the change leaves
|
|
5
|
+
out of date (doc_scope), the files that usually change with the diff's files and are missing from it (code_health), the decision
|
|
6
|
+
notes that name a touched function (dossier) and the notes the edit hook would have added to the whole diff (hook_decision).
|
|
5
7
|
Nothing is published: the agent pastes the report where it belongs.
|
|
6
8
|
"""
|
|
7
9
|
import os
|
|
10
|
+
import re
|
|
8
11
|
|
|
9
12
|
import affected_tests
|
|
13
|
+
import code_health
|
|
10
14
|
import code_impact
|
|
15
|
+
import config
|
|
11
16
|
import doc_scope
|
|
17
|
+
import dossier
|
|
18
|
+
import hook_decision
|
|
12
19
|
|
|
13
20
|
MAX_FUNCTIONS = 15
|
|
14
21
|
MAX_CALLERS = 8
|
|
22
|
+
TEST_DEPTH = 3
|
|
23
|
+
MAX_CHECKED_FILES = 50
|
|
24
|
+
MAX_COUPLED = 10
|
|
25
|
+
MAX_NOTES_PER_FUNCTION = 2
|
|
15
26
|
|
|
16
27
|
|
|
17
28
|
def _callers(root, label, view_id):
|
|
18
|
-
"""Callers of one touched function, or None when the function is new and nothing calls it yet."""
|
|
29
|
+
"""Callers and reaching tests of one touched function, or None when the function is new and nothing calls it yet."""
|
|
19
30
|
try:
|
|
20
|
-
found = code_impact.impact(root, label, view_id, depth=
|
|
31
|
+
found = code_impact.impact(root, label, view_id, depth=TEST_DEPTH, limit=MAX_CALLERS)
|
|
21
32
|
except ValueError:
|
|
22
33
|
return None
|
|
23
|
-
return found["callers"], found["counts"]["callers"]
|
|
34
|
+
return found["callers"], found["counts"]["callers"], found["counts"]["tests"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _hook_notes(root, base, changes):
|
|
38
|
+
"""The edit hook's notes over the changed files of the diff, which commits made without the hook never got.
|
|
39
|
+
|
|
40
|
+
Parameters: root (str) project folder; base (str) git revision; changes (dict) the `affected_tests.affected` answer.
|
|
41
|
+
Returns: tuple (str notes, "" when nothing is worth saying; str problem, "" when the checks ran).
|
|
42
|
+
"""
|
|
43
|
+
paths = sorted({*changes["changed"]["code"], *changes["changed"]["tests"], *changes["changed"]["other"]})
|
|
44
|
+
triples = []
|
|
45
|
+
for rel in paths[:MAX_CHECKED_FILES]:
|
|
46
|
+
full = os.path.join(root, *rel.split("/"))
|
|
47
|
+
if not os.path.isfile(full):
|
|
48
|
+
continue
|
|
49
|
+
try:
|
|
50
|
+
with open(full, encoding="utf-8") as stream:
|
|
51
|
+
after = stream.read()
|
|
52
|
+
except (OSError, UnicodeDecodeError):
|
|
53
|
+
continue
|
|
54
|
+
try:
|
|
55
|
+
before = affected_tests._git(root, "show", f"{base}:./{rel}")
|
|
56
|
+
except ValueError:
|
|
57
|
+
before = None
|
|
58
|
+
triples.append((full, before, after))
|
|
59
|
+
try:
|
|
60
|
+
return hook_decision.review_diff(triples, config.load_config()), ""
|
|
61
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
62
|
+
return "", f"the edit checks could not run: {exc}"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _coupled(root, view_id, changes):
|
|
66
|
+
"""Files that usually change with the diff's files but are not in it, or the reason none could be computed."""
|
|
67
|
+
paths = sorted({*changes["changed"]["code"], *changes["changed"]["tests"], *changes["changed"]["other"]})
|
|
68
|
+
try:
|
|
69
|
+
found = code_health.coupled_with(root, paths, view_id, MAX_COUPLED)
|
|
70
|
+
except ValueError as exc:
|
|
71
|
+
return [], f"The change coupling was not read: {exc}"
|
|
72
|
+
return found["files"], f"The change coupling was not read: {found['note']}" if found["note"] else None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _name_pattern(name):
|
|
76
|
+
"""How a note must mention a function to count. A compound name (snake_case or camelCase) is distinctive, so the bare
|
|
77
|
+
word counts; a single word such as `load` or `report` is also ordinary prose, so only a code reference counts:
|
|
78
|
+
backticks, or a call with parentheses."""
|
|
79
|
+
escaped = re.escape(name)
|
|
80
|
+
if "_" in name or re.search(r"[a-z][A-Z]", name):
|
|
81
|
+
return re.compile(rf"\b(?:{escaped}|{re.escape(name.replace('_', ' '))})\b", re.I)
|
|
82
|
+
return re.compile(rf"`[\w.]*\b{escaped}\b[\w.()]*`|\b{escaped}\(")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _decisions(root, labels):
|
|
86
|
+
"""Decision notes and commit messages that name each touched function: the dossier search over the project's notes and
|
|
87
|
+
commit history, keeping only hits that contain the function's own name, as a term match alone is too loose.
|
|
88
|
+
|
|
89
|
+
Returns: tuple (dict label -> list of hits, str problem, "" when the search ran whole).
|
|
90
|
+
"""
|
|
91
|
+
found, reasons = {}, []
|
|
92
|
+
for label in labels:
|
|
93
|
+
name = label.partition("::")[2].rpartition(".")[2]
|
|
94
|
+
if not name:
|
|
95
|
+
continue
|
|
96
|
+
answer = dossier.query(root, name.replace("_", " "), max_hits=MAX_NOTES_PER_FUNCTION * 3, history=True)
|
|
97
|
+
reasons += [reason for reason in answer["reasons"] if reason not in reasons]
|
|
98
|
+
named = _name_pattern(name)
|
|
99
|
+
hits = [hit for hit in answer["hits"] if named.search(f"{hit['title']} {hit['snippet']}")]
|
|
100
|
+
if hits:
|
|
101
|
+
found[label] = hits[:MAX_NOTES_PER_FUNCTION]
|
|
102
|
+
return found, f"The notes search was cut: {'; '.join(reasons)}" if reasons else ""
|
|
24
103
|
|
|
25
104
|
|
|
26
105
|
def _stale(root, view_id, paths):
|
|
@@ -42,23 +121,31 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
42
121
|
touched = changes["touched_functions"][:MAX_FUNCTIONS]
|
|
43
122
|
callers = {label: _callers(root, label, view_id) for label in touched}
|
|
44
123
|
stale, stale_note = _stale(root, view_id, changed_code)
|
|
124
|
+
untested = [label for label in touched if callers[label] is None or callers[label][2] == 0]
|
|
125
|
+
notes, notes_problem = _hook_notes(root, base, changes)
|
|
126
|
+
coupled, coupled_note = _coupled(root, view_id, changes)
|
|
127
|
+
decisions, decisions_note = _decisions(root, touched)
|
|
45
128
|
lines = [f"## Impact of the change against `{base}`", "",
|
|
46
129
|
f"{changes['counts']['changed_files']} changed file(s), {len(changes['touched_functions'])} touched "
|
|
47
130
|
f"function(s), {changes['counts']['tests']} test file(s) to run.", ""]
|
|
48
131
|
if changes["touched_functions"]:
|
|
49
|
-
lines += ["### Touched functions and who calls them", "",
|
|
132
|
+
lines += ["### Touched functions and who calls them", "",
|
|
133
|
+
"| Function | Callers | Tests | Called from |", "|---|---|---|---|"]
|
|
50
134
|
for label in touched:
|
|
51
135
|
reached = callers[label]
|
|
52
136
|
if reached is None:
|
|
53
|
-
lines.append(f"| `{label}` | 0 | new: nothing calls it yet |")
|
|
137
|
+
lines.append(f"| `{label}` | 0 | none | new: nothing calls it yet |")
|
|
54
138
|
continue
|
|
55
|
-
sites, total = reached
|
|
139
|
+
sites, total, tests = reached
|
|
56
140
|
shown = ", ".join(f"`{s['function']}` ({s['path']}:{s['line']})" for s in sites)
|
|
57
141
|
more = f" (+{total - len(sites)} more)" if total > len(sites) else ""
|
|
58
|
-
lines.append(f"| `{label}` | {total} | {shown or '-'}{more} |")
|
|
142
|
+
lines.append(f"| `{label}` | {total} | {tests or 'none'} | {shown or '-'}{more} |")
|
|
59
143
|
if len(changes["touched_functions"]) > len(touched):
|
|
60
144
|
lines.append(f"\n{len(changes['touched_functions']) - len(touched)} more touched function(s) not listed.")
|
|
61
145
|
lines.append("")
|
|
146
|
+
lines += ["### Touched functions no test reaches", ""]
|
|
147
|
+
lines += [f"- `{label}`" for label in untested] if untested else ["Every touched function is reached by a test."]
|
|
148
|
+
lines.append("")
|
|
62
149
|
lines += ["### Documentation left out of date", ""]
|
|
63
150
|
if stale_note:
|
|
64
151
|
lines += [stale_note, ""]
|
|
@@ -67,6 +154,21 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
67
154
|
lines += ["", "Regenerate with `project_manage action=docs generate=true` and review the prose of these entries.", ""]
|
|
68
155
|
else:
|
|
69
156
|
lines += ["No documented entry in the changed files is out of date.", ""]
|
|
157
|
+
lines += ["### Edit checks over the whole diff", "", notes or notes_problem or "The edit checks find nothing to add.", ""]
|
|
158
|
+
if touched:
|
|
159
|
+
lines += ["### Decision notes of the touched functions", ""]
|
|
160
|
+
lines += [f"- `{label}`: `{hit['id']}` {hit['title']}" for label, hits in decisions.items() for hit in hits]
|
|
161
|
+
if not decisions:
|
|
162
|
+
lines.append("No decision note or commit message names a touched function.")
|
|
163
|
+
lines += ([decisions_note] if decisions_note else []) + [""]
|
|
164
|
+
lines += ["### Files that usually change with these", ""]
|
|
165
|
+
if coupled_note:
|
|
166
|
+
lines += [coupled_note, ""]
|
|
167
|
+
elif coupled:
|
|
168
|
+
lines += [f"- `{f['path']}` changed in {f['shared']} commits with `{f['with']}` ({f['degree']}%) and is not in this diff"
|
|
169
|
+
for f in coupled] + [""]
|
|
170
|
+
else:
|
|
171
|
+
lines += ["No file that usually changes with these is missing from the diff.", ""]
|
|
70
172
|
lines += ["### Tests to run", ""]
|
|
71
173
|
if changes["tests"]:
|
|
72
174
|
lines += [f"- `{t['path']}`: {t['why']}" for t in changes["tests"]]
|
|
@@ -82,5 +184,6 @@ def report(root, base="HEAD", limit=30, view_id=None):
|
|
|
82
184
|
lines += ["", changes["note"]]
|
|
83
185
|
return {"base": base, "markdown": "\n".join(lines).rstrip("\n") + "\n", "touched": touched,
|
|
84
186
|
"callers": {label: (reached[1] if reached else 0) for label, reached in callers.items()},
|
|
187
|
+
"untested": untested, "coupled": coupled, "decisions": decisions, "edit_checks": notes or notes_problem,
|
|
85
188
|
"stale_anchors": stale, "stale_note": stale_note, "tests": changes["tests"],
|
|
86
189
|
"commands": changes["commands"], "run_all": changes["run_all"], "counts": changes["counts"]}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Child processes under a time cap and a memory cap whose whole tree dies with them."""
|
|
2
|
+
import os
|
|
3
|
+
import subprocess
|
|
4
|
+
|
|
5
|
+
_IS_WINDOWS = os.name == "nt"
|
|
6
|
+
JOB_OBJECT_LIMIT_JOB_MEMORY = 0x200
|
|
7
|
+
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
|
|
8
|
+
STDERR_TAIL_BYTES = 2000
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def kill_on_close_job(job_memory_bytes=None):
|
|
12
|
+
"""A Windows job object that kills every process in it when its handle closes, and, with `job_memory_bytes`, caps
|
|
13
|
+
the memory the whole tree may commit.
|
|
14
|
+
|
|
15
|
+
Parameters: job_memory_bytes (cap for all processes of the job together, None for no cap). Returns: `(kernel32,
|
|
16
|
+
job)`, the library and the job handle. Raises: OSError when the system refuses the job."""
|
|
17
|
+
import ctypes
|
|
18
|
+
from ctypes import wintypes
|
|
19
|
+
|
|
20
|
+
class IoCounters(ctypes.Structure):
|
|
21
|
+
_fields_ = [(name, ctypes.c_ulonglong) for name in
|
|
22
|
+
("read_ops", "write_ops", "other_ops", "read_bytes", "write_bytes", "other_bytes")]
|
|
23
|
+
|
|
24
|
+
class BasicLimits(ctypes.Structure):
|
|
25
|
+
_fields_ = [("per_process_time", ctypes.c_longlong), ("per_job_time", ctypes.c_longlong),
|
|
26
|
+
("flags", wintypes.DWORD), ("min_ws", ctypes.c_size_t), ("max_ws", ctypes.c_size_t),
|
|
27
|
+
("active_processes", wintypes.DWORD), ("affinity", ctypes.c_size_t),
|
|
28
|
+
("priority", wintypes.DWORD), ("scheduling", wintypes.DWORD)]
|
|
29
|
+
|
|
30
|
+
class ExtendedLimits(ctypes.Structure):
|
|
31
|
+
_fields_ = [("basic", BasicLimits), ("io", IoCounters), ("process_memory", ctypes.c_size_t),
|
|
32
|
+
("job_memory", ctypes.c_size_t), ("peak_process_memory", ctypes.c_size_t),
|
|
33
|
+
("peak_job_memory", ctypes.c_size_t)]
|
|
34
|
+
|
|
35
|
+
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
36
|
+
kernel32.CreateJobObjectW.restype = wintypes.HANDLE
|
|
37
|
+
job = kernel32.CreateJobObjectW(None, None)
|
|
38
|
+
if not job:
|
|
39
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
40
|
+
limits = ExtendedLimits()
|
|
41
|
+
limits.basic.flags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
|
42
|
+
if job_memory_bytes:
|
|
43
|
+
limits.basic.flags |= JOB_OBJECT_LIMIT_JOB_MEMORY
|
|
44
|
+
limits.job_memory = job_memory_bytes
|
|
45
|
+
if not kernel32.SetInformationJobObject(wintypes.HANDLE(job), 9, ctypes.byref(limits), ctypes.sizeof(limits)):
|
|
46
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
47
|
+
return kernel32, job
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _stderr_tail(path):
|
|
51
|
+
with open(path, "rb") as stream:
|
|
52
|
+
stream.seek(0, os.SEEK_END)
|
|
53
|
+
stream.seek(max(0, stream.tell() - STDERR_TAIL_BYTES))
|
|
54
|
+
return stream.read().decode("utf-8", "replace").strip()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def run_capped(argv, workdir, timeout, memory_bytes):
|
|
58
|
+
"""Runs `argv` in `workdir` with no window, no stdin, its output discarded and its error stream kept in a file there.
|
|
59
|
+
At the deadline the whole process tree is killed; a process that passes `memory_bytes` is killed by the system
|
|
60
|
+
(Windows job memory limit; elsewhere an address-space limit).
|
|
61
|
+
|
|
62
|
+
Parameters: argv, workdir (an existing folder), timeout (seconds), memory_bytes (cap for the tree). Returns:
|
|
63
|
+
`(returncode, stderr_tail)`, the tail being the last 2,000 bytes of the error stream as text. Raises:
|
|
64
|
+
subprocess.TimeoutExpired after killing the tree; OSError when the program cannot start."""
|
|
65
|
+
error_path = os.path.join(workdir, "stderr.txt")
|
|
66
|
+
with open(error_path, "wb") as errors:
|
|
67
|
+
if _IS_WINDOWS:
|
|
68
|
+
returncode = _run_in_job(argv, workdir, timeout, memory_bytes, errors)
|
|
69
|
+
else:
|
|
70
|
+
returncode = _run_in_group(argv, workdir, timeout, memory_bytes, errors)
|
|
71
|
+
return returncode, _stderr_tail(error_path)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _run_in_job(argv, workdir, timeout, memory_bytes, errors):
|
|
75
|
+
import ctypes
|
|
76
|
+
from ctypes import wintypes
|
|
77
|
+
kernel32, job = kill_on_close_job(memory_bytes)
|
|
78
|
+
try:
|
|
79
|
+
proc = subprocess.Popen(argv, cwd=workdir, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=errors,
|
|
80
|
+
creationflags=subprocess.CREATE_NO_WINDOW)
|
|
81
|
+
if not kernel32.AssignProcessToJobObject(wintypes.HANDLE(job), wintypes.HANDLE(proc._handle)):
|
|
82
|
+
proc.kill()
|
|
83
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
84
|
+
try:
|
|
85
|
+
return proc.wait(timeout)
|
|
86
|
+
except subprocess.TimeoutExpired:
|
|
87
|
+
kernel32.TerminateJobObject(wintypes.HANDLE(job), 1)
|
|
88
|
+
proc.wait()
|
|
89
|
+
raise
|
|
90
|
+
finally:
|
|
91
|
+
kernel32.CloseHandle(wintypes.HANDLE(job))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _run_in_group(argv, workdir, timeout, memory_bytes, errors):
|
|
95
|
+
import resource
|
|
96
|
+
import signal
|
|
97
|
+
|
|
98
|
+
def limit_memory():
|
|
99
|
+
resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes))
|
|
100
|
+
|
|
101
|
+
proc = subprocess.Popen(argv, cwd=workdir, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=errors,
|
|
102
|
+
start_new_session=True, preexec_fn=limit_memory)
|
|
103
|
+
try:
|
|
104
|
+
return proc.wait(timeout)
|
|
105
|
+
except subprocess.TimeoutExpired:
|
|
106
|
+
os.killpg(proc.pid, signal.SIGKILL)
|
|
107
|
+
proc.wait()
|
|
108
|
+
raise
|
package/project_monitor.py
CHANGED
|
@@ -70,11 +70,18 @@ def snapshot(root, scope):
|
|
|
70
70
|
|
|
71
71
|
|
|
72
72
|
def index_model(current=None):
|
|
73
|
-
return local_embedder.resolve(config.load_config()
|
|
73
|
+
return local_embedder.resolve(config.load_config(), current)
|
|
74
74
|
|
|
75
75
|
|
|
76
76
|
def needs_model_rebuild(meta, configured_model):
|
|
77
|
-
|
|
77
|
+
"""Whether the index must be built again for `configured_model`; never while the target model is unknown (None:
|
|
78
|
+
the embeddings home is not running and the index keeps answering as it is)."""
|
|
79
|
+
return configured_model is not None and (meta.get("model_id") != configured_model or not meta.get("exists"))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def needs_model_switch(meta, configured_model):
|
|
83
|
+
"""Whether an existing index holds another model than `configured_model` and must be re-embedded."""
|
|
84
|
+
return bool(meta.get("chunks")) and needs_model_rebuild(meta, configured_model)
|
|
78
85
|
|
|
79
86
|
class ProjectMonitor:
|
|
80
87
|
def __init__(self, enqueue, watcher_factory=directory_watch.DirectoryWatcher):
|
|
@@ -92,6 +99,7 @@ class ProjectMonitor:
|
|
|
92
99
|
self._gap_level = {}
|
|
93
100
|
self._filters = {}
|
|
94
101
|
self._views = {}
|
|
102
|
+
self._switch = set()
|
|
95
103
|
self._lock = threading.RLock()
|
|
96
104
|
self._stop = threading.Event()
|
|
97
105
|
self._thread = None
|
|
@@ -238,6 +246,7 @@ class ProjectMonitor:
|
|
|
238
246
|
projects = project_store.all_projects()
|
|
239
247
|
self._sync_watchers(projects, now)
|
|
240
248
|
busy = project_store.projects_with_pending_job()
|
|
249
|
+
switching = bool(busy)
|
|
241
250
|
for project in projects:
|
|
242
251
|
key = project["id"]
|
|
243
252
|
if not monitored(project):
|
|
@@ -269,7 +278,13 @@ class ProjectMonitor:
|
|
|
269
278
|
if scope is not None:
|
|
270
279
|
changed |= snapshot(project["root"], scope) != indexer.manifest_snapshot(project["root"])
|
|
271
280
|
full = time.time() - meta.get("full_scan_at", 0) >= FULL_CHECK_S
|
|
272
|
-
|
|
281
|
+
target = index_model(meta.get("model_id"))
|
|
282
|
+
free = local_embedder.is_local(target) or target == indexer.LEXICAL_MODEL
|
|
283
|
+
if free and needs_model_switch(meta, target):
|
|
284
|
+
self._switch.add(key)
|
|
285
|
+
else:
|
|
286
|
+
self._switch.discard(key)
|
|
287
|
+
changed |= needs_model_rebuild(meta, target)
|
|
273
288
|
project_store.update(key, last_checked=time.time())
|
|
274
289
|
if changed or full or project.get("dirty_seq", 0) > project.get("indexed_seq", 0):
|
|
275
290
|
with self._lock:
|
|
@@ -281,8 +296,17 @@ class ProjectMonitor:
|
|
|
281
296
|
if first:
|
|
282
297
|
project_store.mark_dirty(key, "Reconciliation found a pending update.")
|
|
283
298
|
project_store.record_changes(key, full=full, scope=rescope)
|
|
284
|
-
#
|
|
285
|
-
#
|
|
299
|
+
# A switch to a model that costs nothing (the embeddings home, or lexical) runs in every update mode, one
|
|
300
|
+
# project at a time, so the old index keeps answering and searches keep a free worker. Outside eager mode
|
|
301
|
+
# any other change only stays pending for the next search, which pays the embeddings.
|
|
302
|
+
if key in self._switch:
|
|
303
|
+
if not switching:
|
|
304
|
+
switching = True
|
|
305
|
+
with self._lock:
|
|
306
|
+
self._pending.setdefault(key, {"paths": set(), "last": now - DEBOUNCE_S, "full": False,
|
|
307
|
+
"force_scope": False, "revision": 0})
|
|
308
|
+
self._dispatch(key, now)
|
|
309
|
+
continue
|
|
286
310
|
if project.get('update_mode', 'on_search') != 'eager':
|
|
287
311
|
with self._lock:
|
|
288
312
|
self._pending.pop(key, None)
|
|
@@ -314,7 +338,8 @@ class ProjectMonitor:
|
|
|
314
338
|
|
|
315
339
|
def _dispatch_quiet(self, now):
|
|
316
340
|
"""Between full passes: sends the projects whose debounce ended, without listing every project of the registry.
|
|
317
|
-
A project that is not eager, is busy or is no longer monitored is left to the next full
|
|
341
|
+
A project that is not eager, is busy, is switching model or is no longer monitored is left to the next full
|
|
342
|
+
pass."""
|
|
318
343
|
self._flush_notes(now)
|
|
319
344
|
with self._lock:
|
|
320
345
|
ready = [key for key, pending in self._pending.items()
|
|
@@ -324,7 +349,8 @@ class ProjectMonitor:
|
|
|
324
349
|
busy = project_store.projects_with_pending_job()
|
|
325
350
|
for key in ready:
|
|
326
351
|
project = project_store.get(key)
|
|
327
|
-
if (project and key not in busy and
|
|
352
|
+
if (project and key not in busy and key not in self._switch and monitored(project)
|
|
353
|
+
and project.get("update_mode", "on_search") == "eager"):
|
|
328
354
|
self._dispatch(key, now)
|
|
329
355
|
|
|
330
356
|
def _run(self):
|
package/project_scheduler.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""Fila justa entre raízes; jobs aguardando a mesma pasta não ocupam workers. Um job de prioridade maior passa à frente
|
|
2
|
-
dos de prioridade menor ainda na fila (o que já está rodando não é interrompido).
|
|
2
|
+
dos de prioridade menor ainda na fila (o que já está rodando não é interrompido). Pistas (`lane`) distintas da mesma pasta
|
|
3
|
+
rodam em paralelo. Com `reserved_lane`, as outras pistas nunca ocupam o último worker livre: uma prioridade só ordena
|
|
4
|
+
a fila e não libera worker, então sem a reserva uma busca espera o fim de um job longo."""
|
|
3
5
|
import collections
|
|
4
6
|
import concurrent.futures
|
|
5
7
|
import threading
|
|
@@ -8,16 +10,19 @@ import project_identity
|
|
|
8
10
|
|
|
9
11
|
|
|
10
12
|
class ProjectScheduler:
|
|
11
|
-
def __init__(self, workers=3):
|
|
13
|
+
def __init__(self, workers=3, reserved_lane=None):
|
|
12
14
|
self.workers = workers
|
|
15
|
+
self.reserved_lane = reserved_lane
|
|
13
16
|
self._pool = concurrent.futures.ThreadPoolExecutor(max_workers=workers, thread_name_prefix="project")
|
|
14
17
|
self._lock = threading.RLock()
|
|
15
18
|
self._queues = collections.OrderedDict()
|
|
16
19
|
self._running = set()
|
|
17
20
|
self._closed = False
|
|
18
21
|
|
|
19
|
-
def submit(self, root, function, *args, priority=0):
|
|
20
|
-
|
|
22
|
+
def submit(self, root, function, *args, priority=0, lane=None):
|
|
23
|
+
"""Queues `function`. Jobs of one folder run one at a time within a lane; jobs of one folder in different
|
|
24
|
+
lanes run side by side (the caller keeps them from writing the same data)."""
|
|
25
|
+
key = (project_identity.canonical_root(root), lane)
|
|
21
26
|
future = concurrent.futures.Future()
|
|
22
27
|
with self._lock:
|
|
23
28
|
if self._closed:
|
|
@@ -31,6 +36,9 @@ class ProjectScheduler:
|
|
|
31
36
|
def _dispatch(self):
|
|
32
37
|
while len(self._running) < self.workers:
|
|
33
38
|
waiting = [root for root in self._queues if root not in self._running]
|
|
39
|
+
if self.reserved_lane is not None and sum(
|
|
40
|
+
lane != self.reserved_lane for _root, lane in self._running) >= self.workers - 1:
|
|
41
|
+
waiting = [key for key in waiting if key[1] == self.reserved_lane]
|
|
34
42
|
if not waiting:
|
|
35
43
|
break
|
|
36
44
|
key = max(waiting, key=lambda root: self._queues[root][0][3])
|
package/scout.py
CHANGED
|
@@ -102,7 +102,8 @@ def is_intent_question(question):
|
|
|
102
102
|
|
|
103
103
|
def disk_lines(root, path, start, end):
|
|
104
104
|
"""`[(line_number, text)]` of lines `start..end` of the file as it is on disk now, or None when the file cannot be
|
|
105
|
-
read by the index rules (secret, dotfile, too large, binary, gone) or holds no line text (docx, pdf, pptx, xlsx
|
|
105
|
+
read by the index rules (secret, dotfile, too large, binary, gone) or holds no line text (docx, pdf, pptx, xlsx;
|
|
106
|
+
`indexer.indexed_lines` serves those)."""
|
|
106
107
|
if document_text.location_kind(path) != "source_line":
|
|
107
108
|
return None
|
|
108
109
|
data, encoding = indexer.read_indexable(root, path)
|
|
@@ -152,7 +153,8 @@ def evidence(root, rows, terms, max_files, window_lines):
|
|
|
152
153
|
"""Evidence items `{id, path, start, end, lines, symbols}` for the first `max_files` files of `rows` (search rows
|
|
153
154
|
`{path, lines: "a-b", snippet, symbols}`), at most WINDOWS_PER_FILE non-overlapping windows per file, each the
|
|
154
155
|
`window_lines` lines of the chunk with most `terms`, read from disk now. A file that cannot be read from disk keeps
|
|
155
|
-
the indexed snippet with its chunk range
|
|
156
|
+
the indexed snippet with its chunk range, except a PDF or Office file, whose window is chosen from the lines the
|
|
157
|
+
index stored (it never extracts the document again)."""
|
|
156
158
|
items, files = [], []
|
|
157
159
|
for row in rows:
|
|
158
160
|
path = row["path"]
|
|
@@ -168,6 +170,8 @@ def evidence(root, rows, terms, max_files, window_lines):
|
|
|
168
170
|
if len(mine) >= WINDOWS_PER_FILE or any(item["start"] <= end and start <= item["end"] for item in mine):
|
|
169
171
|
continue
|
|
170
172
|
lines = disk_lines(root, path, start, end)
|
|
173
|
+
if lines is None and document_text.is_extracted(path):
|
|
174
|
+
lines = indexer.indexed_lines(root, path, start, end)
|
|
171
175
|
if lines:
|
|
172
176
|
window = best_window(lines, terms, window_lines)
|
|
173
177
|
if not window:
|
package/setup_audit.py
CHANGED
|
@@ -30,6 +30,7 @@ import urllib.request
|
|
|
30
30
|
import yaml
|
|
31
31
|
|
|
32
32
|
import atomic_io
|
|
33
|
+
import dependency_notice
|
|
33
34
|
import client_hooks
|
|
34
35
|
import endpoint_sync
|
|
35
36
|
import paths
|
|
@@ -56,7 +57,8 @@ DOCS = {"claude_hooks": "https://code.claude.com/docs/en/hooks.md", "codex_hooks
|
|
|
56
57
|
"claude_memory": "https://code.claude.com/docs/en/memory.md",
|
|
57
58
|
"codex_agents_md": "https://developers.openai.com/codex/guides/agents-md",
|
|
58
59
|
"claude_mcp": "https://code.claude.com/docs/en/mcp.md", "mcp_lifecycle":
|
|
59
|
-
"https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle"
|
|
60
|
+
"https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle",
|
|
61
|
+
"osv": "https://google.github.io/osv.dev/post-v1-querybatch/"}
|
|
60
62
|
_SCRIPT_RE = re.compile(r"""["']?([^\s"'|&;<>]+\.(?:js|cjs|mjs|py|ps1|sh|bat|cmd|exe))["']?""", re.IGNORECASE)
|
|
61
63
|
# The quoted program text after -e / -c / -p / -Command / --eval: code, not a file the hook runs.
|
|
62
64
|
_INLINE_CODE_RE = re.compile(r"""\s(?:-e|-c|-p|-Command|--eval)\s+(?:"(?:[^"\\]|\\.)*"|'[^']*')""", re.IGNORECASE)
|
|
@@ -848,6 +850,36 @@ def _public(section):
|
|
|
848
850
|
return out
|
|
849
851
|
|
|
850
852
|
|
|
853
|
+
def check_dependencies(project):
|
|
854
|
+
"""Known advisories of the dependencies the manifests of the project declare, one warning per manifest. Names and
|
|
855
|
+
versions go to OSV; the shared 6-hour cache keeps the 30-minute schedule from asking again.
|
|
856
|
+
|
|
857
|
+
Parameters: project (str) project folder.
|
|
858
|
+
Returns: list of findings; one `info` finding when the check could not run, none for a project without manifests or
|
|
859
|
+
without advisories.
|
|
860
|
+
"""
|
|
861
|
+
try:
|
|
862
|
+
report = dependency_notice.audit(project)
|
|
863
|
+
except dependency_notice.DependencyCheckError as exc:
|
|
864
|
+
return [_finding("dependency_advisories_unchecked", "info", "project", "dependency advisories",
|
|
865
|
+
f"The dependency advisory check could not run: {exc}", DOCS["osv"],
|
|
866
|
+
"Fix the cause named in the detail (OSV unreachable, or the advisory cache file), then run the "
|
|
867
|
+
"audit again.")]
|
|
868
|
+
findings = []
|
|
869
|
+
for manifest in report["manifests"]:
|
|
870
|
+
if not manifest["vulnerable"]:
|
|
871
|
+
continue
|
|
872
|
+
shown = "; ".join(f"{item['package']}@{item['version']}: {', '.join(item['advisories'][:3])}"
|
|
873
|
+
for item in manifest["vulnerable"][:5])
|
|
874
|
+
more = len(manifest["vulnerable"]) - 5
|
|
875
|
+
findings.append(_finding("dependency_advisories", "warn", "project", manifest["path"],
|
|
876
|
+
f"{len(manifest['vulnerable'])} of {manifest['packages']} dependencies have advisories: "
|
|
877
|
+
f"{shown}" + (f"; {more} more" if more > 0 else "") + ".", DOCS["osv"],
|
|
878
|
+
f"In {manifest['path']}, move each listed package to a version past its advisories; "
|
|
879
|
+
"project_manage action=deps lists every advisory ID.", [manifest["path"]]))
|
|
880
|
+
return findings
|
|
881
|
+
|
|
882
|
+
|
|
851
883
|
def run(project, probe=True, save=True, probe_max_age=0, trigger="manual"):
|
|
852
884
|
"""Collects both harnesses for project, runs every check and returns the report; with save, stores it and
|
|
853
885
|
marks findings new or resolved against the previous report of the same project. probe_max_age reuses recent
|
|
@@ -867,6 +899,7 @@ def run(project, probe=True, save=True, probe_max_age=0, trigger="manual"):
|
|
|
867
899
|
findings += check_agents(claude)
|
|
868
900
|
findings += check_instructions(project, claude, codex)
|
|
869
901
|
findings += check_codex_config(codex)
|
|
902
|
+
findings += check_dependencies(project)
|
|
870
903
|
unverified = list(snapshot.unverified)
|
|
871
904
|
if not probe:
|
|
872
905
|
unverified.append("MCP servers and HTTP hook endpoints were not contacted (probe off)")
|