@allansantos-dev/smart-tool 0.9.77 → 0.9.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +3 -3
- package/disk_cleanup.py +64 -12
- package/index_scope.py +165 -6
- package/package.json +1 -1
- package/project_admission.py +37 -4
- package/project_monitor.py +1 -0
- package/smart_tool_daemon.py +4 -4
- package/version.py +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.78 - beta
|
|
4
|
+
|
|
5
|
+
- A registered project whose folder no longer exists is junk: `project_manage action=junk` lists it and the junk
|
|
6
|
+
path (confirm, and the cleanup at daemon start) removes its index files, never any source file. The manifest
|
|
7
|
+
(`junk-manifests/junk-<time>.json`) is written before deleting and records the project, its registration settings,
|
|
8
|
+
counts, scope per view and the bytes freed. A folder is gone only when its volume root exists, so a project on a
|
|
9
|
+
drive or share that is not connected is kept. Tests: `tests/test_project_admission.py`.
|
|
10
|
+
- The index scope skips Python virtual environments (a folder holding `pyvenv.cfg` or with `venv` in its name) at any
|
|
11
|
+
depth and honors the `.gitignore` files inside the project; a folder named in the scope is still indexed. A new
|
|
12
|
+
project under the system temporary folder is refused with the reason. Tests: `tests/test_scope_rules.py`.
|
|
13
|
+
|
|
3
14
|
## 0.9.77 - beta
|
|
4
15
|
|
|
5
16
|
- The Claude Code hooks Smart Tool writes name their owner and the file that holds their port: the two HTTP hook URLs
|
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@ registered as the `playwright` server where this machine lets a browser be drive
|
|
|
17
17
|
An optional hook routes the agent's native Grep/Glob/Read/Bash/WebSearch/WebFetch calls to these tools when they
|
|
18
18
|
are cheaper, or just tells the agent what Smart Tool would do.
|
|
19
19
|
|
|
20
|
-
**Status:** 0.9.
|
|
20
|
+
**Status:** 0.9.78 beta. Windows only. What changed in each version: [CHANGELOG.md](CHANGELOG.md).
|
|
21
21
|
|
|
22
22
|
## How it works
|
|
23
23
|
|
|
@@ -83,7 +83,7 @@ gateway such as Ollama, nothing about your code leaves the machine.
|
|
|
83
83
|
## Install
|
|
84
84
|
|
|
85
85
|
```powershell
|
|
86
|
-
npx @allansantos-dev/smart-tool@0.9.
|
|
86
|
+
npx @allansantos-dev/smart-tool@0.9.78 install
|
|
87
87
|
```
|
|
88
88
|
|
|
89
89
|
Updates install themselves, on by default: the tray checks npm when it starts (at logon) and every 24 h, and when there
|
|
@@ -390,7 +390,7 @@ fails to load is listed in the setup screen with its error and does not affect t
|
|
|
390
390
|
## Uninstall
|
|
391
391
|
|
|
392
392
|
```powershell
|
|
393
|
-
npx @allansantos-dev/smart-tool@0.9.
|
|
393
|
+
npx @allansantos-dev/smart-tool@0.9.78 uninstall [--target DIR] [--keep-data]
|
|
394
394
|
```
|
|
395
395
|
|
|
396
396
|
It asks nothing and opens no window. It stops the tray and the daemon, removes the logon task, the hook entries of Smart
|
package/disk_cleanup.py
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
"""Removes what Smart Tool left on disk that no registered project needs.
|
|
2
2
|
|
|
3
3
|
Runs once when the daemon starts, before the project monitor: the registry of projects (a folder that is not a
|
|
4
|
-
project, see project_admission.junk_reason, is never one of them, and each removal is listed in a
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
project or no longer exists, see project_admission.junk_reason, is never one of them, and each removal is listed in a
|
|
5
|
+
manifest with what re-registering the project needs), write temporaries orphaned by a crash, generations under
|
|
6
|
+
construction that stopped advancing, the `.previous` copies older versions kept of a replaced generation, index files
|
|
7
|
+
of projects no longer registered, and saved code-graph analyses of index views that no longer exist. Source files are
|
|
8
|
+
never touched."""
|
|
9
|
+
import datetime
|
|
7
10
|
import json
|
|
8
11
|
import os
|
|
9
12
|
import re
|
|
13
|
+
import sqlite3
|
|
10
14
|
import time
|
|
11
15
|
|
|
12
16
|
import atomic_io
|
|
@@ -26,6 +30,10 @@ _STATE_FILE = re.compile(r"([0-9a-f]{16})(?:\.v-[0-9a-f]{16})?(?:\.previous|\.bu
|
|
|
26
30
|
_BUILDING = re.compile(r"[0-9a-f]{16}(?:\.v-[0-9a-f]{16})?\.building-[0-9a-f]+\.sqlite3")
|
|
27
31
|
_PREVIOUS = re.compile(r"[0-9a-f]{16}(?:\.v-[0-9a-f]{16})?\.previous\.sqlite3")
|
|
28
32
|
_TEMPORARY = re.compile(r".+" + re.escape(atomic_io.TMP_SUFFIX) + r"[0-9a-f]{32}")
|
|
33
|
+
_VIEW_INDEX = re.compile(r"([0-9a-f]{16})(?:\.v-[0-9a-f]{16})?\.sqlite3")
|
|
34
|
+
_SCOPE_FILE = re.compile(r"([0-9a-f]{16})(?:\.v-[0-9a-f]{16})?\.scope\.json")
|
|
35
|
+
REGISTRATION_KEYS = ("watch", "paused", "enabled", "update_mode", "created_at", "last_indexed")
|
|
36
|
+
SCOPE_KEYS = ("include", "exclude", "user_exclude", "manual")
|
|
29
37
|
|
|
30
38
|
|
|
31
39
|
def _remove(path, report):
|
|
@@ -36,11 +44,15 @@ def _remove(path, report):
|
|
|
36
44
|
report["bytes"] += size
|
|
37
45
|
|
|
38
46
|
|
|
47
|
+
def _project_keys(root):
|
|
48
|
+
return {project_identity.project_id(root), *project_identity.legacy_ids(root)}
|
|
49
|
+
|
|
50
|
+
|
|
39
51
|
def remove_project_files(root, report=None):
|
|
40
52
|
"""Deletes every index, vector cache, scope and saved graph of the project at root (all Git views, the current and
|
|
41
53
|
legacy ids); returns the report {files, bytes}. Raises RuntimeError when another process holds one open."""
|
|
42
54
|
report = report if report is not None else {"files": [], "bytes": 0}
|
|
43
|
-
keys =
|
|
55
|
+
keys = _project_keys(root)
|
|
44
56
|
for name in sorted(os.listdir(indexer.INDEX_DIR)):
|
|
45
57
|
match = _STATE_FILE.fullmatch(name)
|
|
46
58
|
if not match or match.group(1) not in keys:
|
|
@@ -74,12 +86,50 @@ def junk_projects(errors=None):
|
|
|
74
86
|
return found
|
|
75
87
|
|
|
76
88
|
|
|
89
|
+
def recovery_record(entry):
|
|
90
|
+
"""What it takes to register the junk project again, read before its index files go: the junk entry with its
|
|
91
|
+
registration settings, the files and chunks its indexes hold (summed over Git views, `views` counts them) and the
|
|
92
|
+
scope of each view. An index that cannot be read is listed in `count_errors`, never counted as empty."""
|
|
93
|
+
project = project_store.get(entry["id"]) or {}
|
|
94
|
+
keys = _project_keys(entry["root"])
|
|
95
|
+
record = {**entry, "settings": {key: project[key] for key in REGISTRATION_KEYS if key in project},
|
|
96
|
+
"indexed_files": 0, "chunks": 0, "views": 0, "scopes": {}, "count_errors": []}
|
|
97
|
+
for name in sorted(os.listdir(indexer.INDEX_DIR)):
|
|
98
|
+
path = os.path.join(indexer.INDEX_DIR, name)
|
|
99
|
+
if (match := _VIEW_INDEX.fullmatch(name)) and match.group(1) in keys:
|
|
100
|
+
try:
|
|
101
|
+
conn = indexer._readonly(path)
|
|
102
|
+
try:
|
|
103
|
+
record["indexed_files"] += conn.execute("SELECT count(*) FROM manifest").fetchone()[0]
|
|
104
|
+
record["chunks"] += conn.execute("SELECT count(*) FROM chunks").fetchone()[0]
|
|
105
|
+
record["views"] += 1
|
|
106
|
+
finally:
|
|
107
|
+
conn.close()
|
|
108
|
+
except (OSError, sqlite3.Error) as exc:
|
|
109
|
+
record["count_errors"].append(f"{name}: {exc}")
|
|
110
|
+
elif (match := _SCOPE_FILE.fullmatch(name)) and match.group(1) in keys:
|
|
111
|
+
try:
|
|
112
|
+
with open(path, encoding="utf-8") as stream:
|
|
113
|
+
scope = json.load(stream)
|
|
114
|
+
record["scopes"][name] = {key: scope[key] for key in SCOPE_KEYS if key in scope}
|
|
115
|
+
except (OSError, ValueError) as exc:
|
|
116
|
+
record["count_errors"].append(f"{name}: {exc}")
|
|
117
|
+
return record
|
|
118
|
+
|
|
119
|
+
|
|
77
120
|
def remove_junk(entries, log=None):
|
|
78
121
|
"""Unregisters the given junk projects and deletes their index files (never the project's own files), then writes
|
|
79
|
-
a manifest of what was removed
|
|
80
|
-
|
|
122
|
+
a manifest of what was removed (the `recovery_record` of each, with the index files, bytes freed and the time) and
|
|
123
|
+
what was kept; returns {removed, kept, files, bytes, manifest}. The manifest is first written with every project
|
|
124
|
+
as `pending`, before anything is deleted, so a crash half way still leaves what is needed to recover. A project
|
|
125
|
+
whose index another process holds open is kept and listed with the reason."""
|
|
81
126
|
log = log or (lambda message: None)
|
|
82
127
|
report = {"removed": [], "kept": [], "files": 0, "bytes": 0, "manifest": None}
|
|
128
|
+
records = {entry["id"]: recovery_record(entry) for entry in entries}
|
|
129
|
+
if records:
|
|
130
|
+
os.makedirs(MANIFEST_DIR, exist_ok=True)
|
|
131
|
+
report["manifest"] = os.path.join(MANIFEST_DIR, time.strftime("junk-%Y%m%d-%H%M%S.json"))
|
|
132
|
+
_write_manifest(report["manifest"], {"pending": list(records.values()), "removed": [], "kept": []})
|
|
83
133
|
for entry in entries:
|
|
84
134
|
files = {"files": [], "bytes": 0}
|
|
85
135
|
try:
|
|
@@ -89,18 +139,20 @@ def remove_junk(entries, log=None):
|
|
|
89
139
|
log(f"disk cleanup: {entry['root']} kept: {exc}")
|
|
90
140
|
continue
|
|
91
141
|
project_store.forget(entry["id"])
|
|
92
|
-
report["removed"].append({**entry, "index_files": files["files"], "bytes": files["bytes"]
|
|
142
|
+
report["removed"].append({**records[entry["id"]], "index_files": files["files"], "bytes": files["bytes"],
|
|
143
|
+
"removed_at": datetime.datetime.now().astimezone().isoformat(timespec="seconds")})
|
|
93
144
|
report["files"] += len(files["files"])
|
|
94
145
|
report["bytes"] += files["bytes"]
|
|
95
146
|
log(f"disk cleanup: unregistered {entry['root']}: {entry['reason']}")
|
|
96
|
-
if report["
|
|
97
|
-
|
|
98
|
-
report["manifest"] = os.path.join(MANIFEST_DIR, time.strftime("junk-%Y%m%d-%H%M%S.json"))
|
|
99
|
-
atomic_io.write_text(report["manifest"], json.dumps(
|
|
100
|
-
{"removed": report["removed"], "kept": report["kept"], "source_files_touched": False}, indent=1))
|
|
147
|
+
if report["manifest"]:
|
|
148
|
+
_write_manifest(report["manifest"], {"removed": report["removed"], "kept": report["kept"]})
|
|
101
149
|
return report
|
|
102
150
|
|
|
103
151
|
|
|
152
|
+
def _write_manifest(path, body):
|
|
153
|
+
atomic_io.write_text(path, json.dumps({**body, "source_files_touched": False}, indent=1))
|
|
154
|
+
|
|
155
|
+
|
|
104
156
|
def sweep(extra_roots, log=None):
|
|
105
157
|
"""Removes the waste described in the module docstring and returns {junk, files, bytes, errors}.
|
|
106
158
|
|
package/index_scope.py
CHANGED
|
@@ -12,6 +12,7 @@ import hashlib
|
|
|
12
12
|
import json
|
|
13
13
|
import os
|
|
14
14
|
import paths
|
|
15
|
+
import re
|
|
15
16
|
import time
|
|
16
17
|
|
|
17
18
|
import atomic_io
|
|
@@ -125,6 +126,151 @@ def always_excluded(name):
|
|
|
125
126
|
folded = name.casefold()
|
|
126
127
|
return any(fnmatch.fnmatch(folded, pattern.casefold()) for pattern in _ALWAYS_EXCLUDE)
|
|
127
128
|
|
|
129
|
+
|
|
130
|
+
def venv_name(name):
|
|
131
|
+
"""Whether a folder name marks a Python virtual environment: it holds `venv` (venv, venv-ov, gate-venv, .devenv)."""
|
|
132
|
+
return "venv" in name.casefold()
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def is_venv_folder(path):
|
|
136
|
+
"""Whether the folder at path is a Python virtual environment, by its name (`venv_name`) or because it holds
|
|
137
|
+
`pyvenv.cfg`, which every virtual environment writes whatever its folder is called."""
|
|
138
|
+
return venv_name(os.path.basename(path)) or os.path.isfile(os.path.join(path, "pyvenv.cfg"))
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _is_exempt(rel_path, bases):
|
|
142
|
+
"""Whether rel_path is an include base or a folder above one: the scope names it, so no exclusion rule judges it."""
|
|
143
|
+
folded = rel_path.casefold()
|
|
144
|
+
return any(base.casefold() == folded or base.casefold().startswith(folded + "/") for base in bases if base)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _inside_venv(root, parts, is_dir, bases):
|
|
148
|
+
"""Whether a folder on the way to the path (its last part counts only when it is a folder) is a virtual
|
|
149
|
+
environment, leaving out the folders `_is_exempt` spares."""
|
|
150
|
+
for depth in range(1, (len(parts) if is_dir else len(parts) - 1) + 1):
|
|
151
|
+
if not _is_exempt("/".join(parts[:depth]), bases) and is_venv_folder(os.path.join(root, *parts[:depth])):
|
|
152
|
+
return True
|
|
153
|
+
return False
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _gitignore_source(pattern):
|
|
157
|
+
"""Regular expression source of one `.gitignore` pattern, already stripped of `!` and the closing `/`, matched
|
|
158
|
+
against a path relative to the folder of the file: `*` and `?` stay inside a name, `**` crosses folders, `[...]`
|
|
159
|
+
is a class, `\\x` is a literal, and a pattern with a `/` is anchored to that folder."""
|
|
160
|
+
anchored = "/" in pattern
|
|
161
|
+
pattern = pattern.lstrip("/")
|
|
162
|
+
out, index = [], 0
|
|
163
|
+
while index < len(pattern):
|
|
164
|
+
char = pattern[index]
|
|
165
|
+
if pattern.startswith("**/", index):
|
|
166
|
+
out.append("(?:.*/)?")
|
|
167
|
+
index += 3
|
|
168
|
+
continue
|
|
169
|
+
if pattern.startswith("**", index):
|
|
170
|
+
out.append(".*")
|
|
171
|
+
index += 2
|
|
172
|
+
continue
|
|
173
|
+
close = pattern.find("]", index + 2) if char == "[" else -1
|
|
174
|
+
if char == "*":
|
|
175
|
+
out.append("[^/]*")
|
|
176
|
+
elif char == "?":
|
|
177
|
+
out.append("[^/]")
|
|
178
|
+
elif close != -1:
|
|
179
|
+
body = pattern[index + 1:close]
|
|
180
|
+
out.append("[" + ("^" + body[1:] if body.startswith("!") else body).replace("\\", "\\\\") + "]")
|
|
181
|
+
index = close
|
|
182
|
+
elif char == "\\" and index + 1 < len(pattern):
|
|
183
|
+
index += 1
|
|
184
|
+
out.append(re.escape(pattern[index]))
|
|
185
|
+
else:
|
|
186
|
+
out.append(re.escape(char))
|
|
187
|
+
index += 1
|
|
188
|
+
return ("" if anchored else "(?:.*/)?") + "".join(out)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _gitignore_runs(text):
|
|
192
|
+
"""The rules of one `.gitignore` as runs of consecutive rules with the same polarity, in file order:
|
|
193
|
+
`[(negated, regex_for_any_entry, regex_for_files_or_None)]`. A pattern that is not a valid glob matches nothing,
|
|
194
|
+
as in Git."""
|
|
195
|
+
flags = re.IGNORECASE if os.name == "nt" else 0
|
|
196
|
+
runs = []
|
|
197
|
+
for raw in text.splitlines():
|
|
198
|
+
line = raw if raw.endswith("\\ ") else raw.rstrip(" \t")
|
|
199
|
+
if not line or line.startswith("#"):
|
|
200
|
+
continue
|
|
201
|
+
negated = line.startswith("!")
|
|
202
|
+
line = line[1:] if negated else line
|
|
203
|
+
dir_only = line.endswith("/")
|
|
204
|
+
line = line.rstrip("/")
|
|
205
|
+
if not line:
|
|
206
|
+
continue
|
|
207
|
+
source = _gitignore_source(line)
|
|
208
|
+
try:
|
|
209
|
+
re.compile(source, flags)
|
|
210
|
+
except re.error:
|
|
211
|
+
continue
|
|
212
|
+
if not runs or runs[-1][0] != negated:
|
|
213
|
+
runs.append((negated, [], []))
|
|
214
|
+
runs[-1][1].append(source)
|
|
215
|
+
if not dir_only:
|
|
216
|
+
runs[-1][2].append(source)
|
|
217
|
+
|
|
218
|
+
def combine(sources):
|
|
219
|
+
return re.compile("(?:" + "|".join(f"(?:{source})" for source in sources) + r")\Z", flags) if sources else None
|
|
220
|
+
|
|
221
|
+
return [(negated, combine(anywhere), combine(files)) for negated, anywhere, files in runs]
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class GitIgnore:
|
|
225
|
+
"""What the `.gitignore` files inside a project folder ignore: the file of each folder applies to that folder and
|
|
226
|
+
below, a deeper file overrides a shallower one, and within a file the last matching rule wins. Read from the
|
|
227
|
+
project folder only (not Git's global excludes, `.git/info/exclude` or the files of a repository above the
|
|
228
|
+
project), a file is read when first needed and again after `TTL_S`, so one instance can serve a burst of file
|
|
229
|
+
events without going stale."""
|
|
230
|
+
|
|
231
|
+
TTL_S = 5
|
|
232
|
+
|
|
233
|
+
def __init__(self, root):
|
|
234
|
+
self.root = root
|
|
235
|
+
self._folders = {}
|
|
236
|
+
|
|
237
|
+
def _runs(self, folder):
|
|
238
|
+
now = time.monotonic()
|
|
239
|
+
hit = self._folders.get(folder)
|
|
240
|
+
if hit and now - hit[0] < self.TTL_S:
|
|
241
|
+
return hit[1]
|
|
242
|
+
try:
|
|
243
|
+
with open(os.path.join(self.root, folder, ".gitignore"), encoding="utf-8-sig", errors="replace") as stream:
|
|
244
|
+
text = stream.read()
|
|
245
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
246
|
+
text = ""
|
|
247
|
+
runs = _gitignore_runs(text)
|
|
248
|
+
self._folders[folder] = (now, runs)
|
|
249
|
+
return runs
|
|
250
|
+
|
|
251
|
+
def matches(self, rel_path, is_dir):
|
|
252
|
+
"""Whether the rules ignore this entry by themselves, which is the question during a walk that never enters an
|
|
253
|
+
ignored folder. `rel_path` uses `/`."""
|
|
254
|
+
parts = rel_path.split("/")
|
|
255
|
+
verdict = False
|
|
256
|
+
for depth in range(len(parts)):
|
|
257
|
+
target = "/".join(parts[depth:])
|
|
258
|
+
for negated, anywhere, files in self._runs("/".join(parts[:depth])):
|
|
259
|
+
pattern = anywhere if is_dir else files
|
|
260
|
+
if pattern and pattern.match(target):
|
|
261
|
+
verdict = not negated
|
|
262
|
+
return verdict
|
|
263
|
+
|
|
264
|
+
def ignored(self, rel_path, is_dir, bases=()):
|
|
265
|
+
"""Whether the entry or any folder above it is ignored, for a path met out of a walk (a file event). The
|
|
266
|
+
include `bases` and the folders above them are spared: the scope names them."""
|
|
267
|
+
parts = rel_path.split("/")
|
|
268
|
+
for depth in range(1, len(parts) + 1):
|
|
269
|
+
sub = "/".join(parts[:depth])
|
|
270
|
+
if not _is_exempt(sub, bases) and self.matches(sub, depth < len(parts) or is_dir):
|
|
271
|
+
return True
|
|
272
|
+
return False
|
|
273
|
+
|
|
128
274
|
# Versão do contrato de decisão de escopo. Mudar o que é perguntado ao decisor exige
|
|
129
275
|
# incrementar: sem isso, um repositório com escopo já cacheado nunca é reavaliado, porque
|
|
130
276
|
# `needs_rescan` só olha diretório de topo novo.
|
|
@@ -194,6 +340,7 @@ def _top_level_dirs(root):
|
|
|
194
340
|
and not always_excluded(name)
|
|
195
341
|
and name not in skipped
|
|
196
342
|
and os.path.isdir(os.path.join(root, name))
|
|
343
|
+
and not is_venv_folder(os.path.join(root, name))
|
|
197
344
|
and _within_root(root, os.path.join(root, name))
|
|
198
345
|
)
|
|
199
346
|
|
|
@@ -259,6 +406,7 @@ def _structure(root, max_dirs=1500, keep=None):
|
|
|
259
406
|
found = []
|
|
260
407
|
for current, dirs, _files in os.walk(root):
|
|
261
408
|
dirs[:] = sorted(d for d in dirs if not d.startswith(".") and not always_excluded(d)
|
|
409
|
+
and not is_venv_folder(os.path.join(current, d))
|
|
262
410
|
and _within_root(root, os.path.join(current, d))
|
|
263
411
|
and (keep is None or keep(os.path.relpath(os.path.join(current, d), root).replace(os.sep, "/"))))
|
|
264
412
|
for name in dirs:
|
|
@@ -324,6 +472,7 @@ def _inspect_directory(root, rel_path, max_files=4000, sample_size=15, top_ext=1
|
|
|
324
472
|
for dirpath, _dirnames, filenames in os.walk(base_full):
|
|
325
473
|
_dirnames[:] = [name for name in _dirnames if not name.startswith(".")
|
|
326
474
|
and not always_excluded(name)
|
|
475
|
+
and not is_venv_folder(os.path.join(dirpath, name))
|
|
327
476
|
and _within_root(root, os.path.join(dirpath, name))]
|
|
328
477
|
depth = os.path.relpath(dirpath, base_full).count(os.sep) + (0 if dirpath == base_full else 1)
|
|
329
478
|
max_depth = max(max_depth, depth)
|
|
@@ -387,8 +536,10 @@ def _build_scope_prompt(root, ambiguous_dirs, root_files, doc_context):
|
|
|
387
536
|
"documentation.\n\n"
|
|
388
537
|
"`include` lists folder paths relative to the root, without wildcards; a folder you leave out of `include` is "
|
|
389
538
|
"not indexed, and an empty `include` indexes only the loose root files. "
|
|
390
|
-
"
|
|
391
|
-
"
|
|
539
|
+
"Whatever .gitignore ignores, and every folder named like *venv* or holding pyvenv.cfg (a Python virtual "
|
|
540
|
+
"environment), is not indexed, unless `include` names that folder or file: name the gitignored ones that are "
|
|
541
|
+
"useful for search (e.g. docs/, ADRs, CLAUDE.md). "
|
|
542
|
+
"Java, Angular, HTML/CSS, plain text and Markdown are supported sources. "
|
|
392
543
|
"DOCX, PPTX, XLSX and PDF documents are supported through bounded local text extraction; do not exclude "
|
|
393
544
|
"useful Office or PDF documentation merely because its container is binary. "
|
|
394
545
|
"Exclude directories that inspection shows are generated artifacts or "
|
|
@@ -555,15 +706,19 @@ def scope_filter(scope, root=None):
|
|
|
555
706
|
set(mechanical_excludes(root) if root else ()) | _ALWAYS_EXCLUDE | _ALWAYS_EXCLUDE_FILES)
|
|
556
707
|
whole = "" in bases
|
|
557
708
|
nested = _nested_matcher(root)
|
|
709
|
+
ignore = GitIgnore(root) if root else None
|
|
558
710
|
|
|
559
711
|
def keep(rel_path):
|
|
560
712
|
rel_path = rel_path.replace("\\", "/")
|
|
561
713
|
parts = rel_path.split("/")
|
|
562
714
|
if matcher(rel_path) or is_hidden(rel_path) or nested(rel_path):
|
|
563
715
|
return False
|
|
564
|
-
if whole or len(parts) == 1:
|
|
716
|
+
if not (whole or len(parts) == 1 or any(rel_path == base or rel_path.startswith(base + "/") for base in bases)):
|
|
717
|
+
return False
|
|
718
|
+
if not root:
|
|
565
719
|
return True
|
|
566
|
-
|
|
720
|
+
is_dir = os.path.isdir(os.path.join(root, *parts))
|
|
721
|
+
return not (_inside_venv(root, parts, is_dir, bases) or ignore.ignored(rel_path, is_dir, bases))
|
|
567
722
|
|
|
568
723
|
return keep
|
|
569
724
|
|
|
@@ -581,6 +736,7 @@ def resolve_included_files(root, scope, cancel_check=None):
|
|
|
581
736
|
excluded = _exclude_matcher(set(scope.get("exclude") or []) | set(scope.get("user_exclude") or []) |
|
|
582
737
|
set(mechanical_excludes(root)) | _ALWAYS_EXCLUDE | _ALWAYS_EXCLUDE_FILES)
|
|
583
738
|
nested = _nested_matcher(root)
|
|
739
|
+
ignore = GitIgnore(root)
|
|
584
740
|
|
|
585
741
|
def matcher(rel_path):
|
|
586
742
|
return excluded(rel_path) or nested(rel_path)
|
|
@@ -589,7 +745,7 @@ def resolve_included_files(root, scope, cancel_check=None):
|
|
|
589
745
|
def walk_error(error):
|
|
590
746
|
raise OSError(f"Could not list {error.filename} ({error.strerror}); try again when it is accessible.") from error
|
|
591
747
|
if "" not in include:
|
|
592
|
-
files.extend(_root_level_files(root, matcher))
|
|
748
|
+
files.extend(_root_level_files(root, lambda rel: matcher(rel) or ignore.matches(rel, False)))
|
|
593
749
|
for base in include:
|
|
594
750
|
base_full = os.path.join(root, base) if base else root
|
|
595
751
|
# `include` vem de tool call de um LLM cujo prompt embute documentação do
|
|
@@ -613,12 +769,15 @@ def resolve_included_files(root, scope, cancel_check=None):
|
|
|
613
769
|
d for d in dirnames
|
|
614
770
|
if not d.startswith(".")
|
|
615
771
|
and not matcher(f"{rel_dir}/{d}" if rel_dir else d)
|
|
772
|
+
and not is_venv_folder(os.path.join(dirpath, d))
|
|
773
|
+
and not ignore.matches(f"{rel_dir}/{d}" if rel_dir else d, True)
|
|
616
774
|
# `os.walk` não desce em symlink de diretório, mas `os.path.islink()` é
|
|
617
775
|
# False para junction do Windows (`mklink /J`, sem admin) — e desce nela.
|
|
618
776
|
and _within_root(root, os.path.join(dirpath, d))
|
|
619
777
|
]
|
|
620
778
|
for name in filenames:
|
|
621
779
|
rel_path = f"{rel_dir}/{name}" if rel_dir else name
|
|
622
|
-
if not name.startswith(".") and not matcher(rel_path) and
|
|
780
|
+
if (not name.startswith(".") and not matcher(rel_path) and not ignore.matches(rel_path, False)
|
|
781
|
+
and _within_root(root, os.path.join(root, rel_path))):
|
|
623
782
|
files.append(rel_path)
|
|
624
783
|
return sorted(set(files))
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.78",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Smart Tool contributors",
|
package/project_admission.py
CHANGED
|
@@ -3,14 +3,17 @@
|
|
|
3
3
|
A project is the folder where the agent session started (or the repository that holds it), never a folder an agent
|
|
4
4
|
merely navigated to: personal folders, drive roots, application data, system folders, build output and a Java home
|
|
5
5
|
are refused, a subfolder resolves to the registered project or the repository that holds it, and a session whose
|
|
6
|
-
origin is known only indexes its own project. Registered folders that break these rules
|
|
6
|
+
origin is known only indexes its own project. Registered folders that break these rules, and registered folders that no
|
|
7
|
+
longer exist, are junk (`junk_reason`)."""
|
|
7
8
|
import os
|
|
8
9
|
import tempfile
|
|
9
10
|
|
|
10
11
|
import project_identity
|
|
11
12
|
import project_store
|
|
13
|
+
import redirect_rule
|
|
12
14
|
|
|
13
15
|
PERSONAL_NAMES = ("Desktop", "Documents", "Downloads", "Pictures", "Videos", "Music")
|
|
16
|
+
GONE_REASON = "its folder no longer exists"
|
|
14
17
|
TOOL_FOLDERS = frozenset(("bin", "obj", "node_modules", ".git", "site-packages", "__pycache__"))
|
|
15
18
|
POSIX_SYSTEM = ("/usr", "/bin", "/sbin", "/etc", "/var", "/opt", "/lib", "/proc", "/sys", "/dev", "/boot")
|
|
16
19
|
|
|
@@ -100,6 +103,17 @@ def refusal(root):
|
|
|
100
103
|
return None
|
|
101
104
|
|
|
102
105
|
|
|
106
|
+
def temporary_refusal(root):
|
|
107
|
+
"""Why a new project cannot live in root, or None: a copy made in the system's temporary folder for a measurement
|
|
108
|
+
or a build is gone soon and would leave its index behind. Folders of `SMART_TOOL_SCRATCH` stay allowed, and a
|
|
109
|
+
registered project is not asked (see `admit`)."""
|
|
110
|
+
norm = _norm(project_identity.display_root(root))
|
|
111
|
+
if _inside(norm, _norm(project_identity.display_root(redirect_rule.system_temp()))) and not any(
|
|
112
|
+
_inside(norm, base) for base in _env_folders("SMART_TOOL_SCRATCH")):
|
|
113
|
+
return "inside the system temporary folder"
|
|
114
|
+
return None
|
|
115
|
+
|
|
116
|
+
|
|
103
117
|
def repository_root(path):
|
|
104
118
|
"""The nearest folder at or above path that holds `.git` and could be a project, or None. A repository the rules
|
|
105
119
|
refuse (a dotfiles repository in the home folder) is not a project to resolve to."""
|
|
@@ -134,7 +148,7 @@ def admit(path, origin=None):
|
|
|
134
148
|
root, registered = resolve(path)
|
|
135
149
|
if registered:
|
|
136
150
|
return root
|
|
137
|
-
reason = refusal(root)
|
|
151
|
+
reason = refusal(root) or temporary_refusal(root)
|
|
138
152
|
if reason:
|
|
139
153
|
raise ValueError(f"Not indexing {root}: it is {reason}. A project is the folder where the session started; "
|
|
140
154
|
"pass that folder as project_root.")
|
|
@@ -151,10 +165,29 @@ def admit(path, origin=None):
|
|
|
151
165
|
return root
|
|
152
166
|
|
|
153
167
|
|
|
168
|
+
def folder_gone(root):
|
|
169
|
+
"""Whether root is confirmed missing, not merely unreachable. Both must hold: looking at root fails with "does not
|
|
170
|
+
exist", and the drive or network share root it sits on (`D:\\`, `\\\\server\\share\\`, `/`) is present, so the
|
|
171
|
+
missing part is below a mounted volume. A drive that is not mounted, a share that is offline and any other error
|
|
172
|
+
(access denied, device not ready) are not "gone": the folder may come back."""
|
|
173
|
+
try:
|
|
174
|
+
os.stat(root)
|
|
175
|
+
return False
|
|
176
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
177
|
+
pass
|
|
178
|
+
except OSError:
|
|
179
|
+
return False
|
|
180
|
+
drive = os.path.splitdrive(os.path.abspath(root))[0]
|
|
181
|
+
return os.path.isdir(drive + os.sep)
|
|
182
|
+
|
|
183
|
+
|
|
154
184
|
def junk_reason(project):
|
|
155
|
-
"""Why a registered project was never a project, or None: its folder
|
|
156
|
-
more projects, or it is a subfolder of a repository or of another
|
|
185
|
+
"""Why a registered project was never a project, or None: its folder no longer exists (`folder_gone`), its folder
|
|
186
|
+
is refused by `refusal`, it holds two or more projects, or it is a subfolder of a repository or of another
|
|
187
|
+
registered project. A project whose folder exists is judged by the last three rules only."""
|
|
157
188
|
root = project["root"]
|
|
189
|
+
if folder_gone(root):
|
|
190
|
+
return GONE_REASON
|
|
158
191
|
if not os.path.isdir(root):
|
|
159
192
|
return None
|
|
160
193
|
reason = refusal(root)
|
package/project_monitor.py
CHANGED
|
@@ -35,6 +35,7 @@ def relevant_path(path):
|
|
|
35
35
|
parts = path.replace("\\", "/").strip("/").split("/")
|
|
36
36
|
return bool(parts and all(part not in ("", ".", "..") and not part.startswith(".")
|
|
37
37
|
and not index_scope.always_excluded(part) for part in parts)
|
|
38
|
+
and not any(index_scope.venv_name(part) for part in parts[:-1])
|
|
38
39
|
and not index_scope.is_transient(parts[-1]))
|
|
39
40
|
|
|
40
41
|
|
package/smart_tool_daemon.py
CHANGED
|
@@ -373,7 +373,7 @@ SERVER_INSTRUCTIONS = (
|
|
|
373
373
|
TOOLS = [
|
|
374
374
|
{
|
|
375
375
|
"name": "project_manage",
|
|
376
|
-
"description": "Manages folders and indexing. Every action takes project_root (the folder) or project_id. list gives a short line per project (a project whose folder is gone shows as unavailable with the reason); status shows storage, scope and jobs, and with a path inside a registered project answers for the deepest one that holds it (resolved_from). graph with symbol (function, Class.method, class name, or path::name when the name repeats) answers what an edit touches: definition, callers with line, calls, tests that reach it through static calls (depth hops, default 3) and files importing it; use it before changing a function to know what to update. affected_tests answers which tests to run after editing: every test file importing a changed file (git diff against base, default HEAD, untracked included) plus changed tests, likeliest failures first, with the command to run them and run_all when config or unanalyzed code changed. pr_impact (base, default HEAD; main or origin/main for a branch) returns a Markdown report of the change for a pull request: the functions the diff touches with their callers, the documentation entries it leaves out of date and the tests to run with their command; it publishes nothing. graph without symbol reads the indexed view: coverage, references and static calls in Java, Angular, Python, Go and JS/TS/JSX/TSX; includes HTML/CSS/Markdown references and TXT/DOCX coverage; with file_path, narrows to that file's symbols, imports and calls (lists capped by limit). inspect lists the files of a view_id; with file_path, returns only that file's chunks. usage shows consumption and reuse; docs lists public functions without a docstring (Python docstring, JSDoc, Javadoc, Go doc comment starting with the name; private, nested and override functions excluded) with coverage percent, excluding tests unless include_tests=true, for documenting a project that started without it; docs generate=true writes the documentation scope to docs/scope in the project in three levels tied to the code: technical pages by folder (an entry per public function anchored on path::name with signature, first doc line, layer, callers, calls and tests, plus a Mermaid flow by layer), functional.md (each entry point and the functions its flow reaches, by layer) and simple.md; the prose between the prose markers is kept by regeneration; docs write_prose=true writes the empty prose blocks with the Docs model of the setup (each paragraph checked against the code first) within about 40 s and says how many are left, call it again until none is; docs check=true reports anchors whose function changed signature or is gone and broken links; docs validate=true checks every claim of docs/scope against the code (anchors, links, line numbers, callers and tests, coverage, entry points, command-line options, prose names, counts, glossary, navigation, versions) and writes docs/scope/validation.md with pass/fail per check, the evidence, a score and the verdict (generate runs it too); docs with symbol lists where that function is documented; layers declares the project's layers (ordered highest first, each with path globs and annotations): without layers it returns the saved rule or a suggestion read from folder names and annotations, with files, symbols and calls between layers (direction down or up); pass layers to preview a rule and the same layers with confirm=true to save it in the project (.smart-tool/layers.json); duplicates lists duplicated functions (identical, near-identical and semantic bodies, min_similarity default 0.90; below that most pairs measured were false positives), excluding tests unless include_tests=true; fix real duplicates; only dismiss with duplicates_dismiss (finding_id, reason=false_positive|intentional, note) what is not a duplicate or is a copy kept on purpose: the finding stays hidden until the code changes, and duplicates_restore reopens it; integration shows the MCP client, hook and tool usage metrics; install_hook_preview shows the change that would install the hook redirecting Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this session's client and returns a plan_id; install_hook writes it with that plan_id and confirm=true, with a backup, only after the user approves the change; search_limits saves the project's default top_k per block ({code, test, doc}, each 1-20, total at most 20); storage shows disk usage and views; pin_view protects a view; each view carries stale (not updated for 30 days, the test of the daily cleanup that removes them) and stale_bytes sums them; cleanup_preview and cleanup_commit remove selected views after confirmation; compare compares two stored views, without checkout or embeddings. register adds a folder; preview checks the scope; index/rebuild update; pause/cancel keep checkpoints; resume continues; policy picks on_search/eager; scope adjusts the scope; remove deletes only the local index. setup_audit (project_root) reports what Claude Code and Codex actually load for that folder (settings layers, plugins, hooks, MCP servers, skills, CLAUDE.md/AGENTS.md) and where they collide: hooks deciding on the same tool in parallel, missing hook scripts, silenced hooks, MCP servers down or offering duplicate tools, skill name and budget problems, AGENTS.md that only Codex reads, broken @imports; what could not be checked is listed as not verified; each finding carries its fix (the file to edit or the command to run), applied by the agent when the user agrees, never by Smart Tool; mode=last returns the stored report instead of auditing again. redirect_proposals (no project_root) lists the redirect patterns proposed from allowed searches, most matched first, with a guide; redirect_decide decides them by ids: accept, instruct (tip without blocking), reject, or remove an accepted one. junk (no project_root) lists the registered projects that were never projects: a personal, system, application-data or build folder, a subfolder of a repository or of another project, a folder of projects; with confirm=true it unregisters them and deletes only their index files, and the manifest says what went. A project is the folder where the session started, so smart_search and register refuse a folder that is not a project and index a new one only from the session that started in it. Jobs via smart_search_result.",
|
|
376
|
+
"description": "Manages folders and indexing. Every action takes project_root (the folder) or project_id. list gives a short line per project (a project whose folder is gone shows as unavailable with the reason); status shows storage, scope and jobs, and with a path inside a registered project answers for the deepest one that holds it (resolved_from). graph with symbol (function, Class.method, class name, or path::name when the name repeats) answers what an edit touches: definition, callers with line, calls, tests that reach it through static calls (depth hops, default 3) and files importing it; use it before changing a function to know what to update. affected_tests answers which tests to run after editing: every test file importing a changed file (git diff against base, default HEAD, untracked included) plus changed tests, likeliest failures first, with the command to run them and run_all when config or unanalyzed code changed. pr_impact (base, default HEAD; main or origin/main for a branch) returns a Markdown report of the change for a pull request: the functions the diff touches with their callers, the documentation entries it leaves out of date and the tests to run with their command; it publishes nothing. graph without symbol reads the indexed view: coverage, references and static calls in Java, Angular, Python, Go and JS/TS/JSX/TSX; includes HTML/CSS/Markdown references and TXT/DOCX coverage; with file_path, narrows to that file's symbols, imports and calls (lists capped by limit). inspect lists the files of a view_id; with file_path, returns only that file's chunks. usage shows consumption and reuse; docs lists public functions without a docstring (Python docstring, JSDoc, Javadoc, Go doc comment starting with the name; private, nested and override functions excluded) with coverage percent, excluding tests unless include_tests=true, for documenting a project that started without it; docs generate=true writes the documentation scope to docs/scope in the project in three levels tied to the code: technical pages by folder (an entry per public function anchored on path::name with signature, first doc line, layer, callers, calls and tests, plus a Mermaid flow by layer), functional.md (each entry point and the functions its flow reaches, by layer) and simple.md; the prose between the prose markers is kept by regeneration; docs write_prose=true writes the empty prose blocks with the Docs model of the setup (each paragraph checked against the code first) within about 40 s and says how many are left, call it again until none is; docs check=true reports anchors whose function changed signature or is gone and broken links; docs validate=true checks every claim of docs/scope against the code (anchors, links, line numbers, callers and tests, coverage, entry points, command-line options, prose names, counts, glossary, navigation, versions) and writes docs/scope/validation.md with pass/fail per check, the evidence, a score and the verdict (generate runs it too); docs with symbol lists where that function is documented; layers declares the project's layers (ordered highest first, each with path globs and annotations): without layers it returns the saved rule or a suggestion read from folder names and annotations, with files, symbols and calls between layers (direction down or up); pass layers to preview a rule and the same layers with confirm=true to save it in the project (.smart-tool/layers.json); duplicates lists duplicated functions (identical, near-identical and semantic bodies, min_similarity default 0.90; below that most pairs measured were false positives), excluding tests unless include_tests=true; fix real duplicates; only dismiss with duplicates_dismiss (finding_id, reason=false_positive|intentional, note) what is not a duplicate or is a copy kept on purpose: the finding stays hidden until the code changes, and duplicates_restore reopens it; integration shows the MCP client, hook and tool usage metrics; install_hook_preview shows the change that would install the hook redirecting Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this session's client and returns a plan_id; install_hook writes it with that plan_id and confirm=true, with a backup, only after the user approves the change; search_limits saves the project's default top_k per block ({code, test, doc}, each 1-20, total at most 20); storage shows disk usage and views; pin_view protects a view; each view carries stale (not updated for 30 days, the test of the daily cleanup that removes them) and stale_bytes sums them; cleanup_preview and cleanup_commit remove selected views after confirmation; compare compares two stored views, without checkout or embeddings. register adds a folder; preview checks the scope; index/rebuild update; pause/cancel keep checkpoints; resume continues; policy picks on_search/eager; scope adjusts the scope; remove deletes only the local index. setup_audit (project_root) reports what Claude Code and Codex actually load for that folder (settings layers, plugins, hooks, MCP servers, skills, CLAUDE.md/AGENTS.md) and where they collide: hooks deciding on the same tool in parallel, missing hook scripts, silenced hooks, MCP servers down or offering duplicate tools, skill name and budget problems, AGENTS.md that only Codex reads, broken @imports; what could not be checked is listed as not verified; each finding carries its fix (the file to edit or the command to run), applied by the agent when the user agrees, never by Smart Tool; mode=last returns the stored report instead of auditing again. redirect_proposals (no project_root) lists the redirect patterns proposed from allowed searches, most matched first, with a guide; redirect_decide decides them by ids: accept, instruct (tip without blocking), reject, or remove an accepted one. junk (no project_root) lists the registered projects that were never projects: a personal, system, application-data or build folder, a subfolder of a repository or of another project, a folder of projects, or a folder that no longer exists (on a drive or share that is mounted); with confirm=true it unregisters them and deletes only their index files, and the manifest says what went. A project is the folder where the session started, so smart_search and register refuse a folder that is not a project and index a new one only from the session that started in it. Jobs via smart_search_result.",
|
|
377
377
|
"inputSchema": {
|
|
378
378
|
"type": "object", "properties": {
|
|
379
379
|
"action": {"type": "string", "enum": ["list", "status", "register", "preview", "index", "rebuild", "pause", "cancel", "resume", "watch", "scope", "remove", "relocate", "probe", "policy", "inspect", "graph", "usage", "storage", "pin_view", "cleanup_preview", "cleanup_commit", "compare", "search_limits", "integration", "install_hook_preview", "install_hook", "duplicates", "duplicates_dismiss", "duplicates_restore", "docs", "layers", "affected_tests", "pr_impact", "setup_audit", "redirect_proposals", "redirect_decide", "junk"]},
|
|
@@ -2191,9 +2191,9 @@ def _audited_roots():
|
|
|
2191
2191
|
|
|
2192
2192
|
def _junk_action(arguments):
|
|
2193
2193
|
"""`junk` lists the registered projects that were never projects (a personal or system folder, a subfolder of
|
|
2194
|
-
a repository or of another project, a folder of projects); with confirm=true it
|
|
2195
|
-
index files, never the project's own files, and returns the path of the
|
|
2196
|
-
kept and listed."""
|
|
2194
|
+
a repository or of another project, a folder of projects) or whose folder no longer exists; with confirm=true it
|
|
2195
|
+
unregisters them and deletes their index files, never the project's own files, and returns the path of the
|
|
2196
|
+
manifest, which records what re-registering each needs. A project with a running job is kept and listed."""
|
|
2197
2197
|
entries = disk_cleanup.junk_projects()
|
|
2198
2198
|
if arguments.get("confirm") is not True:
|
|
2199
2199
|
return {"junk": entries, "detail": "Nothing was removed. Pass confirm=true to unregister these and delete "
|
package/version.py
CHANGED