@allansantos-dev/smart-tool 0.9.53 → 0.9.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +2 -2
- package/disk_cleanup.py +119 -0
- package/index_scope.py +25 -4
- package/indexer.py +108 -31
- package/install.py +19 -0
- package/package.json +1 -1
- package/project_store.py +40 -0
- package/smart_tool_daemon.py +7 -7
- package/version.py +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.54 - beta
|
|
4
|
+
|
|
5
|
+
- A folder whose subfolders are projects is not indexed as one project. When it has no `.git` of its own and two or
|
|
6
|
+
more subfolders are registered projects or Git repositories, `smart_search`, `smart_scout` and `project_manage
|
|
7
|
+
action=register` fail naming those subfolders and asking for one of them as `project_root`; such a folder already
|
|
8
|
+
registered is unregistered and its index removed when the daemon starts. A project registered inside another is left
|
|
9
|
+
out of the outer one's scope, so the same files are no longer indexed twice; at its next update the outer index drops
|
|
10
|
+
their chunks from the current Git view and from the other views kept for that folder, compacting each view left
|
|
11
|
+
mostly free (`tests/test_disk_waste.py`).
|
|
12
|
+
- A folder of the indexing scope that was deleted no longer fails every indexing attempt of the project: it is left out
|
|
13
|
+
and the rest of the scope is indexed. A folder that cannot be listed fails naming it and the reason
|
|
14
|
+
(`tests/test_disk_waste.py`).
|
|
15
|
+
- A file whose date or size does not fit SQLite's 64-bit integer (a date past the year 2262) no longer fails every
|
|
16
|
+
indexing attempt with `OverflowError`: it is skipped by name with a `daemon.log` line saying how to index it, and the
|
|
17
|
+
rest of the folder is indexed. An index left with the journal of a write the daemon did not finish is rolled back on
|
|
18
|
+
the next read, with a log line, instead of failing as read-only on every attempt (`tests/test_disk_waste.py`).
|
|
19
|
+
- Disk left behind. A replaced index generation is no longer kept as `.previous.sqlite3` (nothing read it). An index
|
|
20
|
+
that lost at least 8 MB and a quarter of its pages to removed files is compacted (FTS optimize and `VACUUM`) after the
|
|
21
|
+
update. When the daemon starts it removes write temporaries orphaned by a crash, generations under construction
|
|
22
|
+
untouched for 7 days, the old `.previous` copies, the index files of folders that are not registered and saved graphs
|
|
23
|
+
of views that no longer exist; a registered folder it cannot list is logged and the rest of the cleanup goes on. The
|
|
24
|
+
installer removes `SmartTool.bak-*` and `.smart-tool-backup-*` copies next to the installation after the new version
|
|
25
|
+
is installed. `project_manage action=remove` reports the files and bytes it removed
|
|
26
|
+
(`tests/test_disk_waste.py`, `tests/test_graph_cache.py`).
|
|
27
|
+
|
|
3
28
|
## 0.9.53 - beta
|
|
4
29
|
|
|
5
30
|
- An update started in the first minutes after a release finds it. The update screen and the manual command in the
|
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@ registered as the `playwright` server where this machine lets a browser be drive
|
|
|
17
17
|
An optional hook routes the agent's native Grep/Glob/Read/Bash/WebSearch/WebFetch calls to these tools when they
|
|
18
18
|
are cheaper, or just tells the agent what Smart Tool would do.
|
|
19
19
|
|
|
20
|
-
**Status:** 0.9.
|
|
20
|
+
**Status:** 0.9.54 beta. Windows only. What changed in each version: [CHANGELOG.md](CHANGELOG.md).
|
|
21
21
|
|
|
22
22
|
## How it works
|
|
23
23
|
|
|
@@ -83,7 +83,7 @@ gateway such as Ollama, nothing about your code leaves the machine.
|
|
|
83
83
|
## Install
|
|
84
84
|
|
|
85
85
|
```powershell
|
|
86
|
-
npx @allansantos-dev/smart-tool@0.9.
|
|
86
|
+
npx @allansantos-dev/smart-tool@0.9.54 install
|
|
87
87
|
```
|
|
88
88
|
|
|
89
89
|
Updates install themselves, on by default: the tray checks npm when it starts (at logon) and every 24 h, and when there
|
package/disk_cleanup.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Removes what Smart Tool left on disk that no registered project needs.
|
|
2
|
+
|
|
3
|
+
Runs once when the daemon starts, before the project monitor: the registry of projects (a folder of projects is
|
|
4
|
+
never one of them), write temporaries orphaned by a crash, generations under construction that stopped advancing,
|
|
5
|
+
the `.previous` copies older versions kept of a replaced generation, index files of projects no longer registered,
|
|
6
|
+
and saved code-graph analyses of index views that no longer exist. Source files are never touched."""
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
import atomic_io
|
|
12
|
+
import indexer
|
|
13
|
+
import project_identity
|
|
14
|
+
import project_store
|
|
15
|
+
|
|
16
|
+
# A generation under construction resumes where it stopped; one untouched this long belongs to a model or scope that
|
|
17
|
+
# is no longer used, and a new one reuses the cached vectors anyway.
|
|
18
|
+
STALE_BUILD_S = 7 * 86400
|
|
19
|
+
_STATE_FILE = re.compile(r"([0-9a-f]{16})(?:\.v-[0-9a-f]{16})?(?:\.previous|\.building-[0-9a-f]+|\.vectors)?"
|
|
20
|
+
r"(?:\.sqlite3(?:-journal|-wal|-shm)?|\.scope\.json)")
|
|
21
|
+
_BUILDING = re.compile(r"[0-9a-f]{16}(?:\.v-[0-9a-f]{16})?\.building-[0-9a-f]+\.sqlite3")
|
|
22
|
+
_PREVIOUS = re.compile(r"[0-9a-f]{16}(?:\.v-[0-9a-f]{16})?\.previous\.sqlite3")
|
|
23
|
+
_TEMPORARY = re.compile(r".+" + re.escape(atomic_io.TMP_SUFFIX) + r"[0-9a-f]{32}")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _remove(path, report):
|
|
27
|
+
"""Deletes one state file and adds its name and size to report."""
|
|
28
|
+
size = os.path.getsize(path)
|
|
29
|
+
os.remove(project_identity.state_path(path, indexer.INDEX_DIR))
|
|
30
|
+
report["files"].append(os.path.basename(path))
|
|
31
|
+
report["bytes"] += size
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def remove_project_files(root, report=None):
|
|
35
|
+
"""Deletes every index, vector cache, scope and saved graph of the project at root (all Git views, the current and
|
|
36
|
+
legacy ids); returns the report {files, bytes}. Raises RuntimeError when another process holds one open."""
|
|
37
|
+
report = report if report is not None else {"files": [], "bytes": 0}
|
|
38
|
+
keys = {project_identity.project_id(root), *project_identity.legacy_ids(root)}
|
|
39
|
+
for name in sorted(os.listdir(indexer.INDEX_DIR)):
|
|
40
|
+
match = _STATE_FILE.fullmatch(name)
|
|
41
|
+
if not match or match.group(1) not in keys:
|
|
42
|
+
continue
|
|
43
|
+
path = os.path.join(indexer.INDEX_DIR, name)
|
|
44
|
+
try:
|
|
45
|
+
_remove(path, report)
|
|
46
|
+
except PermissionError as exc:
|
|
47
|
+
raise RuntimeError(f"{name} is open in another process. Close the external reader and try again.") from exc
|
|
48
|
+
graph = indexer.graph_cache_path(path)
|
|
49
|
+
if name.endswith(".sqlite3") and os.path.isfile(graph):
|
|
50
|
+
report["bytes"] += os.path.getsize(graph)
|
|
51
|
+
os.remove(graph)
|
|
52
|
+
indexer.invalidate_cache(root)
|
|
53
|
+
return report
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def sweep(extra_roots, log=None):
|
|
57
|
+
"""Removes the waste described in the module docstring and returns {containers, files, bytes, errors}.
|
|
58
|
+
|
|
59
|
+
extra_roots are folders that own a vector cache without being projects (the web_fetch cache). A file that
|
|
60
|
+
cannot be removed, or a registered folder that cannot be listed, is reported in errors and logged, and the sweep
|
|
61
|
+
goes on with the rest: the daemon starts its project monitor right after, so the sweep must not stop it."""
|
|
62
|
+
log = log or (lambda message: None)
|
|
63
|
+
report = {"containers": [], "files": [], "bytes": 0, "errors": []}
|
|
64
|
+
for project in project_store.all_projects():
|
|
65
|
+
if not os.path.isdir(project["root"]):
|
|
66
|
+
continue
|
|
67
|
+
try:
|
|
68
|
+
refused = project_store.container_error(project["root"])
|
|
69
|
+
except OSError as exc:
|
|
70
|
+
report["errors"].append(f"{project['root']}: {exc}")
|
|
71
|
+
log(f"disk cleanup: {project['root']} not checked as a folder of projects: {exc}")
|
|
72
|
+
continue
|
|
73
|
+
if refused:
|
|
74
|
+
try:
|
|
75
|
+
remove_project_files(project["root"], report)
|
|
76
|
+
except RuntimeError as exc:
|
|
77
|
+
report["errors"].append(str(exc))
|
|
78
|
+
log(f"disk cleanup: {project['root']} kept: {exc}")
|
|
79
|
+
continue
|
|
80
|
+
project_store.forget(project["id"])
|
|
81
|
+
report["containers"].append(project["root"])
|
|
82
|
+
log(f"disk cleanup: unregistered {refused}")
|
|
83
|
+
owners = [p["root"] for p in project_store.all_projects()] + list(extra_roots)
|
|
84
|
+
keep = {key for root in owners for key in (project_identity.project_id(root), *project_identity.legacy_ids(root))}
|
|
85
|
+
now = time.time()
|
|
86
|
+
for name in sorted(os.listdir(indexer.INDEX_DIR)):
|
|
87
|
+
path = os.path.join(indexer.INDEX_DIR, name)
|
|
88
|
+
match = _STATE_FILE.fullmatch(name)
|
|
89
|
+
try:
|
|
90
|
+
if not os.path.isfile(path):
|
|
91
|
+
continue
|
|
92
|
+
age = now - os.path.getmtime(path)
|
|
93
|
+
if _TEMPORARY.fullmatch(name):
|
|
94
|
+
waste = age > atomic_io.ORPHAN_AGE_S
|
|
95
|
+
elif match:
|
|
96
|
+
waste = (match.group(1) not in keep or bool(_PREVIOUS.fullmatch(name)) or
|
|
97
|
+
bool(_BUILDING.fullmatch(name)) and age > STALE_BUILD_S)
|
|
98
|
+
else:
|
|
99
|
+
waste = False
|
|
100
|
+
if waste:
|
|
101
|
+
_remove(path, report)
|
|
102
|
+
except OSError as exc:
|
|
103
|
+
report["errors"].append(f"{name}: {exc}")
|
|
104
|
+
graphs = os.path.join(indexer.INDEX_DIR, "graph-cache")
|
|
105
|
+
if os.path.isdir(graphs):
|
|
106
|
+
for name in sorted(os.listdir(graphs)):
|
|
107
|
+
view = os.path.join(indexer.INDEX_DIR, name.removesuffix(".json.gz") + ".sqlite3")
|
|
108
|
+
if name.endswith(".json.gz") and not os.path.isfile(view):
|
|
109
|
+
path = os.path.join(graphs, name)
|
|
110
|
+
try:
|
|
111
|
+
report["bytes"] += os.path.getsize(path)
|
|
112
|
+
os.remove(path)
|
|
113
|
+
report["files"].append("graph-cache/" + name)
|
|
114
|
+
except OSError as exc:
|
|
115
|
+
report["errors"].append(f"graph-cache/{name}: {exc}")
|
|
116
|
+
if report["files"] or report["errors"]:
|
|
117
|
+
log(f"disk cleanup: removed {len(report['files'])} file(s), {report['bytes']} bytes; "
|
|
118
|
+
f"{len(report['errors'])} error(s): {'; '.join(report['errors'][:5])}")
|
|
119
|
+
return report
|
package/index_scope.py
CHANGED
|
@@ -16,6 +16,7 @@ import time
|
|
|
16
16
|
|
|
17
17
|
import atomic_io
|
|
18
18
|
import project_identity
|
|
19
|
+
import project_store
|
|
19
20
|
import index_views
|
|
20
21
|
|
|
21
22
|
import model_client
|
|
@@ -532,6 +533,18 @@ def _exclude_matcher(exclude):
|
|
|
532
533
|
return matches
|
|
533
534
|
|
|
534
535
|
|
|
536
|
+
def _nested_matcher(root):
|
|
537
|
+
"""Predicate "is this inside a registered project nested under root?", by exact path prefix: that project is
|
|
538
|
+
indexed on its own, so the folder holding it leaves it out. A folder elsewhere with the same name stays in."""
|
|
539
|
+
nested = tuple(path.lower() for path in project_store.nested_projects(root)) if root else ()
|
|
540
|
+
|
|
541
|
+
def inside(rel_path):
|
|
542
|
+
target = rel_path.replace("\\", "/").strip("/").lower()
|
|
543
|
+
return any(target == path or target.startswith(path + "/") for path in nested)
|
|
544
|
+
|
|
545
|
+
return inside
|
|
546
|
+
|
|
547
|
+
|
|
535
548
|
def scope_filter(scope, root=None):
|
|
536
549
|
"""Predicate `keep(rel_path)`: whether resolve_included_files would list rel_path (a file) or walk it (a folder) —
|
|
537
550
|
under an include base, outside the exclusions, outside hidden folders. Built once so a burst of file events does not
|
|
@@ -540,11 +553,12 @@ def scope_filter(scope, root=None):
|
|
|
540
553
|
matcher = _exclude_matcher(set(scope.get("exclude") or []) | set(scope.get("user_exclude") or []) |
|
|
541
554
|
set(mechanical_excludes(root) if root else ()) | _ALWAYS_EXCLUDE | _ALWAYS_EXCLUDE_FILES)
|
|
542
555
|
whole = "" in bases
|
|
556
|
+
nested = _nested_matcher(root)
|
|
543
557
|
|
|
544
558
|
def keep(rel_path):
|
|
545
559
|
rel_path = rel_path.replace("\\", "/")
|
|
546
560
|
parts = rel_path.split("/")
|
|
547
|
-
if matcher(rel_path) or is_hidden(rel_path):
|
|
561
|
+
if matcher(rel_path) or is_hidden(rel_path) or nested(rel_path):
|
|
548
562
|
return False
|
|
549
563
|
if whole or len(parts) == 1:
|
|
550
564
|
return True
|
|
@@ -563,11 +577,16 @@ def resolve_included_files(root, scope, cancel_check=None):
|
|
|
563
577
|
if not valid_scope(scope):
|
|
564
578
|
raise ValueError("Invalid scope; request a new project analysis.")
|
|
565
579
|
include, _dropped = normalize_include(scope.get("include"))
|
|
566
|
-
|
|
567
|
-
|
|
580
|
+
excluded = _exclude_matcher(set(scope.get("exclude") or []) | set(scope.get("user_exclude") or []) |
|
|
581
|
+
set(mechanical_excludes(root)) | _ALWAYS_EXCLUDE | _ALWAYS_EXCLUDE_FILES)
|
|
582
|
+
nested = _nested_matcher(root)
|
|
583
|
+
|
|
584
|
+
def matcher(rel_path):
|
|
585
|
+
return excluded(rel_path) or nested(rel_path)
|
|
586
|
+
|
|
568
587
|
files = []
|
|
569
588
|
def walk_error(error):
|
|
570
|
-
raise OSError("Could not list
|
|
589
|
+
raise OSError(f"Could not list {error.filename} ({error.strerror}); try again when it is accessible.") from error
|
|
571
590
|
if "" not in include:
|
|
572
591
|
files.extend(_root_level_files(root, matcher))
|
|
573
592
|
for base in include:
|
|
@@ -577,6 +596,8 @@ def resolve_included_files(root, scope, cancel_check=None):
|
|
|
577
596
|
# projeto e mandar conteúdo de fora pro endpoint de embeddings.
|
|
578
597
|
if not _within_root(root, base_full):
|
|
579
598
|
continue
|
|
599
|
+
if not os.path.lexists(base_full):
|
|
600
|
+
continue
|
|
580
601
|
if os.path.isfile(base_full):
|
|
581
602
|
rel = os.path.relpath(base_full, root).replace(os.sep, "/")
|
|
582
603
|
if not matcher(rel) and not is_hidden(rel):
|
package/indexer.py
CHANGED
|
@@ -14,6 +14,7 @@ import os
|
|
|
14
14
|
import paths
|
|
15
15
|
import re
|
|
16
16
|
import sqlite3
|
|
17
|
+
import sys
|
|
17
18
|
import threading
|
|
18
19
|
import time
|
|
19
20
|
import uuid
|
|
@@ -21,6 +22,8 @@ from pathlib import Path
|
|
|
21
22
|
|
|
22
23
|
import atomic_io
|
|
23
24
|
import project_identity
|
|
25
|
+
import index_scope
|
|
26
|
+
import project_store
|
|
24
27
|
import index_views
|
|
25
28
|
import embedding_cache
|
|
26
29
|
import document_text
|
|
@@ -87,6 +90,23 @@ SKIP_GRAPHIC = "large svg graphic"
|
|
|
87
90
|
SKIP_BINARY = "binary"
|
|
88
91
|
SKIP_MINIFIED = "minified"
|
|
89
92
|
SKIP_UNREADABLE = "unreadable"
|
|
93
|
+
SKIP_OUT_OF_RANGE = "date or size outside what the index stores"
|
|
94
|
+
# SQLite INTEGER is a signed 64-bit value: a file dated past 2262 (or before 1677) has an st_mtime_ns beyond it.
|
|
95
|
+
_SQLITE_INTEGER = range(-2**63, 2**63)
|
|
96
|
+
# A removal that leaves this much free space in an index file is followed by VACUUM: with secure_delete the freed
|
|
97
|
+
# pages are zeroed but the file keeps its size until compacted.
|
|
98
|
+
COMPACT_MIN_FREE_BYTES = 8 * 1024 * 1024
|
|
99
|
+
COMPACT_MIN_FREE_SHARE = 0.25
|
|
100
|
+
# Where notices about a skipped file or a recovered index go; the daemon points this at daemon.log.
|
|
101
|
+
NOTICE = None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _notice(message):
|
|
105
|
+
"""Sends one diagnostic line to NOTICE (daemon.log when the daemon runs), or to stderr without a daemon."""
|
|
106
|
+
if NOTICE is None:
|
|
107
|
+
print(message, file=sys.stderr)
|
|
108
|
+
else:
|
|
109
|
+
NOTICE(message)
|
|
90
110
|
|
|
91
111
|
|
|
92
112
|
LEXICAL_MODEL = "lexical"
|
|
@@ -207,20 +227,48 @@ def existing_db_path(root):
|
|
|
207
227
|
|
|
208
228
|
|
|
209
229
|
def _readonly(path):
|
|
230
|
+
"""Read-only connection to an index. A rollback journal left by a crashed writer (a hot journal) cannot be rolled
|
|
231
|
+
back by a read-only connection, which then fails every read with "attempt to write a readonly database": the
|
|
232
|
+
journal is rolled back once through a read-write connection, logged, and the read-only open is retried."""
|
|
233
|
+
conn = sqlite3.connect(Path(path).absolute().as_uri() + "?mode=ro", uri=True, timeout=10)
|
|
234
|
+
try:
|
|
235
|
+
conn.execute("PRAGMA schema_version").fetchone()
|
|
236
|
+
return conn
|
|
237
|
+
except sqlite3.OperationalError as exc:
|
|
238
|
+
conn.close()
|
|
239
|
+
journal = str(path) + "-journal"
|
|
240
|
+
if "readonly" not in str(exc) or not os.path.isfile(journal):
|
|
241
|
+
raise
|
|
242
|
+
size = os.path.getsize(journal)
|
|
243
|
+
writer = sqlite3.connect(path, timeout=10)
|
|
244
|
+
try:
|
|
245
|
+
writer.execute("PRAGMA schema_version").fetchone()
|
|
246
|
+
finally:
|
|
247
|
+
writer.close()
|
|
248
|
+
_notice(f"index {os.path.basename(str(path))}: rolled back the {size}-byte journal a crashed write left behind")
|
|
210
249
|
return sqlite3.connect(Path(path).absolute().as_uri() + "?mode=ro", uri=True, timeout=10)
|
|
211
250
|
|
|
212
251
|
|
|
213
252
|
def _backup(source, target):
|
|
253
|
+
"""Consistent copy of the index at source into target, through a temporary file removed if the copy fails."""
|
|
214
254
|
target = project_identity.state_path(target, INDEX_DIR)
|
|
215
255
|
temp = atomic_io.tmp_path_for(target)
|
|
216
|
-
src, dst = _readonly(source), sqlite3.connect(temp)
|
|
217
256
|
try:
|
|
218
|
-
src
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
257
|
+
src = _readonly(source)
|
|
258
|
+
try:
|
|
259
|
+
dst = sqlite3.connect(temp)
|
|
260
|
+
try:
|
|
261
|
+
src.backup(dst)
|
|
262
|
+
dst.commit()
|
|
263
|
+
finally:
|
|
264
|
+
dst.close()
|
|
265
|
+
finally:
|
|
266
|
+
src.close()
|
|
267
|
+
atomic_io.atomic_replace(temp, target)
|
|
268
|
+
except BaseException:
|
|
269
|
+
if os.path.exists(temp):
|
|
270
|
+
os.remove(temp)
|
|
271
|
+
raise
|
|
224
272
|
|
|
225
273
|
|
|
226
274
|
def migrate_legacy(root):
|
|
@@ -513,6 +561,8 @@ def _stat_candidate(root, rel):
|
|
|
513
561
|
return None, SKIP_TOO_LARGE
|
|
514
562
|
if name.endswith(".svg") and stat.st_size > MAX_SVG_INDEXABLE_BYTES:
|
|
515
563
|
return None, SKIP_GRAPHIC
|
|
564
|
+
if stat.st_mtime_ns not in _SQLITE_INTEGER or stat.st_size not in _SQLITE_INTEGER:
|
|
565
|
+
return None, SKIP_OUT_OF_RANGE
|
|
516
566
|
return (stat.st_mtime_ns, stat.st_size), None
|
|
517
567
|
|
|
518
568
|
|
|
@@ -620,6 +670,10 @@ def _reindex(root, included_paths, embed_batch, chunk_lines=CHUNK_LINES, on_prog
|
|
|
620
670
|
stat, reason = _stat_candidate(root, rel)
|
|
621
671
|
if stat is None:
|
|
622
672
|
stats["skipped"][reason] = stats["skipped"].get(reason, 0) + 1
|
|
673
|
+
if reason == SKIP_OUT_OF_RANGE:
|
|
674
|
+
stats.setdefault("out_of_range", []).append(rel)
|
|
675
|
+
_notice(f"index {root}: skipped {rel}: its date or size is outside the 64-bit range SQLite stores; "
|
|
676
|
+
"fix the file's date to index it")
|
|
623
677
|
continue
|
|
624
678
|
observed[rel] = stat
|
|
625
679
|
force_file = any(rel.casefold() == p or rel.casefold().startswith(p + "/") for p in forced)
|
|
@@ -724,6 +778,8 @@ def _reindex(root, included_paths, embed_batch, chunk_lines=CHUNK_LINES, on_prog
|
|
|
724
778
|
if building or stats["added"] or stats["changed"] or stats["removed"]:
|
|
725
779
|
_set_metadata(conn, {"updated_at": time.time()})
|
|
726
780
|
conn.commit()
|
|
781
|
+
if stats["removed"] and not building:
|
|
782
|
+
_compact(conn)
|
|
727
783
|
stats.update(generation=generation, dimensions=dimensions,
|
|
728
784
|
embedded_chunks=cache.misses, reused_chunks=cache.hits,
|
|
729
785
|
embedded_characters=cache.characters, view=index_views.public(view),
|
|
@@ -738,16 +794,59 @@ def _reindex(root, included_paths, embed_batch, chunk_lines=CHUNK_LINES, on_prog
|
|
|
738
794
|
cache.close()
|
|
739
795
|
check()
|
|
740
796
|
if building and not stats["failed"]:
|
|
741
|
-
if os.path.isfile(target):
|
|
742
|
-
_backup(target, target[:-8] + ".previous.sqlite3")
|
|
743
797
|
atomic_io.atomic_replace(work_path, project_identity.state_path(target, INDEX_DIR))
|
|
744
798
|
stats["ready"] = not bool(stats["failed"])
|
|
745
799
|
stats["path"] = target if stats["ready"] else work_path
|
|
746
800
|
if building or total or stats["removed"] or full_check:
|
|
747
801
|
invalidate_cache(root)
|
|
802
|
+
_trim_other_views(root, target)
|
|
748
803
|
return stats
|
|
749
804
|
|
|
750
805
|
|
|
806
|
+
def _trim_other_views(root, target):
|
|
807
|
+
"""Drops, from the project's other Git views, the files of projects registered inside root (their own index holds
|
|
808
|
+
them; target already left them out through its scope), then compacts each view left mostly free. A view another
|
|
809
|
+
process holds is logged and trimmed at the next update. Runs on every update, changed files or not, so views written
|
|
810
|
+
before the inner project was registered are trimmed too; reading the paths is an index scan per view."""
|
|
811
|
+
if not project_store.nested_projects(root):
|
|
812
|
+
return
|
|
813
|
+
nested = index_scope._nested_matcher(root)
|
|
814
|
+
key = project_identity.project_id(root)
|
|
815
|
+
for name in sorted(os.listdir(INDEX_DIR)):
|
|
816
|
+
path = os.path.join(INDEX_DIR, name)
|
|
817
|
+
if (not name.startswith(key + ".v-") or not name.endswith(".sqlite3") or ".building-" in name
|
|
818
|
+
or ".previous." in name or os.path.normcase(path) == os.path.normcase(target)):
|
|
819
|
+
continue
|
|
820
|
+
conn = sqlite3.connect(path, timeout=10)
|
|
821
|
+
try:
|
|
822
|
+
conn.execute("PRAGMA secure_delete=ON")
|
|
823
|
+
states = conn.execute("SELECT 1 FROM sqlite_master WHERE name='file_state'").fetchone()
|
|
824
|
+
for (rel,) in conn.execute("SELECT path FROM manifest UNION SELECT DISTINCT path FROM chunks").fetchall():
|
|
825
|
+
if nested(rel):
|
|
826
|
+
remove_file(conn, rel)
|
|
827
|
+
if states:
|
|
828
|
+
conn.execute("DELETE FROM file_state WHERE path = ?", (rel,))
|
|
829
|
+
conn.commit()
|
|
830
|
+
_compact(conn)
|
|
831
|
+
except sqlite3.OperationalError as exc:
|
|
832
|
+
_notice(f"index {name}: other view not trimmed now ({exc}); it is trimmed at the next update")
|
|
833
|
+
finally:
|
|
834
|
+
conn.close()
|
|
835
|
+
|
|
836
|
+
|
|
837
|
+
def _compact(conn):
|
|
838
|
+
"""VACUUM when the removals left at least COMPACT_MIN_FREE_BYTES and COMPACT_MIN_FREE_SHARE of the file free."""
|
|
839
|
+
page = conn.execute("PRAGMA page_size").fetchone()[0]
|
|
840
|
+
free = conn.execute("PRAGMA freelist_count").fetchone()[0] * page
|
|
841
|
+
total = conn.execute("PRAGMA page_count").fetchone()[0] * page
|
|
842
|
+
if free >= COMPACT_MIN_FREE_BYTES and free >= COMPACT_MIN_FREE_SHARE * total:
|
|
843
|
+
if conn.execute("SELECT 1 FROM sqlite_master WHERE name='chunks_fts'").fetchone():
|
|
844
|
+
# FTS5 records a deletion as a tombstone; merging the segments drops the deleted text for good.
|
|
845
|
+
conn.execute("INSERT INTO chunks_fts(chunks_fts) VALUES('optimize')")
|
|
846
|
+
conn.commit()
|
|
847
|
+
conn.execute("VACUUM")
|
|
848
|
+
|
|
849
|
+
|
|
751
850
|
def _vector_rows(root, path):
|
|
752
851
|
stat = os.stat(path)
|
|
753
852
|
key = (project_identity.canonical_root(root), path, stat.st_mtime_ns, stat.st_size)
|
|
@@ -967,28 +1066,6 @@ def graph_cache_path(view_path):
|
|
|
967
1066
|
os.path.basename(view_path).removesuffix(".sqlite3") + ".json.gz")
|
|
968
1067
|
|
|
969
1068
|
|
|
970
|
-
def remove_index(root):
|
|
971
|
-
"""Remove somente arquivos de índice da raiz conhecida, nunca arquivos do projeto."""
|
|
972
|
-
import glob
|
|
973
|
-
removed = []
|
|
974
|
-
for key in set([project_identity.project_id(root), *project_identity.legacy_ids(root)]):
|
|
975
|
-
pattern = re.compile(re.escape(key) + r"(?:\.v-[a-f0-9]{16})?(?:\.previous|\.building-[a-f0-9]+|\.vectors)?\.sqlite3$")
|
|
976
|
-
for path in glob.glob(os.path.join(INDEX_DIR, key + "*.sqlite3")):
|
|
977
|
-
if pattern.fullmatch(os.path.basename(path)):
|
|
978
|
-
safe = project_identity.state_path(path, INDEX_DIR)
|
|
979
|
-
try:
|
|
980
|
-
os.remove(safe)
|
|
981
|
-
except PermissionError as exc:
|
|
982
|
-
raise RuntimeError("The index is open in another process. Close the external reader and try removing it again.") from exc
|
|
983
|
-
removed.append(safe)
|
|
984
|
-
try:
|
|
985
|
-
os.remove(graph_cache_path(safe))
|
|
986
|
-
except FileNotFoundError:
|
|
987
|
-
pass
|
|
988
|
-
invalidate_cache(root)
|
|
989
|
-
return removed
|
|
990
|
-
|
|
991
|
-
|
|
992
1069
|
def copy_project_indexes(root, destination):
|
|
993
1070
|
"""Copia todas as visões e cache ao reassociar uma raiz, preservando a origem."""
|
|
994
1071
|
import glob
|
package/install.py
CHANGED
|
@@ -382,9 +382,28 @@ def main():
|
|
|
382
382
|
mark_updating(False)
|
|
383
383
|
if not args.no_start:
|
|
384
384
|
enable_browser_control(target, python)
|
|
385
|
+
for folder in prune_backups(target):
|
|
386
|
+
print(f"Removed the old copy {folder}.")
|
|
385
387
|
print("Smart Tool ready." if not args.no_start else "Files installed; daemon not restarted (--no-start).")
|
|
386
388
|
|
|
387
389
|
|
|
390
|
+
def prune_backups(target):
|
|
391
|
+
"""After a successful install, deletes the copies of older versions beside target: `.smart-tool-backup-*` (kept by
|
|
392
|
+
an incomplete rollback) and `<target name>.bak-*` (copies made by hand before an update). Returns the removed
|
|
393
|
+
paths; a copy that cannot be removed is reported on stderr and kept."""
|
|
394
|
+
removed = []
|
|
395
|
+
for folder in sorted(target.parent.iterdir()):
|
|
396
|
+
if not folder.is_dir() or not (folder.name.startswith(".smart-tool-backup-") or
|
|
397
|
+
folder.name.startswith(target.name + ".bak-")):
|
|
398
|
+
continue
|
|
399
|
+
try:
|
|
400
|
+
shutil.rmtree(folder)
|
|
401
|
+
removed.append(str(folder))
|
|
402
|
+
except OSError as exc:
|
|
403
|
+
print(f"Could not remove the old copy {folder}: {exc}", file=sys.stderr)
|
|
404
|
+
return removed
|
|
405
|
+
|
|
406
|
+
|
|
388
407
|
def _update(args, target, files, existed, python, pythonw):
|
|
389
408
|
"""Replaces the installed files, dependencies and daemon, restoring the previous version on any failure."""
|
|
390
409
|
stop_processes(target)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.54",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Allan Santos",
|
package/project_store.py
CHANGED
|
@@ -38,10 +38,50 @@ def _db():
|
|
|
38
38
|
db.close()
|
|
39
39
|
|
|
40
40
|
|
|
41
|
+
CONTAINER_MIN_PROJECTS = 2
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def project_subfolders(root):
|
|
45
|
+
"""Names of the immediate subfolders of root that are projects: registered, or holding a `.git` folder or file."""
|
|
46
|
+
registered = {project_identity.canonical_root(p["root"]) for p in all_projects()}
|
|
47
|
+
names = []
|
|
48
|
+
with os.scandir(root) as entries:
|
|
49
|
+
for entry in entries:
|
|
50
|
+
if entry.name.startswith(".") or not entry.is_dir():
|
|
51
|
+
continue
|
|
52
|
+
if (project_identity.canonical_root(entry.path) in registered or
|
|
53
|
+
os.path.exists(os.path.join(entry.path, ".git"))):
|
|
54
|
+
names.append(entry.name)
|
|
55
|
+
return sorted(names)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def container_error(root):
|
|
59
|
+
"""Why root cannot be one project, or None: a folder without its own `.git` whose subfolders are at least
|
|
60
|
+
CONTAINER_MIN_PROJECTS projects holds projects (each indexed on its own); indexing it would store them twice."""
|
|
61
|
+
if os.path.exists(os.path.join(root, ".git")):
|
|
62
|
+
return None
|
|
63
|
+
names = project_subfolders(root)
|
|
64
|
+
if len(names) < CONTAINER_MIN_PROJECTS:
|
|
65
|
+
return None
|
|
66
|
+
shown = ", ".join(names[:8]) + (", ..." if len(names) > 8 else "")
|
|
67
|
+
return (f"{root} is a folder of {len(names)} projects ({shown}), not one project: "
|
|
68
|
+
"pass the folder of one of them as project_root.")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def nested_projects(root):
|
|
72
|
+
"""Paths, relative to root with forward slashes, of the registered projects strictly inside root."""
|
|
73
|
+
base = project_identity.canonical_root(root)
|
|
74
|
+
inner = (project_identity.canonical_root(p["root"]) for p in all_projects())
|
|
75
|
+
return sorted(os.path.relpath(path, base).replace(os.sep, "/") for path in inner if path.startswith(base + os.sep))
|
|
76
|
+
|
|
77
|
+
|
|
41
78
|
def register(root, watch=None):
|
|
42
79
|
display = project_identity.display_root(root)
|
|
43
80
|
if not os.path.isdir(display):
|
|
44
81
|
raise ValueError("The project folder does not exist or is not accessible.")
|
|
82
|
+
refused = container_error(display)
|
|
83
|
+
if refused:
|
|
84
|
+
raise ValueError(refused)
|
|
45
85
|
key = project_identity.project_id(display)
|
|
46
86
|
with _LOCK, _db() as db:
|
|
47
87
|
row = db.execute("SELECT data FROM projects WHERE id=?", (key,)).fetchone()
|
package/smart_tool_daemon.py
CHANGED
|
@@ -42,6 +42,7 @@ import install_runtime
|
|
|
42
42
|
import integration_metrics
|
|
43
43
|
import node_locator
|
|
44
44
|
import config
|
|
45
|
+
import disk_cleanup
|
|
45
46
|
import freshness
|
|
46
47
|
import model_client
|
|
47
48
|
import index_scope
|
|
@@ -845,6 +846,8 @@ def _log(message):
|
|
|
845
846
|
pass
|
|
846
847
|
|
|
847
848
|
|
|
849
|
+
indexer.NOTICE = _log
|
|
850
|
+
|
|
848
851
|
_WEB_WARNINGS = contextvars.ContextVar("web_warnings", default=None)
|
|
849
852
|
_WEB_TRACE = contextvars.ContextVar("web_trace", default=None)
|
|
850
853
|
|
|
@@ -2318,14 +2321,10 @@ def _project_action_locked(arguments):
|
|
|
2318
2321
|
if active:
|
|
2319
2322
|
raise ValueError("Cancel the jobs and wait for them to stop before removing the index.")
|
|
2320
2323
|
project_store.update(key, watch=False, paused=True, enabled=False)
|
|
2321
|
-
removed =
|
|
2322
|
-
import glob
|
|
2323
|
-
for scope_id in set([key, *project_identity.legacy_ids(project["root"])]):
|
|
2324
|
-
for path in glob.glob(os.path.join(index_scope.SCOPE_DIR, scope_id + '*.scope.json')):
|
|
2325
|
-
if re.fullmatch(re.escape(scope_id) + r'(?:\.v-[a-f0-9]{16})?\.scope\.json', os.path.basename(path)):
|
|
2326
|
-
os.remove(project_identity.state_path(path, index_scope.SCOPE_DIR))
|
|
2324
|
+
removed = disk_cleanup.remove_project_files(project["root"])
|
|
2327
2325
|
project_store.forget(key)
|
|
2328
|
-
return {"removed_index_files": len(removed), "
|
|
2326
|
+
return {"removed_index_files": len(removed["files"]), "removed_bytes": removed["bytes"],
|
|
2327
|
+
"source_files_preserved": True}
|
|
2329
2328
|
elif action == "relocate":
|
|
2330
2329
|
if active:
|
|
2331
2330
|
raise ValueError("Wait for the jobs to stop before relocating the folder.")
|
|
@@ -4569,6 +4568,7 @@ def main():
|
|
|
4569
4568
|
threading.Thread(target=_heal_browser_control, daemon=True).start()
|
|
4570
4569
|
try:
|
|
4571
4570
|
recovered = project_store.recover_interrupted()
|
|
4571
|
+
disk_cleanup.sweep([WEB_FETCH_VECTOR_ROOT], _log)
|
|
4572
4572
|
project_store.import_known_legacy(indexer.INDEX_DIR)
|
|
4573
4573
|
_PROJECT_MONITOR = project_monitor.ProjectMonitor(_enqueue_monitored_project)
|
|
4574
4574
|
_PROJECT_MONITOR.start()
|
package/version.py
CHANGED