@allansantos-dev/smart-tool 0.9.25 → 0.9.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +1 -1
- package/index_scope.py +54 -21
- package/index_views.py +5 -0
- package/package.json +1 -1
- package/project_monitor.py +45 -11
- package/project_store.py +65 -11
- package/smart_tool_daemon.py +2 -1
- package/version.py +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,38 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.26 - beta
|
|
4
|
+
|
|
5
|
+
- The daemon no longer re-reads and re-hashes every watched folder in a loop. Measured on the owner's machine: one
|
|
6
|
+
daemon process read 181 GB and wrote 29.7 GB in 15 h (3.6 MB/s read, 2.2 MB/s write while idle), which kept the disk
|
|
7
|
+
busy and made Defender scan the files again and again. Causes, all fixed:
|
|
8
|
+
- A file-event buffer overflow, or more than 1 000 changed paths, raised a full check that opened and hashed every
|
|
9
|
+
file of the scope; the flag was cleared only when no event arrived during the job, so under constant events (the
|
|
10
|
+
`~/.claude` session logs, a state folder) every job hashed everything again, and the stuck scope flag also asked the
|
|
11
|
+
scope model to decide the scope again on every job. An overflow now asks for a stat comparison (mtime and size) of
|
|
12
|
+
the scope, and a finished job clears the full-check and scope flags raised before it started.
|
|
13
|
+
- An event under a folder the scope leaves out (`plugins/cache`, `projects/*.jsonl` of `~/.claude`) marked the project
|
|
14
|
+
dirty and queued an index job. Events are now filtered by the saved scope, which is reloaded only when its file
|
|
15
|
+
changes.
|
|
16
|
+
- A folder growing inside an excluded one counted as a structure change and called the scope model again. The
|
|
17
|
+
structure kept in the scope and the one compared against it skip excluded folders.
|
|
18
|
+
- The runtime folders of a `.claude` home (`plugins`, `projects`, `sessions`, `shell-snapshots`, `file-history`,
|
|
19
|
+
`cache`, `debug`, `telemetry`, ...) are excluded mechanically, whatever the scope says.
|
|
20
|
+
- The monitor decrypted up to 50 job payloads per watched folder every second just to learn whether a job was
|
|
21
|
+
pending; it now asks the `status` column once per tick. A Git folder is identified again (four git processes)
|
|
22
|
+
only when `HEAD`, the index or the refs moved, not every 5 s. With a live watcher the stat reconciliation runs
|
|
23
|
+
every 15 min instead of 3; the tick sleeps 5 s instead of 1 s while nothing is pending.
|
|
24
|
+
- A burst of events cost two reads and two writes of the project registry per batch; it is one of each, and the
|
|
25
|
+
registry schema is created once per process instead of per connection.
|
|
26
|
+
- Every save of a job (several per index job) ordered the whole `jobs` table, payloads included, to prune the history:
|
|
27
|
+
1.7 MB read per save. The pruning now takes its threshold from an `updated_at` index (50 saves: 85.7 MB read before,
|
|
28
|
+
5.0 MB after, `tests/save_job_io_probe.py`).
|
|
29
|
+
- The pending-job query of the monitor tick scanned the whole `jobs` table (2.7 MB) every second; it now reads the
|
|
30
|
+
covering index `jobs_status`.
|
|
31
|
+
- A folder that changes all the time (the orchestrator's state folder queued an index job every 8 s) is indexed by the
|
|
32
|
+
monitor at most once a minute; the changes accumulate and one job covers them.
|
|
33
|
+
- Measured with `tests/monitor_io_probe.py` on a copy of the real registry (14 watched folders, 100 event batches
|
|
34
|
+
under excluded folders with a buffer overflow every 10th): 149 MB read and 45.7 MB written before, 28 MB read and
|
|
35
|
+
5.4 MB written after; folders left with a pending full check 14 before, 0 after.
|
|
3
36
|
## 0.9.25 - beta
|
|
4
37
|
|
|
5
38
|
- A redirected search made while the first index of a folder is still being built no longer waits in the folder's queue
|
package/README.md
CHANGED
|
@@ -15,7 +15,7 @@ registered as the `playwright` server where this machine lets a browser be drive
|
|
|
15
15
|
An optional hook routes the agent's native Grep/Glob/Read/Bash/WebSearch/WebFetch calls to these tools when they
|
|
16
16
|
are cheaper, or just tells the agent what Smart Tool would do.
|
|
17
17
|
|
|
18
|
-
**Status:** 0.9.
|
|
18
|
+
**Status:** 0.9.26 beta. Windows only. What changed in each version: [CHANGELOG.md](CHANGELOG.md).
|
|
19
19
|
|
|
20
20
|
## How it works
|
|
21
21
|
|
package/index_scope.py
CHANGED
|
@@ -41,6 +41,19 @@ _ALWAYS_EXCLUDE_FILES = {
|
|
|
41
41
|
|
|
42
42
|
_ROOT_DOC_NAMES = ("AGENTS.md", "CLAUDE.md", "README.md", "README")
|
|
43
43
|
|
|
44
|
+
# Runtime folders a tool keeps inside its own config home (conversation logs, plugin caches, snapshots): never project
|
|
45
|
+
# source, written all day by the tool itself. Keyed by the root's folder name; excluded mechanically, whatever the
|
|
46
|
+
# scope decision says, so a registered `~/.claude` never indexes or re-scans them.
|
|
47
|
+
_TOOL_STATE_DIRS = {
|
|
48
|
+
".claude": ("backups", "cache", "daemon", "debug", "downloads", "file-history", "ide", "paste-cache", "plugins",
|
|
49
|
+
"projects", "session-env", "sessions", "shell-snapshots", "state", "statsig", "telemetry", "todos"),
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def tool_state_excludes(root):
|
|
54
|
+
"""Names of the tool-runtime folders under `root` (empty unless `root` is a known tool config home)."""
|
|
55
|
+
return _TOOL_STATE_DIRS.get(os.path.basename(os.path.normpath(root)).casefold(), ())
|
|
56
|
+
|
|
44
57
|
|
|
45
58
|
def always_excluded(name):
|
|
46
59
|
"""Directory name in _ALWAYS_EXCLUDE, entries may be globs (e.g. *.egg-info)."""
|
|
@@ -173,11 +186,14 @@ def save_scope(root, scope):
|
|
|
173
186
|
{**scope, "root": project_identity.canonical_root(root)}, indent=2, ensure_ascii=False))
|
|
174
187
|
|
|
175
188
|
|
|
176
|
-
def _structure(root, max_dirs=1500):
|
|
189
|
+
def _structure(root, max_dirs=1500, keep=None):
|
|
190
|
+
"""Relative folders of `root` (at most `max_dirs`). `keep(rel)` prunes the walk: a folder it rejects is neither
|
|
191
|
+
listed nor entered."""
|
|
177
192
|
found = []
|
|
178
193
|
for current, dirs, _files in os.walk(root):
|
|
179
194
|
dirs[:] = sorted(d for d in dirs if not d.startswith(".") and not always_excluded(d)
|
|
180
|
-
and _within_root(root, os.path.join(current, d))
|
|
195
|
+
and _within_root(root, os.path.join(current, d))
|
|
196
|
+
and (keep is None or keep(os.path.relpath(os.path.join(current, d), root).replace(os.sep, "/"))))
|
|
181
197
|
for name in dirs:
|
|
182
198
|
found.append(os.path.relpath(os.path.join(current, name), root).replace(os.sep, "/"))
|
|
183
199
|
if len(found) >= max_dirs:
|
|
@@ -200,8 +216,10 @@ def needs_rescan(root, cached_scope):
|
|
|
200
216
|
return True
|
|
201
217
|
known = set(cached_scope.get("top_level_dirs", []))
|
|
202
218
|
current = set(_top_level_dirs(root))
|
|
219
|
+
# A folder the scope excludes (a cache that grows all day) is not a structure change: the walk skips it.
|
|
203
220
|
return (not current.issubset(known) or
|
|
204
|
-
not set(_structure(root
|
|
221
|
+
not set(_structure(root, keep=scope_filter(cached_scope, root))).issubset(
|
|
222
|
+
set(cached_scope.get("structure", []))))
|
|
205
223
|
|
|
206
224
|
|
|
207
225
|
def _read_root_docs(root, max_chars=4000):
|
|
@@ -315,7 +333,7 @@ def decide_scope(root, router_model, force=False, deadline=None, cancel_check=No
|
|
|
315
333
|
return cached
|
|
316
334
|
|
|
317
335
|
top_level_dirs = _top_level_dirs(root)
|
|
318
|
-
ambiguous_dirs = [d for d in top_level_dirs if not always_excluded(d)]
|
|
336
|
+
ambiguous_dirs = [d for d in top_level_dirs if not always_excluded(d) and d not in tool_state_excludes(root)]
|
|
319
337
|
root_files = _root_level_files(root, _exclude_matcher(_ALWAYS_EXCLUDE_FILES))
|
|
320
338
|
doc_context = _read_root_docs(root)
|
|
321
339
|
|
|
@@ -374,14 +392,17 @@ def decide_scope(root, router_model, force=False, deadline=None, cancel_check=No
|
|
|
374
392
|
|
|
375
393
|
if decision is None:
|
|
376
394
|
raise RuntimeError("The model did not define a valid scope. Check the model or set the scope manually.")
|
|
395
|
+
final = {"include": (decision or {}).get("include") or [],
|
|
396
|
+
"exclude": [*((decision or {}).get("exclude") or []), *tool_state_excludes(root)],
|
|
397
|
+
"user_exclude": (cached or {}).get("user_exclude", [])}
|
|
377
398
|
scope = {
|
|
378
|
-
"include":
|
|
379
|
-
"exclude":
|
|
399
|
+
"include": final["include"],
|
|
400
|
+
"exclude": final["exclude"],
|
|
380
401
|
"top_level_dirs": top_level_dirs,
|
|
381
402
|
"scope_version": SCOPE_VERSION,
|
|
382
|
-
"structure": _structure(root),
|
|
403
|
+
"structure": _structure(root, keep=scope_filter(final, root)),
|
|
383
404
|
"docs_digest": _docs_digest(root),
|
|
384
|
-
"user_exclude":
|
|
405
|
+
"user_exclude": final["user_exclude"],
|
|
385
406
|
"decided_at": time.time(),
|
|
386
407
|
}
|
|
387
408
|
save_scope(root, scope)
|
|
@@ -441,20 +462,32 @@ def _exclude_matcher(exclude):
|
|
|
441
462
|
return matches
|
|
442
463
|
|
|
443
464
|
|
|
444
|
-
def
|
|
445
|
-
"""
|
|
446
|
-
hidden folders.
|
|
447
|
-
|
|
465
|
+
def scope_filter(scope, root=None):
|
|
466
|
+
"""Predicate `keep(rel_path)`: whether resolve_included_files would list rel_path (a file) or walk it (a folder) —
|
|
467
|
+
under an include base, outside the exclusions, outside hidden folders. Built once so a burst of file events does not
|
|
468
|
+
rebuild the matcher per path; `root` adds the tool-runtime folders of a known config home."""
|
|
448
469
|
include = scope.get("include") or [""]
|
|
449
470
|
matcher = _exclude_matcher(set(scope.get("exclude") or []) | set(scope.get("user_exclude") or []) |
|
|
450
|
-
_ALWAYS_EXCLUDE | _ALWAYS_EXCLUDE_FILES)
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
471
|
+
set(tool_state_excludes(root) if root else ()) | _ALWAYS_EXCLUDE | _ALWAYS_EXCLUDE_FILES)
|
|
472
|
+
bases = [base.strip("/") for base in include]
|
|
473
|
+
whole = any(base in ("", ".") for base in include)
|
|
474
|
+
|
|
475
|
+
def keep(rel_path):
|
|
476
|
+
rel_path = rel_path.replace("\\", "/")
|
|
477
|
+
parts = rel_path.split("/")
|
|
478
|
+
if matcher(rel_path) or any(part.startswith(".") for part in parts[:-1]):
|
|
479
|
+
return False
|
|
480
|
+
if whole or len(parts) == 1:
|
|
481
|
+
return True
|
|
482
|
+
return any(rel_path == base or rel_path.startswith(base + "/") for base in bases)
|
|
483
|
+
|
|
484
|
+
return keep
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def in_scope(scope, rel_path, root=None):
|
|
488
|
+
"""Whether resolve_included_files would list rel_path: under an include base, outside the exclusions and outside
|
|
489
|
+
hidden folders."""
|
|
490
|
+
return scope_filter(scope, root)(rel_path)
|
|
458
491
|
|
|
459
492
|
|
|
460
493
|
def resolve_included_files(root, scope, cancel_check=None):
|
|
@@ -462,7 +495,7 @@ def resolve_included_files(root, scope, cancel_check=None):
|
|
|
462
495
|
raise ValueError("Invalid scope; request a new project analysis.")
|
|
463
496
|
include = scope.get("include") or [""]
|
|
464
497
|
matcher = _exclude_matcher(set(scope.get("exclude") or []) | set(scope.get("user_exclude") or []) |
|
|
465
|
-
_ALWAYS_EXCLUDE | _ALWAYS_EXCLUDE_FILES)
|
|
498
|
+
set(tool_state_excludes(root)) | _ALWAYS_EXCLUDE | _ALWAYS_EXCLUDE_FILES)
|
|
466
499
|
files = []
|
|
467
500
|
def walk_error(error):
|
|
468
501
|
raise OSError("Could not list a folder in the scope; try again when it is accessible.") from error
|
package/index_views.py
CHANGED
|
@@ -97,6 +97,11 @@ def storage_key(root):
|
|
|
97
97
|
return key + '.v-' + view['view_id'] if view['git'] else key
|
|
98
98
|
|
|
99
99
|
|
|
100
|
+
def stamp_changed(view):
|
|
101
|
+
"""Whether the Git files that define `view` moved since it was described (a stat of five files, no process)."""
|
|
102
|
+
return bool(view.get('git')) and _stamp(view) != view.get('stamp')
|
|
103
|
+
|
|
104
|
+
|
|
100
105
|
def assert_current(view):
|
|
101
106
|
if view.get('git') and _stamp(view) != view.get('stamp'):
|
|
102
107
|
_CACHE.pop(view['root'], None)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.26",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Allan Santos",
|
package/project_monitor.py
CHANGED
|
@@ -13,6 +13,13 @@ import index_views
|
|
|
13
13
|
|
|
14
14
|
DEBOUNCE_S = 5
|
|
15
15
|
RECONCILE_S = 180
|
|
16
|
+
# With a live watcher the events carry the changes; the stat reconciliation is only the safety net for what the buffer
|
|
17
|
+
# lost, so it runs less often (a buffer overflow already forces a stat comparison of the whole scope).
|
|
18
|
+
RECONCILE_WATCHED_S = 900
|
|
19
|
+
IDLE_TICK_S = 5
|
|
20
|
+
# A folder that changes all the time (a tool's state folder) is indexed at most this often by the monitor; the changes
|
|
21
|
+
# keep accumulating and one job covers them, and a search meanwhile reads the stale files straight from disk.
|
|
22
|
+
MIN_JOB_GAP_S = 60
|
|
16
23
|
FULL_CHECK_S = 86400
|
|
17
24
|
MAX_EVENT_PATHS = 1000
|
|
18
25
|
|
|
@@ -50,6 +57,9 @@ class ProjectMonitor:
|
|
|
50
57
|
self._git_seen = {}
|
|
51
58
|
self._git_checked = {}
|
|
52
59
|
self._pending = {}
|
|
60
|
+
self._next_job = {}
|
|
61
|
+
self._filters = {}
|
|
62
|
+
self._views = {}
|
|
53
63
|
self._lock = threading.RLock()
|
|
54
64
|
self._stop = threading.Event()
|
|
55
65
|
self._thread = None
|
|
@@ -71,16 +81,33 @@ class ProjectMonitor:
|
|
|
71
81
|
def refresh(self):
|
|
72
82
|
self._checked.clear()
|
|
73
83
|
|
|
84
|
+
def _scope_for(self, project):
|
|
85
|
+
"""`(scope, keep)` for the project's current view; reloaded only when the saved scope file changed, so a burst
|
|
86
|
+
of events does not read the scope and rebuild its matcher once per batch."""
|
|
87
|
+
path = index_scope.scope_path(project["root"])
|
|
88
|
+
try:
|
|
89
|
+
stamp = (path, os.stat(path).st_mtime_ns)
|
|
90
|
+
except OSError:
|
|
91
|
+
stamp = (path, None)
|
|
92
|
+
cached = self._filters.get(project["id"])
|
|
93
|
+
if cached and cached[0] == stamp:
|
|
94
|
+
return cached[1], cached[2]
|
|
95
|
+
scope = index_scope.load_scope(project["root"]) or {}
|
|
96
|
+
keep = index_scope.scope_filter(scope, project["root"]) if scope else (lambda rel: True)
|
|
97
|
+
self._filters[project["id"]] = (stamp, scope, keep)
|
|
98
|
+
return scope, keep
|
|
99
|
+
|
|
74
100
|
def events(self, project_id, rows, overflow=False):
|
|
75
101
|
project = project_store.get(project_id)
|
|
76
102
|
if not project or project.get("paused") or not project.get("watch") or not project.get("enabled"):
|
|
77
103
|
return
|
|
78
|
-
|
|
104
|
+
# An event under a folder the scope leaves out (plugin cache, session logs) changes nothing in the index.
|
|
105
|
+
scope, keep = self._scope_for(project)
|
|
106
|
+
paths = {row["path"] for row in rows if relevant_path(row.get("path", "")) and keep(row["path"])}
|
|
79
107
|
if not paths and not overflow:
|
|
80
108
|
return
|
|
81
109
|
# Um diretório existente também recebe MODIFIED ao mudar um arquivo filho.
|
|
82
110
|
# Só uma estrutura nova ou documentação da raiz exige reavaliar o escopo.
|
|
83
|
-
scope = index_scope.load_scope(project["root"]) or {}
|
|
84
111
|
known_dirs = {p.casefold() for p in scope.get("structure", [])}
|
|
85
112
|
docs = {name.casefold() for name in index_scope._ROOT_DOC_NAMES}
|
|
86
113
|
force_scope = any(path.casefold() in docs for path in paths) or any(
|
|
@@ -94,12 +121,12 @@ class ProjectMonitor:
|
|
|
94
121
|
state["revision"] = state.get("revision", 0) + 1
|
|
95
122
|
state["paths"].update(paths)
|
|
96
123
|
state["last"] = time.monotonic()
|
|
97
|
-
state["full"] |= overflow or len(state["paths"]) > MAX_EVENT_PATHS
|
|
98
124
|
state["force_scope"] |= force_scope
|
|
99
|
-
|
|
125
|
+
# Lost or too many events: the update compares stat of the whole scope (mtime and size), which finds what
|
|
126
|
+
# changed without reading and hashing every file the way a full check does.
|
|
127
|
+
if len(state["paths"]) > MAX_EVENT_PATHS:
|
|
100
128
|
state["paths"].clear()
|
|
101
|
-
project_store.
|
|
102
|
-
project_store.record_changes(project_id, paths, full=overflow, scope=force_scope)
|
|
129
|
+
project_store.note_changes(project_id, "Changes detected by the monitor.", paths, scope=force_scope)
|
|
103
130
|
|
|
104
131
|
def _watch_error(self, project_id, message):
|
|
105
132
|
project_store.update(project_id, watch_backend="reconciliation", watch_error=message[:300])
|
|
@@ -132,6 +159,7 @@ class ProjectMonitor:
|
|
|
132
159
|
now = time.monotonic()
|
|
133
160
|
projects = project_store.all_projects()
|
|
134
161
|
self._sync_watchers(projects, now)
|
|
162
|
+
busy = project_store.projects_with_pending_job()
|
|
135
163
|
for project in projects:
|
|
136
164
|
key = project["id"]
|
|
137
165
|
if not project.get("watch") or not project.get("enabled") or project.get("paused"):
|
|
@@ -139,16 +167,21 @@ class ProjectMonitor:
|
|
|
139
167
|
if not os.path.isdir(project["root"]):
|
|
140
168
|
project_store.update(key, status="unavailable", last_error="The folder is not accessible.")
|
|
141
169
|
continue
|
|
142
|
-
if
|
|
170
|
+
if key in busy:
|
|
143
171
|
continue # Não planejar outra geração com metadados de uma ainda em curso.
|
|
144
172
|
if now - self._git_checked.get(key, -5) >= 5:
|
|
145
|
-
view
|
|
173
|
+
# A Git view is re-identified (four git processes) only when the files that define it moved.
|
|
174
|
+
view = self._views.get(key)
|
|
175
|
+
if view is None or not view.get('git') or index_views.stamp_changed(view):
|
|
176
|
+
view = self._views[key] = index_views.describe(project['root'], fresh=True)
|
|
146
177
|
signature = (view['view_id'], view.get('commit'))
|
|
147
178
|
if key in self._git_seen and signature != self._git_seen[key]:
|
|
148
179
|
self.events(key, [], overflow=True)
|
|
149
180
|
self._git_seen[key] = signature
|
|
150
181
|
self._git_checked[key] = now
|
|
151
|
-
|
|
182
|
+
watcher = self._watchers.get(key)
|
|
183
|
+
interval = RECONCILE_WATCHED_S if watcher is not None and watcher.alive else RECONCILE_S
|
|
184
|
+
if now - self._checked.get(key, -interval) >= interval:
|
|
152
185
|
self._checked[key] = now
|
|
153
186
|
scope = index_scope.load_scope(project["root"])
|
|
154
187
|
meta = indexer.info(project["root"])
|
|
@@ -177,7 +210,7 @@ class ProjectMonitor:
|
|
|
177
210
|
continue
|
|
178
211
|
with self._lock:
|
|
179
212
|
pending = self._pending.get(key)
|
|
180
|
-
if not pending or now - pending["last"] < DEBOUNCE_S:
|
|
213
|
+
if not pending or now - pending["last"] < DEBOUNCE_S or now < self._next_job.get(key, 0):
|
|
181
214
|
continue
|
|
182
215
|
options = {"changed_paths": sorted(pending["paths"]), "full_check": pending["full"],
|
|
183
216
|
"force_scope": pending["force_scope"]}
|
|
@@ -190,6 +223,7 @@ class ProjectMonitor:
|
|
|
190
223
|
except Exception as exc:
|
|
191
224
|
project_store.update(key, status="error", last_error=str(exc)[:300], next_retry_at=time.time() + 60)
|
|
192
225
|
continue
|
|
226
|
+
self._next_job[key] = now + MIN_JOB_GAP_S
|
|
193
227
|
with self._lock:
|
|
194
228
|
# Um evento recebido durante o submit continua pendente para a próxima geração.
|
|
195
229
|
if self._pending.get(key) is pending and pending.get("revision", 0) == revision:
|
|
@@ -201,4 +235,4 @@ class ProjectMonitor:
|
|
|
201
235
|
self.tick()
|
|
202
236
|
except Exception:
|
|
203
237
|
pass # Falha opcional do monitor não derruba o daemon MCP.
|
|
204
|
-
self._stop.wait(1)
|
|
238
|
+
self._stop.wait(1 if self._pending else IDLE_TICK_S)
|
package/project_store.py
CHANGED
|
@@ -13,18 +13,25 @@ import secret_store
|
|
|
13
13
|
|
|
14
14
|
DB_PATH = os.path.join(paths.DATA_DIR, "projects.sqlite3")
|
|
15
15
|
_LOCK = threading.RLock()
|
|
16
|
+
_SCHEMA_READY = set()
|
|
16
17
|
MAX_JOBS = 500
|
|
17
18
|
|
|
18
19
|
|
|
19
20
|
@contextmanager
|
|
20
21
|
def _db():
|
|
21
22
|
os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
|
|
23
|
+
if not os.path.isfile(DB_PATH):
|
|
24
|
+
_SCHEMA_READY.discard(DB_PATH)
|
|
22
25
|
db = sqlite3.connect(DB_PATH, timeout=10)
|
|
23
26
|
try:
|
|
24
27
|
db.execute("PRAGMA secure_delete=ON")
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
+
if DB_PATH not in _SCHEMA_READY:
|
|
29
|
+
db.execute("CREATE TABLE IF NOT EXISTS projects(id TEXT PRIMARY KEY, root TEXT NOT NULL, data TEXT NOT NULL, updated_at REAL NOT NULL)")
|
|
30
|
+
db.execute("CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, project_id TEXT, status TEXT NOT NULL, updated_at REAL NOT NULL, payload TEXT NOT NULL)")
|
|
31
|
+
db.execute("CREATE INDEX IF NOT EXISTS jobs_project ON jobs(project_id,updated_at)")
|
|
32
|
+
db.execute("CREATE INDEX IF NOT EXISTS jobs_updated ON jobs(updated_at)")
|
|
33
|
+
db.execute("CREATE INDEX IF NOT EXISTS jobs_status ON jobs(status,project_id)")
|
|
34
|
+
_SCHEMA_READY.add(DB_PATH)
|
|
28
35
|
with db:
|
|
29
36
|
yield db
|
|
30
37
|
finally:
|
|
@@ -102,17 +109,62 @@ def record_changes(project_id, paths=(), full=False, scope=False):
|
|
|
102
109
|
project = get(project_id)
|
|
103
110
|
if not project:
|
|
104
111
|
return
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
112
|
+
update(project_id, **_changes_fields(project, paths, full, scope))
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _changes_fields(project, paths, full, scope):
|
|
116
|
+
"""`needs_full_check` / `scope_dirty` remember the `dirty_seq` at which they were raised (`*_seq`), so a job that ran
|
|
117
|
+
after that sequence can clear them even while events keep arriving."""
|
|
118
|
+
merged = set(project.get('dirty_paths', [])) | set(paths)
|
|
119
|
+
full = full or project.get('needs_full_check', False)
|
|
120
|
+
scope = scope or project.get('scope_dirty', False)
|
|
121
|
+
seq = project.get('dirty_seq', 0)
|
|
122
|
+
# Past 1000 paths the list stops helping: the update compares stat of the whole scope instead.
|
|
123
|
+
return {"dirty_paths": [] if len(merged) > 1000 else sorted(merged), "needs_full_check": full,
|
|
124
|
+
"scope_dirty": scope,
|
|
125
|
+
"full_check_seq": seq if full and not project.get('needs_full_check') else project.get('full_check_seq', 0),
|
|
126
|
+
"scope_dirty_seq": seq if scope and not project.get('scope_dirty') else project.get('scope_dirty_seq', 0)}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def note_changes(project_id, reason, paths=(), full=False, scope=False):
|
|
130
|
+
"""mark_dirty + record_changes in one read and one write of the registry; a burst of file events costs one commit per
|
|
131
|
+
batch instead of two reads and two commits."""
|
|
132
|
+
with _LOCK, _db() as db:
|
|
133
|
+
row = db.execute("SELECT data FROM projects WHERE id=?", (project_id,)).fetchone()
|
|
134
|
+
if not row:
|
|
135
|
+
return None
|
|
136
|
+
data = json.loads(row[0])
|
|
137
|
+
data["dirty_seq"] = data.get("dirty_seq", 0) + 1
|
|
138
|
+
data.update(status="dirty" if not data.get("paused") else "paused", dirty_reason=reason,
|
|
139
|
+
last_event=time.time(), **_changes_fields(data, paths, full, scope))
|
|
140
|
+
db.execute("UPDATE projects SET data=?,updated_at=? WHERE id=?",
|
|
141
|
+
(json.dumps(data, ensure_ascii=False), time.time(), project_id))
|
|
142
|
+
return data
|
|
109
143
|
|
|
110
144
|
|
|
111
145
|
def confirm_changes(project_id, sequence):
|
|
146
|
+
"""Clears what the finished job covered: the path list only if nothing changed since the job captured `sequence`;
|
|
147
|
+
the full-check and scope flags if they were raised at or before it (the job did both from that state)."""
|
|
112
148
|
with _LOCK:
|
|
113
149
|
project = get(project_id)
|
|
114
|
-
if
|
|
115
|
-
|
|
150
|
+
if not project:
|
|
151
|
+
return
|
|
152
|
+
fields = {}
|
|
153
|
+
if project.get('dirty_seq', 0) == sequence:
|
|
154
|
+
fields.update(dirty_paths=[], needs_full_check=False, scope_dirty=False)
|
|
155
|
+
else:
|
|
156
|
+
if project.get('needs_full_check') and project.get('full_check_seq', 0) <= sequence:
|
|
157
|
+
fields["needs_full_check"] = False
|
|
158
|
+
if project.get('scope_dirty') and project.get('scope_dirty_seq', 0) <= sequence:
|
|
159
|
+
fields["scope_dirty"] = False
|
|
160
|
+
if fields:
|
|
161
|
+
update(project_id, **fields)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def projects_with_pending_job():
|
|
165
|
+
"""Ids of the projects with a job still pending, from the `status` column: no job payload is decrypted."""
|
|
166
|
+
with _LOCK, _db() as db:
|
|
167
|
+
return {row[0] for row in db.execute("SELECT DISTINCT project_id FROM jobs WHERE status='pending'")}
|
|
116
168
|
|
|
117
169
|
|
|
118
170
|
def save_job(job_id, payload):
|
|
@@ -123,8 +175,10 @@ def save_job(job_id, payload):
|
|
|
123
175
|
with _LOCK, _db() as db:
|
|
124
176
|
db.execute("INSERT OR REPLACE INTO jobs VALUES(?,?,?,?,?)",
|
|
125
177
|
(job_id, payload.get("project_id"), payload.get("status", "pending"), time.time(), protected))
|
|
126
|
-
|
|
127
|
-
|
|
178
|
+
# The threshold comes from the updated_at index alone: ordering the payload rows read the whole jobs table
|
|
179
|
+
# (1.7 MB) on every save, several saves per job.
|
|
180
|
+
db.execute("DELETE FROM jobs WHERE status!='pending' AND updated_at < "
|
|
181
|
+
"(SELECT updated_at FROM jobs ORDER BY updated_at DESC LIMIT 1 OFFSET ?)", (MAX_JOBS - 1,))
|
|
128
182
|
|
|
129
183
|
|
|
130
184
|
def _decode(payload):
|
package/smart_tool_daemon.py
CHANGED
|
@@ -1873,7 +1873,8 @@ def _project_action_locked(arguments):
|
|
|
1873
1873
|
"storage_dir": indexer.INDEX_DIR,
|
|
1874
1874
|
"gateway": dict(_GATEWAY_STATUS),
|
|
1875
1875
|
"monitor": {"debounce_s": project_monitor.DEBOUNCE_S,
|
|
1876
|
-
"reconcile_s": project_monitor.RECONCILE_S
|
|
1876
|
+
"reconcile_s": project_monitor.RECONCILE_S,
|
|
1877
|
+
"reconcile_watched_s": project_monitor.RECONCILE_WATCHED_S}}
|
|
1877
1878
|
if action == "register":
|
|
1878
1879
|
root = arguments.get("project_root")
|
|
1879
1880
|
if not isinstance(root, str) or not os.path.isabs(root):
|
package/version.py
CHANGED