@allansantos-dev/smart-tool 0.9.66 → 0.9.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +2 -2
- package/browser_control.py +24 -6
- package/codex_config.py +126 -0
- package/endpoint_sync.py +12 -3
- package/index_scope.py +4 -3
- package/package.json +1 -1
- package/project_monitor.py +120 -32
- package/project_store.py +24 -4
- package/scratch.py +84 -0
- package/version.py +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.67 - beta
|
|
4
|
+
|
|
5
|
+
- The idle daemon no longer rewrites its own registry in a loop. The folder where Smart Tool keeps its state was a
|
|
6
|
+
watched project: every registry commit raised file events whose handling wrote the registry again (about 1,300
|
|
7
|
+
events and 570-700 writes of 60 KB per 120 s). That folder is never watched, reconciled or noted
|
|
8
|
+
(`project_monitor.holds_own_state`), SQLite `-journal`, `-wal` and `-shm` files, Office lock files (`~$*`) and unfinished downloads (`*.crdownload`, `*.part`) are transient (`index_scope.py`), the
|
|
9
|
+
event batches of one debounce window are recorded in a single registry write, and a quiet tick dispatches ready jobs
|
|
10
|
+
without listing every project (`project_monitor.py`, `tests/test_monitor_io.py`). Measured with
|
|
11
|
+
`tests/daemon_idle_io_probe.py` on the same registry copy, 75 watched projects, idle: 1.068 to 0.263 MB/s read and
|
|
12
|
+
0.393 to 0.071 MB/s written; `note_changes` calls 346 to 54. Note: `docs/research/2026-11-08-daemon-idle-disk-load.md`.
|
|
13
|
+
- The project registry no longer runs with `secure_delete=ON`: it holds folder paths and change notes, no content or
|
|
14
|
+
secrets (the indexes keep it), and SQLite documents that option as zero-filling every freed page, extra I/O it leaves
|
|
15
|
+
off by default. `project_store.all_projects` is read from disk again only after a write by the process or a change of
|
|
16
|
+
the file: the monitor listed the whole registry (about 565 KB) every 5 s, which was nearly all of the idle read. A
|
|
17
|
+
project whose index was last updated more than 7 days ago gets no folder watcher and is reconciled every 15 minutes;
|
|
18
|
+
its first update gives it a watcher again. A project whose folder no longer exists is marked `unavailable` once instead
|
|
19
|
+
of on every 5 s pass, and a start no longer rewrites `watch_backend` that already holds the right value
|
|
20
|
+
(`project_store.py`, `project_monitor.py`, `tests/test_monitor_io.py`).
|
|
21
|
+
|
|
3
22
|
## 0.9.66 - beta
|
|
4
23
|
|
|
5
24
|
- A daemon that starts while its port is busy retries that port for a few seconds, re-checking `/health`, before it
|
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@ registered as the `playwright` server where this machine lets a browser be drive
|
|
|
17
17
|
An optional hook routes the agent's native Grep/Glob/Read/Bash/WebSearch/WebFetch calls to these tools when they
|
|
18
18
|
are cheaper, or just tells the agent what Smart Tool would do.
|
|
19
19
|
|
|
20
|
-
**Status:** 0.9.
|
|
20
|
+
**Status:** 0.9.67 beta. Windows only. What changed in each version: [CHANGELOG.md](CHANGELOG.md).
|
|
21
21
|
|
|
22
22
|
## How it works
|
|
23
23
|
|
|
@@ -83,7 +83,7 @@ gateway such as Ollama, nothing about your code leaves the machine.
|
|
|
83
83
|
## Install
|
|
84
84
|
|
|
85
85
|
```powershell
|
|
86
|
-
npx @allansantos-dev/smart-tool@0.9.
|
|
86
|
+
npx @allansantos-dev/smart-tool@0.9.67 install
|
|
87
87
|
```
|
|
88
88
|
|
|
89
89
|
Updates install themselves, on by default: the tray checks npm when it starts (at logon) and every 24 h, and when there
|
package/browser_control.py
CHANGED
|
@@ -29,6 +29,7 @@ import time
|
|
|
29
29
|
import tomllib
|
|
30
30
|
|
|
31
31
|
import atomic_io
|
|
32
|
+
import codex_config
|
|
32
33
|
import endpoint_sync
|
|
33
34
|
import install_runtime
|
|
34
35
|
import lazy_mcp_proxy
|
|
@@ -358,6 +359,23 @@ def _clip(text, limit=300):
|
|
|
358
359
|
return text if len(text) <= limit else text[:limit // 3] + " ... " + text[-(limit - limit // 3 - 5):]
|
|
359
360
|
|
|
360
361
|
|
|
362
|
+
def _add(client, command):
|
|
363
|
+
"""Registers the stdio server: Codex's config.toml is edited in place (its CLI rewrites the whole file and drops
|
|
364
|
+
other tools' comments), Claude Code goes through its CLI."""
|
|
365
|
+
if client == "codex":
|
|
366
|
+
codex_config.set_server(_codex_config(), SERVER, {"command": command[0], "args": command[1:]})
|
|
367
|
+
else:
|
|
368
|
+
endpoint_sync.run_cli(register_command(client, command))
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _remove(client):
|
|
372
|
+
"""Removes the server `_add` registered."""
|
|
373
|
+
if client == "codex":
|
|
374
|
+
codex_config.remove_server(_codex_config(), SERVER)
|
|
375
|
+
else:
|
|
376
|
+
endpoint_sync.run_cli(remove_command(client))
|
|
377
|
+
|
|
378
|
+
|
|
361
379
|
def ensure(client, log=None):
|
|
362
380
|
"""Registers Playwright MCP in the client, or refreshes the registration Smart Tool made before (Node, Chromium
|
|
363
381
|
and install paths change between versions). A `playwright` server the user registered is kept. When the browser
|
|
@@ -376,22 +394,22 @@ def ensure(client, log=None):
|
|
|
376
394
|
problem = str(exc)
|
|
377
395
|
try:
|
|
378
396
|
if ours and registered().get(client):
|
|
379
|
-
|
|
397
|
+
_remove(client)
|
|
380
398
|
if problem:
|
|
381
399
|
if ours and client == "claude":
|
|
382
400
|
set_claude_deny(False)
|
|
383
401
|
return _record(client, "unavailable", problem[:300], log)
|
|
384
|
-
|
|
402
|
+
_add(client, lazy_command(command))
|
|
385
403
|
if not registered().get(client):
|
|
386
|
-
return _record(client, "unavailable", "the
|
|
387
|
-
"configuration", log)
|
|
404
|
+
return _record(client, "unavailable", "the registration finished, but the server did not appear in the "
|
|
405
|
+
"client's configuration", log)
|
|
388
406
|
try:
|
|
389
407
|
deny_unsafe_tools(client)
|
|
390
408
|
except (OSError, ValueError) as exc:
|
|
391
409
|
problem = f"{', '.join(DENIED_TOOLS)} could not be denied ({exc})"
|
|
392
410
|
try:
|
|
393
|
-
|
|
394
|
-
except RuntimeError as removal:
|
|
411
|
+
_remove(client)
|
|
412
|
+
except (OSError, RuntimeError, ValueError) as removal:
|
|
395
413
|
return _record(client, "undenied", f"remove it with `{' '.join(remove_command(client))}`: {problem} "
|
|
396
414
|
f"and removing it failed ({removal})"[:300], log)
|
|
397
415
|
return _record(client, "unavailable", f"{problem}, so the server was removed"[:300], log)
|
package/codex_config.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""In-place edits of the `[mcp_servers.<name>]` tables of Codex's config.toml.
|
|
2
|
+
|
|
3
|
+
`codex mcp add` and `codex mcp remove` serialize the whole file again and drop the comments, among them the markers
|
|
4
|
+
other tools put around the blocks they manage. Here only the table of the named server is added, replaced or removed;
|
|
5
|
+
every other byte of the file stays as it was.
|
|
6
|
+
"""
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import tomllib
|
|
10
|
+
|
|
11
|
+
import atomic_io
|
|
12
|
+
|
|
13
|
+
_BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _header_path(line):
|
|
17
|
+
"""Key path of a table header line (`[a.b]` -> ["a", "b"]); None when the line is not a header."""
|
|
18
|
+
if not line.lstrip().startswith("["):
|
|
19
|
+
return None
|
|
20
|
+
try:
|
|
21
|
+
node = tomllib.loads(line)
|
|
22
|
+
except tomllib.TOMLDecodeError:
|
|
23
|
+
return None
|
|
24
|
+
path = []
|
|
25
|
+
while isinstance(node, dict) and len(node) == 1:
|
|
26
|
+
key, node = next(iter(node.items()))
|
|
27
|
+
path.append(key)
|
|
28
|
+
if isinstance(node, list):
|
|
29
|
+
break
|
|
30
|
+
return path
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _span(lines, name):
|
|
34
|
+
"""(start, end) line indexes of the server's table and its subtables, without the comments and blank lines that
|
|
35
|
+
trail it (they belong to what comes next); None when the file has no such table."""
|
|
36
|
+
start = end = None
|
|
37
|
+
for index, line in enumerate(lines):
|
|
38
|
+
path = _header_path(line)
|
|
39
|
+
if path is None:
|
|
40
|
+
continue
|
|
41
|
+
if path[:2] == ["mcp_servers", name]:
|
|
42
|
+
start = index if start is None else start
|
|
43
|
+
elif start is not None and end is None:
|
|
44
|
+
end = index
|
|
45
|
+
if start is None:
|
|
46
|
+
return None
|
|
47
|
+
end = len(lines) if end is None else end
|
|
48
|
+
while end > start + 1 and (not lines[end - 1].strip() or lines[end - 1].lstrip().startswith("#")):
|
|
49
|
+
end -= 1
|
|
50
|
+
return start, end
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _value(value):
|
|
54
|
+
if isinstance(value, str):
|
|
55
|
+
return '"' + value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n").replace("\r", "\\r") \
|
|
56
|
+
.replace("\t", "\\t") + '"'
|
|
57
|
+
if isinstance(value, (list, tuple)):
|
|
58
|
+
return "[" + ", ".join(_value(item) for item in value) + "]"
|
|
59
|
+
raise TypeError(f"Unsupported value for config.toml: {value!r}")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _table(name, fields, eol):
|
|
63
|
+
key = name if _BARE_KEY.match(name) else _value(name)
|
|
64
|
+
return eol.join([f"[mcp_servers.{key}]", *(f"{field} = {_value(value)}" for field, value in fields.items())]) + eol
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _read(path):
|
|
68
|
+
try:
|
|
69
|
+
with open(path, encoding="utf-8", newline="") as stream:
|
|
70
|
+
return stream.read()
|
|
71
|
+
except FileNotFoundError:
|
|
72
|
+
return ""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _others(text, name):
|
|
76
|
+
"""Everything the file defines except the server `name`, as parsed TOML."""
|
|
77
|
+
data = tomllib.loads(text)
|
|
78
|
+
servers = data.get("mcp_servers") or {}
|
|
79
|
+
servers.pop(name, None)
|
|
80
|
+
if not servers:
|
|
81
|
+
data.pop("mcp_servers", None)
|
|
82
|
+
return data
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _write(path, before, after, name, fields):
|
|
86
|
+
"""Writes after (the edited text) when it parses, holds exactly `fields` for the server and changed nothing else.
|
|
87
|
+
Raises ValueError otherwise, leaving the file alone."""
|
|
88
|
+
try:
|
|
89
|
+
parsed = tomllib.loads(after)
|
|
90
|
+
unchanged = _others(after, name) == _others(before, name)
|
|
91
|
+
except tomllib.TOMLDecodeError as exc:
|
|
92
|
+
raise ValueError(f"{path} would not be valid TOML after editing [mcp_servers.{name}]: {exc}") from None
|
|
93
|
+
held = (parsed.get("mcp_servers") or {}).get(name)
|
|
94
|
+
if not unchanged or held != (fields or None):
|
|
95
|
+
raise ValueError(f"Editing [mcp_servers.{name}] in {path} would change more than that table "
|
|
96
|
+
f"(a layout this editor does not handle); edit the file by hand.")
|
|
97
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
98
|
+
atomic_io.write_text(path, after)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def set_server(path, name, fields):
|
|
102
|
+
"""Makes `[mcp_servers.<name>]` in the config.toml at path hold exactly `fields` (string or list of strings per
|
|
103
|
+
key): the existing table is replaced where it is, a missing one is appended. Raises ValueError, writing nothing,
|
|
104
|
+
when the file is not valid TOML or the edit would touch anything else."""
|
|
105
|
+
before = _read(path)
|
|
106
|
+
eol = "\r\n" if "\r\n" in before else "\n"
|
|
107
|
+
lines = before.splitlines(keepends=True)
|
|
108
|
+
span = _span(lines, name)
|
|
109
|
+
table = _table(name, fields, eol)
|
|
110
|
+
if span:
|
|
111
|
+
after = "".join(lines[:span[0]]) + table + "".join(lines[span[1]:])
|
|
112
|
+
else:
|
|
113
|
+
gap = "" if not before or before.endswith(eol + eol) else (eol if before.endswith(eol) else eol + eol)
|
|
114
|
+
after = before + gap + table
|
|
115
|
+
_write(path, before, after, name, fields)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def remove_server(path, name):
|
|
119
|
+
"""Removes `[mcp_servers.<name>]` and its subtables from the config.toml at path; nothing happens when it is not
|
|
120
|
+
there. Raises ValueError, writing nothing, when the edit would touch anything else."""
|
|
121
|
+
before = _read(path)
|
|
122
|
+
lines = before.splitlines(keepends=True)
|
|
123
|
+
span = _span(lines, name)
|
|
124
|
+
if not span:
|
|
125
|
+
return
|
|
126
|
+
_write(path, before, "".join(lines[:span[0]] + lines[span[1]:]), name, {})
|
package/endpoint_sync.py
CHANGED
|
@@ -12,6 +12,7 @@ import subprocess
|
|
|
12
12
|
import time
|
|
13
13
|
import urllib.parse
|
|
14
14
|
|
|
15
|
+
import codex_config
|
|
15
16
|
import node_locator
|
|
16
17
|
|
|
17
18
|
SERVER = "smart-tool"
|
|
@@ -139,13 +140,21 @@ def remove_command(client):
|
|
|
139
140
|
|
|
140
141
|
|
|
141
142
|
def register(client, base):
|
|
142
|
-
"""Makes the client's smart-tool MCP entry point to base
|
|
143
|
-
~/.claude.json is rewritten by Claude Code all the time)
|
|
144
|
-
|
|
143
|
+
"""Makes the client's smart-tool MCP entry point to base. Claude Code goes through its own CLI (it owns its config
|
|
144
|
+
file; ~/.claude.json is rewritten by Claude Code all the time): another URL (the port changed) is removed first
|
|
145
|
+
because `mcp add` refuses an existing name, no entry only adds. Codex's config.toml is edited in place, its table
|
|
146
|
+
only (`codex mcp add` rewrites the whole file and drops other tools' comment markers). Idempotent: the same URL
|
|
147
|
+
changes nothing. Returns what was done."""
|
|
145
148
|
url = f"{base}/mcp"
|
|
146
149
|
current = registrations().get(client)
|
|
147
150
|
if current == url:
|
|
148
151
|
return f"nothing to run: {client} already points to {url}"
|
|
152
|
+
if client == "codex":
|
|
153
|
+
path = os.path.join(codex_dir(), "config.toml")
|
|
154
|
+
codex_config.set_server(path, SERVER, {"url": url})
|
|
155
|
+
if registrations().get(client) != url:
|
|
156
|
+
raise RuntimeError(f"{path} was edited, but [mcp_servers.{SERVER}] does not hold {url}.")
|
|
157
|
+
return f"edited {path}: [mcp_servers.{SERVER}] url = {url}"
|
|
149
158
|
command = register_command(client, base)
|
|
150
159
|
if current:
|
|
151
160
|
run_cli(remove_command(client))
|
package/index_scope.py
CHANGED
|
@@ -41,8 +41,9 @@ _PROJECT_TYPE_EXCLUDES = (
|
|
|
41
41
|
(("Podfile",), ("Pods",)),
|
|
42
42
|
)
|
|
43
43
|
|
|
44
|
-
# Files a tool writes while it works and replaces or deletes right after (atomic-write temp, backup, editor swap
|
|
45
|
-
|
|
44
|
+
# Files a tool writes while it works and replaces or deletes right after (atomic-write temp, backup, editor swap, the
|
|
45
|
+
# journal and shared-memory files SQLite keeps beside a database).
|
|
46
|
+
TRANSIENT_FILES = {"*.tmp", "*.bak", "*.swp", "*~", "*-journal", "*-wal", "*-shm", "~$*", "*.crdownload", "*.part"}
|
|
46
47
|
|
|
47
48
|
# Mesmo critério do `_ALWAYS_EXCLUDE`, para arquivo: o significado não varia por
|
|
48
49
|
# projeto. Lockfile é grafo de dependência resolvido por ferramenta — nunca é fonte de
|
|
@@ -115,7 +116,7 @@ def normalize_include(include):
|
|
|
115
116
|
|
|
116
117
|
|
|
117
118
|
def is_transient(name):
|
|
118
|
-
"""Whether a file name is a temporary, backup or editor-swap file (`*.tmp`, `*.bak`, `*.swp`,
|
|
119
|
+
"""Whether a file name is a temporary, backup or editor-swap file (`*.tmp`, `*.bak`, `*.swp`, `*~`, SQLite `*-journal`)."""
|
|
119
120
|
return any(fnmatch.fnmatch(name.casefold(), pattern) for pattern in TRANSIENT_FILES)
|
|
120
121
|
|
|
121
122
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.67",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Allan Santos",
|
package/project_monitor.py
CHANGED
|
@@ -8,6 +8,8 @@ import directory_watch
|
|
|
8
8
|
import indexer
|
|
9
9
|
import local_embedder
|
|
10
10
|
import index_scope
|
|
11
|
+
import paths
|
|
12
|
+
import project_identity
|
|
11
13
|
import project_store
|
|
12
14
|
import index_views
|
|
13
15
|
|
|
@@ -17,6 +19,9 @@ RECONCILE_S = 180
|
|
|
17
19
|
# lost, so it runs less often (a buffer overflow already forces a stat comparison of the whole scope).
|
|
18
20
|
RECONCILE_WATCHED_S = 900
|
|
19
21
|
IDLE_TICK_S = 5
|
|
22
|
+
# A project whose index was last updated longer ago than this gets no folder watcher: its folder is checked by the
|
|
23
|
+
# reconciliation instead, and the first update it needs gives it a watcher again.
|
|
24
|
+
DORMANT_S = 7 * 86400
|
|
20
25
|
# A folder that changes all the time (a tool's state folder) is indexed at most this often by the monitor; the changes
|
|
21
26
|
# keep accumulating and one job covers them, and a search meanwhile reads the stale files straight from disk.
|
|
22
27
|
MIN_JOB_GAP_S = 60
|
|
@@ -33,6 +38,26 @@ def relevant_path(path):
|
|
|
33
38
|
and not index_scope.is_transient(parts[-1]))
|
|
34
39
|
|
|
35
40
|
|
|
41
|
+
def holds_own_state(root):
|
|
42
|
+
"""Whether `root` is the folder where Smart Tool keeps its state, or lies inside it. Every write the daemon makes
|
|
43
|
+
there (registry, indexes, logs, metrics) would raise a file event for the folder, and handling that event writes the
|
|
44
|
+
registry again, so such a folder is never watched or reconciled by the monitor."""
|
|
45
|
+
folder, state = (project_identity.canonical_root(path) for path in (root, paths.HOME_DIR))
|
|
46
|
+
return folder == state or folder.startswith(state + os.sep)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def monitored(project):
|
|
50
|
+
"""Whether the monitor follows this project: watched by the user, enabled, not paused, and not Smart Tool's own state."""
|
|
51
|
+
return bool(project.get("watch") and project.get("enabled") and not project.get("paused")
|
|
52
|
+
and not holds_own_state(project["root"]))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def dormant(project, now=None):
|
|
56
|
+
"""Whether the index of a monitored project has not been updated for DORMANT_S; one never updated is not dormant."""
|
|
57
|
+
indexed = project.get("last_indexed")
|
|
58
|
+
return bool(indexed) and (time.time() if now is None else now) - indexed > DORMANT_S
|
|
59
|
+
|
|
60
|
+
|
|
36
61
|
def snapshot(root, scope):
|
|
37
62
|
result = {}
|
|
38
63
|
for rel in index_scope.resolve_included_files(root, scope):
|
|
@@ -60,6 +85,8 @@ class ProjectMonitor:
|
|
|
60
85
|
self._git_seen = {}
|
|
61
86
|
self._git_checked = {}
|
|
62
87
|
self._pending = {}
|
|
88
|
+
self._unsaved = {}
|
|
89
|
+
self._recent = {}
|
|
63
90
|
self._next_job = {}
|
|
64
91
|
self._gap_level = {}
|
|
65
92
|
self._filters = {}
|
|
@@ -76,6 +103,7 @@ class ProjectMonitor:
|
|
|
76
103
|
self._stop.set()
|
|
77
104
|
if self._thread:
|
|
78
105
|
self._thread.join(timeout=3)
|
|
106
|
+
self._flush_notes(time.monotonic(), force=True)
|
|
79
107
|
with self._lock:
|
|
80
108
|
watchers = list(self._watchers.values())
|
|
81
109
|
self._watchers.clear()
|
|
@@ -102,8 +130,12 @@ class ProjectMonitor:
|
|
|
102
130
|
return scope, keep
|
|
103
131
|
|
|
104
132
|
def events(self, project_id, rows, overflow=False):
|
|
105
|
-
|
|
106
|
-
|
|
133
|
+
now = time.monotonic()
|
|
134
|
+
with self._lock:
|
|
135
|
+
recent = self._recent.get(project_id)
|
|
136
|
+
window_open = bool(recent) and now - recent[0] < DEBOUNCE_S
|
|
137
|
+
project = recent[1] if window_open else project_store.get(project_id)
|
|
138
|
+
if not project or not monitored(project):
|
|
107
139
|
return
|
|
108
140
|
# An event under a folder the scope leaves out (plugin cache, session logs) changes nothing in the index.
|
|
109
141
|
scope, keep = self._scope_for(project)
|
|
@@ -130,7 +162,31 @@ class ProjectMonitor:
|
|
|
130
162
|
# changed without reading and hashing every file the way a full check does.
|
|
131
163
|
if len(state["paths"]) > MAX_EVENT_PATHS:
|
|
132
164
|
state["paths"].clear()
|
|
133
|
-
|
|
165
|
+
if window_open:
|
|
166
|
+
unsaved = self._unsaved.setdefault(project_id, {"paths": set(), "scope": False})
|
|
167
|
+
unsaved["paths"].update(paths)
|
|
168
|
+
unsaved["scope"] |= force_scope
|
|
169
|
+
return
|
|
170
|
+
self._note(project_id, paths, force_scope, now)
|
|
171
|
+
|
|
172
|
+
def _note(self, project_id, paths, scope, now):
|
|
173
|
+
"""Records the changes in the registry; later batches of the same project wait in memory and go in one write
|
|
174
|
+
once DEBOUNCE_S has passed, because a folder that changes all the time would otherwise rewrite the registry
|
|
175
|
+
for every batch of events."""
|
|
176
|
+
data = project_store.note_changes(project_id, "Changes detected by the monitor.", paths, scope=scope)
|
|
177
|
+
with self._lock:
|
|
178
|
+
if data:
|
|
179
|
+
self._recent[project_id] = (now, data)
|
|
180
|
+
else:
|
|
181
|
+
self._recent.pop(project_id, None)
|
|
182
|
+
|
|
183
|
+
def _flush_notes(self, now, force=False):
|
|
184
|
+
"""Writes the batches held back by `_note` for the projects whose window has passed (all of them when forced)."""
|
|
185
|
+
with self._lock:
|
|
186
|
+
due = [key for key in self._unsaved if force or now - self._recent.get(key, (0,))[0] >= DEBOUNCE_S]
|
|
187
|
+
batches = {key: self._unsaved.pop(key) for key in due}
|
|
188
|
+
for key, batch in batches.items():
|
|
189
|
+
self._note(key, batch["paths"], batch["scope"], now)
|
|
134
190
|
|
|
135
191
|
def _watch_error(self, project_id, message):
|
|
136
192
|
project_store.update(project_id, watch_backend="reconciliation", watch_error=message[:300])
|
|
@@ -138,13 +194,17 @@ class ProjectMonitor:
|
|
|
138
194
|
self.events(project_id, [], overflow=True)
|
|
139
195
|
|
|
140
196
|
def _sync_watchers(self, projects, now):
|
|
141
|
-
|
|
142
|
-
|
|
197
|
+
followed = {p["id"]: p for p in projects if monitored(p) and os.path.isdir(p["root"])}
|
|
198
|
+
wall = time.time()
|
|
199
|
+
wanted = {key: p for key, p in followed.items() if not dormant(p, wall)}
|
|
143
200
|
for key in list(self._watchers):
|
|
144
201
|
if key not in wanted:
|
|
145
202
|
watcher = self._watchers.pop(key)
|
|
146
203
|
watcher.stop()
|
|
147
|
-
project_store.update(key, watch_backend="off")
|
|
204
|
+
project_store.update(key, watch_backend="reconciliation" if key in followed else "off")
|
|
205
|
+
for key, project in followed.items():
|
|
206
|
+
if key not in wanted and project.get("watch_backend") != "reconciliation":
|
|
207
|
+
project_store.update(key, watch_backend="reconciliation", watch_error="")
|
|
148
208
|
for key, project in wanted.items():
|
|
149
209
|
current = self._watchers.get(key)
|
|
150
210
|
if current and current.alive:
|
|
@@ -156,7 +216,8 @@ class ProjectMonitor:
|
|
|
156
216
|
lambda error, k=key: self._watch_error(k, error),
|
|
157
217
|
)
|
|
158
218
|
self._watchers[key] = watcher
|
|
159
|
-
|
|
219
|
+
if project.get("watch_backend") != "windows_events" or project.get("watch_error"):
|
|
220
|
+
project_store.update(key, watch_backend="windows_events", watch_error="")
|
|
160
221
|
watcher.start()
|
|
161
222
|
|
|
162
223
|
def _job_gap(self, key, now):
|
|
@@ -172,15 +233,17 @@ class ProjectMonitor:
|
|
|
172
233
|
|
|
173
234
|
def tick(self):
|
|
174
235
|
now = time.monotonic()
|
|
236
|
+
self._flush_notes(now)
|
|
175
237
|
projects = project_store.all_projects()
|
|
176
238
|
self._sync_watchers(projects, now)
|
|
177
239
|
busy = project_store.projects_with_pending_job()
|
|
178
240
|
for project in projects:
|
|
179
241
|
key = project["id"]
|
|
180
|
-
if not
|
|
242
|
+
if not monitored(project):
|
|
181
243
|
continue
|
|
182
244
|
if not os.path.isdir(project["root"]):
|
|
183
|
-
|
|
245
|
+
if project.get("status") != "unavailable":
|
|
246
|
+
project_store.update(key, status="unavailable", last_error="The folder is not accessible.")
|
|
184
247
|
continue
|
|
185
248
|
if key in busy:
|
|
186
249
|
continue # Não planejar outra geração com metadados de uma ainda em curso.
|
|
@@ -195,7 +258,7 @@ class ProjectMonitor:
|
|
|
195
258
|
self._git_seen[key] = signature
|
|
196
259
|
self._git_checked[key] = now
|
|
197
260
|
watcher = self._watchers.get(key)
|
|
198
|
-
interval = RECONCILE_WATCHED_S if watcher is not None and watcher.alive else RECONCILE_S
|
|
261
|
+
interval = RECONCILE_WATCHED_S if watcher is not None and watcher.alive or dormant(project) else RECONCILE_S
|
|
199
262
|
if now - self._checked.get(key, -interval) >= interval:
|
|
200
263
|
self._checked[key] = now
|
|
201
264
|
scope = index_scope.load_scope(project["root"])
|
|
@@ -223,31 +286,56 @@ class ProjectMonitor:
|
|
|
223
286
|
with self._lock:
|
|
224
287
|
self._pending.pop(key, None)
|
|
225
288
|
continue
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
self.
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
289
|
+
self._dispatch(key, now)
|
|
290
|
+
|
|
291
|
+
def _dispatch(self, key, now):
|
|
292
|
+
"""Enqueues the update of a project whose changes have been quiet for DEBOUNCE_S and whose job gap has passed."""
|
|
293
|
+
with self._lock:
|
|
294
|
+
pending = self._pending.get(key)
|
|
295
|
+
if not pending or now - pending["last"] < DEBOUNCE_S or now < self._next_job.get(key, 0):
|
|
296
|
+
return
|
|
297
|
+
options = {"changed_paths": sorted(pending["paths"]), "full_check": pending["full"],
|
|
298
|
+
"force_scope": pending["force_scope"]}
|
|
299
|
+
revision = pending.get("revision", 0)
|
|
300
|
+
current = project_store.get(key)
|
|
301
|
+
if time.time() < current.get("next_retry_at", 0):
|
|
302
|
+
return
|
|
303
|
+
try:
|
|
304
|
+
self.enqueue(key, **options)
|
|
305
|
+
except Exception as exc:
|
|
306
|
+
project_store.update(key, status="error", last_error=str(exc)[:300], next_retry_at=time.time() + 60)
|
|
307
|
+
return
|
|
308
|
+
self._next_job[key] = now + self._job_gap(key, now)
|
|
309
|
+
with self._lock:
|
|
310
|
+
# Um evento recebido durante o submit continua pendente para a próxima geração.
|
|
311
|
+
if self._pending.get(key) is pending and pending.get("revision", 0) == revision:
|
|
312
|
+
self._pending.pop(key, None)
|
|
313
|
+
|
|
314
|
+
def _dispatch_quiet(self, now):
|
|
315
|
+
"""Between full passes: sends the projects whose debounce ended, without listing every project of the registry.
|
|
316
|
+
A project that is not eager, is busy or is no longer monitored is left to the next full pass."""
|
|
317
|
+
self._flush_notes(now)
|
|
318
|
+
with self._lock:
|
|
319
|
+
ready = [key for key, pending in self._pending.items()
|
|
320
|
+
if now - pending["last"] >= DEBOUNCE_S and now >= self._next_job.get(key, 0)]
|
|
321
|
+
if not ready:
|
|
322
|
+
return
|
|
323
|
+
busy = project_store.projects_with_pending_job()
|
|
324
|
+
for key in ready:
|
|
325
|
+
project = project_store.get(key)
|
|
326
|
+
if (project and key not in busy and monitored(project) and project.get("update_mode", "on_search") == "eager"):
|
|
327
|
+
self._dispatch(key, now)
|
|
246
328
|
|
|
247
329
|
def _run(self):
|
|
330
|
+
last_pass = None
|
|
248
331
|
while not self._stop.is_set():
|
|
332
|
+
now = time.monotonic()
|
|
249
333
|
try:
|
|
250
|
-
|
|
334
|
+
if last_pass is None or now - last_pass >= IDLE_TICK_S:
|
|
335
|
+
last_pass = now
|
|
336
|
+
self.tick()
|
|
337
|
+
else:
|
|
338
|
+
self._dispatch_quiet(now)
|
|
251
339
|
except Exception:
|
|
252
340
|
pass # Falha opcional do monitor não derruba o daemon MCP.
|
|
253
|
-
self._stop.wait(1 if self._pending else IDLE_TICK_S)
|
|
341
|
+
self._stop.wait(1 if self._pending or self._unsaved else IDLE_TICK_S)
|
package/project_store.py
CHANGED
|
@@ -15,6 +15,8 @@ DB_PATH = os.path.join(paths.DATA_DIR, "projects.sqlite3")
|
|
|
15
15
|
_LOCK = threading.RLock()
|
|
16
16
|
_SCHEMA_READY = set()
|
|
17
17
|
MAX_JOBS = 500
|
|
18
|
+
_GENERATION = 0
|
|
19
|
+
_LISTING = {"key": None, "rows": []}
|
|
18
20
|
|
|
19
21
|
|
|
20
22
|
@contextmanager
|
|
@@ -24,7 +26,7 @@ def _db():
|
|
|
24
26
|
_SCHEMA_READY.discard(DB_PATH)
|
|
25
27
|
db = sqlite3.connect(DB_PATH, timeout=10)
|
|
26
28
|
try:
|
|
27
|
-
db.execute("PRAGMA secure_delete=
|
|
29
|
+
db.execute("PRAGMA secure_delete=OFF")
|
|
28
30
|
if DB_PATH not in _SCHEMA_READY:
|
|
29
31
|
db.execute("CREATE TABLE IF NOT EXISTS projects(id TEXT PRIMARY KEY, root TEXT NOT NULL, data TEXT NOT NULL, updated_at REAL NOT NULL)")
|
|
30
32
|
db.execute("CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, project_id TEXT, status TEXT NOT NULL, updated_at REAL NOT NULL, payload TEXT NOT NULL)")
|
|
@@ -34,10 +36,17 @@ def _db():
|
|
|
34
36
|
_SCHEMA_READY.add(DB_PATH)
|
|
35
37
|
with db:
|
|
36
38
|
yield db
|
|
39
|
+
if db.total_changes:
|
|
40
|
+
_bump_generation()
|
|
37
41
|
finally:
|
|
38
42
|
db.close()
|
|
39
43
|
|
|
40
44
|
|
|
45
|
+
def _bump_generation():
|
|
46
|
+
global _GENERATION
|
|
47
|
+
_GENERATION += 1
|
|
48
|
+
|
|
49
|
+
|
|
41
50
|
CONTAINER_MIN_PROJECTS = 2
|
|
42
51
|
|
|
43
52
|
|
|
@@ -115,9 +124,20 @@ def containing(path):
|
|
|
115
124
|
|
|
116
125
|
|
|
117
126
|
def all_projects():
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
127
|
+
"""Every registered project. The rows are read again only after a write by this process or a change of the
|
|
128
|
+
registry file, because the monitor lists the whole registry every few seconds."""
|
|
129
|
+
with _LOCK:
|
|
130
|
+
try:
|
|
131
|
+
stat = os.stat(DB_PATH)
|
|
132
|
+
key = (DB_PATH, _GENERATION, stat.st_mtime_ns, stat.st_size)
|
|
133
|
+
except OSError:
|
|
134
|
+
key = None
|
|
135
|
+
if key is None or _LISTING["key"] != key:
|
|
136
|
+
with _db() as db:
|
|
137
|
+
rows = [row[0] for row in db.execute("SELECT data FROM projects ORDER BY root COLLATE NOCASE")]
|
|
138
|
+
_LISTING["key"], _LISTING["rows"] = key, rows
|
|
139
|
+
rows = _LISTING["rows"]
|
|
140
|
+
return [json.loads(row) for row in rows]
|
|
121
141
|
|
|
122
142
|
|
|
123
143
|
def update(project_id, **fields):
|
package/scratch.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Where Smart Tool's tests, probes and evaluation scripts keep their temporary folders.
|
|
2
|
+
|
|
3
|
+
Every run lives in `%LOCALAPPDATA%\\tmp\\smart-tool\\<run>`, never in the root of the system's temporary folder, is
|
|
4
|
+
removed when it ends, and a run that was killed is removed by the next one once it is older than `MAX_AGE_S`.
|
|
5
|
+
`SMART_TOOL_SCRATCH` replaces the base folder. Pure stdlib so any script can import it."""
|
|
6
|
+
import contextlib
|
|
7
|
+
import os
|
|
8
|
+
import shutil
|
|
9
|
+
import tempfile
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
MAX_AGE_S = 24 * 3600
|
|
13
|
+
ENV_BASE = "SMART_TOOL_SCRATCH"
|
|
14
|
+
_swept = set()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def base():
|
|
18
|
+
"""The folder that holds every run of this tool, created and swept of its own leftovers once per process.
|
|
19
|
+
|
|
20
|
+
Returns: str, the absolute path. Raises RuntimeError when neither SMART_TOOL_SCRATCH nor LOCALAPPDATA is set."""
|
|
21
|
+
folder = os.environ.get(ENV_BASE)
|
|
22
|
+
if not folder:
|
|
23
|
+
local = os.environ.get("LOCALAPPDATA")
|
|
24
|
+
if not local:
|
|
25
|
+
raise RuntimeError(f"Set {ENV_BASE} or LOCALAPPDATA: there is no per-user folder for temporary runs.")
|
|
26
|
+
folder = os.path.join(local, "tmp", "smart-tool")
|
|
27
|
+
folder = os.path.abspath(folder)
|
|
28
|
+
os.makedirs(folder, exist_ok=True)
|
|
29
|
+
if folder not in _swept:
|
|
30
|
+
_swept.add(folder)
|
|
31
|
+
sweep(folder)
|
|
32
|
+
return folder
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def remove(path):
|
|
36
|
+
"""Deletes a folder tree including read-only files (Git objects), retrying while another process lets go of it.
|
|
37
|
+
|
|
38
|
+
Raises OSError when the folder is still there after the retries."""
|
|
39
|
+
def writable(function, target, _error):
|
|
40
|
+
os.chmod(target, 0o700)
|
|
41
|
+
function(target)
|
|
42
|
+
|
|
43
|
+
for attempt in range(5):
|
|
44
|
+
if os.path.isdir(path):
|
|
45
|
+
shutil.rmtree(path, onerror=writable)
|
|
46
|
+
if not os.path.exists(path):
|
|
47
|
+
return
|
|
48
|
+
time.sleep(attempt + 1)
|
|
49
|
+
raise OSError(f"Could not remove {path}")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def sweep(folder, max_age_s=MAX_AGE_S):
|
|
53
|
+
"""Deletes the runs directly under folder that were last touched more than max_age_s ago; a run that is still in
|
|
54
|
+
use is left for the next sweep.
|
|
55
|
+
|
|
56
|
+
Returns: int, how many runs were deleted."""
|
|
57
|
+
cutoff, deleted = time.time() - max_age_s, 0
|
|
58
|
+
for entry in os.scandir(folder):
|
|
59
|
+
if entry.stat().st_mtime >= cutoff:
|
|
60
|
+
continue
|
|
61
|
+
try:
|
|
62
|
+
if entry.is_dir():
|
|
63
|
+
remove(entry.path)
|
|
64
|
+
else:
|
|
65
|
+
os.remove(entry.path)
|
|
66
|
+
deleted += 1
|
|
67
|
+
except OSError:
|
|
68
|
+
continue
|
|
69
|
+
return deleted
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def make(prefix):
|
|
73
|
+
"""Creates an empty run folder named prefix plus a random suffix under base() and returns its path."""
|
|
74
|
+
return tempfile.mkdtemp(prefix=prefix, dir=base())
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@contextlib.contextmanager
|
|
78
|
+
def run(prefix):
|
|
79
|
+
"""A run folder that is removed when the block ends, whether it succeeded or failed."""
|
|
80
|
+
path = make(prefix)
|
|
81
|
+
try:
|
|
82
|
+
yield path
|
|
83
|
+
finally:
|
|
84
|
+
remove(path)
|
package/version.py
CHANGED