@allansantos-dev/smart-tool 0.9.19 → 0.9.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +1 -1
- package/affected_tests.py +38 -8
- package/browser_control.py +2 -2
- package/client_hooks.py +1 -1
- package/endpoint_sync.py +7 -2
- package/indexer.py +30 -1
- package/package.json +1 -1
- package/setup_audit.py +1 -1
- package/smart_tool_daemon.py +48 -2
- package/version.py +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.20 - beta
|
|
4
|
+
|
|
5
|
+
- The first search of a folder that has no index yet no longer waits for the index. It answers at once with the files
|
|
6
|
+
that hold a query term, read from disk and ranked by keyword (up to 8 s of scanning, and it says how many files it
|
|
7
|
+
reached), and builds the index in the background; the answer says "Indexing in progress: this folder has no index
|
|
8
|
+
yet". A folder with no saved scope and no model to decide it still asks the agent for the scope, as before.
|
|
9
|
+
- `smart_search_result` hands the agent the result text itself when the job is done, the same text a search that
|
|
10
|
+
finishes within the wait returns. Before, it was a JSON wrapper that escaped the text and repeated the index stats
|
|
11
|
+
and the git view (~1.5 thousand characters of metadata per answer). A job still pending or failed comes without
|
|
12
|
+
those fields too.
|
|
13
|
+
- Playwright MCP starts with `--caps=` so a `PLAYWRIGHT_MCP_CAPS=network` in the user's environment cannot turn on
|
|
14
|
+
`browser_route` (measured on 0.0.83: with the variable set, 29 tools including `browser_route`; with `--caps=`, 25
|
|
15
|
+
and no `browser_route`).
|
|
16
|
+
- `CODEX_HOME` is followed everywhere the Codex config is read or written: the MCP registration check, the endpoint
|
|
17
|
+
sync after a port change and the Codex hook file (before, only the browser control and the setup audit followed it).
|
|
18
|
+
- `affected_tests` links a Python test to the modules it imports inside a string, as in
|
|
19
|
+
`subprocess.run([sys.executable, "-c", "import click"])`, so a change to that module selects the test.
|
|
20
|
+
|
|
3
21
|
## 0.9.19 - beta
|
|
4
22
|
|
|
5
23
|
- A search no longer waits for the index to catch up. It answers from the existing index at once, searches the files
|
package/README.md
CHANGED
|
@@ -15,7 +15,7 @@ registered as the `playwright` server where this machine lets a browser be drive
|
|
|
15
15
|
An optional hook routes the agent's native Grep/Glob/Read/Bash/WebSearch/WebFetch calls to these tools when they
|
|
16
16
|
are cheaper, or just tells the agent what Smart Tool would do.
|
|
17
17
|
|
|
18
|
-
**Status:** 0.9.
|
|
18
|
+
**Status:** 0.9.20 beta. Windows only. What changed in each version: [CHANGELOG.md](CHANGELOG.md).
|
|
19
19
|
|
|
20
20
|
## How it works
|
|
21
21
|
|
package/affected_tests.py
CHANGED
|
@@ -33,6 +33,8 @@ NOTE = ("Selection from the indexed import graph: tests that load code through s
|
|
|
33
33
|
"files and imports count before reindexing; pytest fixtures link a test to the conftest.py that defines them.")
|
|
34
34
|
TEST_SOURCES = (".py", ".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".mts", ".cts", ".java")
|
|
35
35
|
_PATH_LITERAL = re.compile(r"""['"`]((?:\.{0,2}/)?[\w.-]+(?:/[\w.-]+)*\.(?:py|js|mjs|cjs|ts|mts|cts|java))['"`]""")
|
|
36
|
+
_STRING_LITERAL = re.compile(r"""'([^'\n]*)'|"([^"\n]*)\"""")
|
|
37
|
+
_IMPORT_STATEMENT = re.compile(r"(?:^|;)\s*(?:import|from)\s+([A-Za-z_]\w*(?:\.\w+)*)")
|
|
36
38
|
_HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@", re.M)
|
|
37
39
|
|
|
38
40
|
|
|
@@ -213,12 +215,7 @@ def _path_literal_users(root, test_files, known):
|
|
|
213
215
|
if not test.lower().endswith(TEST_SOURCES):
|
|
214
216
|
continue
|
|
215
217
|
source = _read(os.path.join(root, test))
|
|
216
|
-
folders
|
|
217
|
-
while True:
|
|
218
|
-
folders.append(folder)
|
|
219
|
-
if not folder:
|
|
220
|
-
break
|
|
221
|
-
folder = os.path.dirname(folder)
|
|
218
|
+
folders = _folders_above(test)
|
|
222
219
|
for literal in set(_PATH_LITERAL.findall(source)):
|
|
223
220
|
for base in folders:
|
|
224
221
|
candidate = posixpath.normpath(posixpath.join(base, literal.removeprefix("./")))
|
|
@@ -228,6 +225,37 @@ def _path_literal_users(root, test_files, known):
|
|
|
228
225
|
return users
|
|
229
226
|
|
|
230
227
|
|
|
228
|
+
def _import_string_users(root, test_files, known):
|
|
229
|
+
"""Python modules a test imports inside a string (subprocess.run([sys.executable, "-c", "import click"])), resolved
|
|
230
|
+
to the package or module file under the project root, a `src` folder or a folder above the test: the code runs in
|
|
231
|
+
another process, so no import statement of the test links it."""
|
|
232
|
+
users = defaultdict(set)
|
|
233
|
+
for test in test_files:
|
|
234
|
+
if not test.lower().endswith(".py"):
|
|
235
|
+
continue
|
|
236
|
+
folders = ["src", *_folders_above(test)]
|
|
237
|
+
literals = (single or double for single, double in _STRING_LITERAL.findall(_read(os.path.join(root, test))))
|
|
238
|
+
for module in {name for text in literals for name in _IMPORT_STATEMENT.findall(text)}:
|
|
239
|
+
stem = module.replace(".", "/")
|
|
240
|
+
for base in folders:
|
|
241
|
+
target = next((candidate for candidate in (posixpath.join(base, stem + ".py"),
|
|
242
|
+
posixpath.join(base, stem, "__init__.py"))
|
|
243
|
+
if candidate in known and candidate != test), None)
|
|
244
|
+
if target:
|
|
245
|
+
users[target].add(test)
|
|
246
|
+
break
|
|
247
|
+
return users
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _folders_above(path):
|
|
251
|
+
"""The folder of `path` and every folder above it up to the project root (`""`), nearest first."""
|
|
252
|
+
folders, folder = [], os.path.dirname(path)
|
|
253
|
+
while folder:
|
|
254
|
+
folders.append(folder)
|
|
255
|
+
folder = os.path.dirname(folder)
|
|
256
|
+
return folders + [""]
|
|
257
|
+
|
|
258
|
+
|
|
231
259
|
_TEST_CASES = {
|
|
232
260
|
".py": re.compile(r"^\s*(async\s+)?def\s+test\w*\s*\(|^\s*class\s+Test\w*|unittest\.TestCase", re.M),
|
|
233
261
|
".java": re.compile(r"@(Test|ParameterizedTest|RepeatedTest|TestFactory|TestTemplate|RunWith|Suite)\b|"
|
|
@@ -296,8 +324,10 @@ def affected(root, base="HEAD", limit=30, view_id=None):
|
|
|
296
324
|
callers[call["target"]].add(call["source"])
|
|
297
325
|
for conftest, users in _fixture_users(root, test_files).items():
|
|
298
326
|
importers[conftest] |= users
|
|
299
|
-
|
|
300
|
-
|
|
327
|
+
known_files = {f["path"] for f in data.get("files") or []}
|
|
328
|
+
for linked in (_path_literal_users(root, test_files, known_files), _import_string_users(root, test_files, known_files)):
|
|
329
|
+
for target, users in linked.items():
|
|
330
|
+
importers[target] |= users
|
|
301
331
|
by_id = {s["id"]: s for s in data.get("symbols") or []}
|
|
302
332
|
run_all, not_analyzed, changed_code, changed_tests, conftests = [], [], [], [], []
|
|
303
333
|
for path, change in sorted(changes.items()):
|
package/browser_control.py
CHANGED
|
@@ -62,7 +62,7 @@ def server_command():
|
|
|
62
62
|
raise RuntimeError("Supported Node.js not found for Playwright MCP.")
|
|
63
63
|
if not CLI.is_file():
|
|
64
64
|
raise RuntimeError(f"Playwright MCP is not installed ({CLI}); reinstall Smart Tool.")
|
|
65
|
-
return [node, str(CLI), "--isolated", "--headless", "--no-webmcp", "--config", write_config(),
|
|
65
|
+
return [node, str(CLI), "--isolated", "--headless", "--no-webmcp", "--caps=", "--config", write_config(),
|
|
66
66
|
"--executable-path", chromium_executable(), "--output-dir", OUTPUT_DIR,
|
|
67
67
|
"--output-max-size", str(OUTPUT_MAX_BYTES)]
|
|
68
68
|
|
|
@@ -159,7 +159,7 @@ def registered():
|
|
|
159
159
|
|
|
160
160
|
def _codex_config():
|
|
161
161
|
"""Codex keeps config.toml in CODEX_HOME when set, else ~/.codex."""
|
|
162
|
-
return os.path.join(
|
|
162
|
+
return os.path.join(endpoint_sync.codex_dir(), "config.toml")
|
|
163
163
|
|
|
164
164
|
|
|
165
165
|
def _replace_text(path, text):
|
package/client_hooks.py
CHANGED
|
@@ -21,7 +21,7 @@ PREVIEW_TTL_S = 600
|
|
|
21
21
|
CLIENTS = {
|
|
22
22
|
"claude": {"label": "Claude Code", "names": {"claude-code"}, "files": [os.path.join(endpoint_sync.claude_dir(), "settings.json")],
|
|
23
23
|
"docs": "https://code.claude.com/docs/en/hooks", "event": "PreToolUse", "matcher": "Grep|Glob|Read|Bash|PowerShell|WebSearch|WebFetch|Edit|MultiEdit|Write|mcp__smart-tool__smart_search", "http": True},
|
|
24
|
-
"codex": {"label": "Codex", "prefix": "codex", "files": [os.path.join(
|
|
24
|
+
"codex": {"label": "Codex", "prefix": "codex", "files": [os.path.join(endpoint_sync.codex_dir(), "hooks.json"), os.path.join(endpoint_sync.codex_dir(), "config.toml")],
|
|
25
25
|
"docs": "https://learn.chatgpt.com/docs/hooks", "event": "PreToolUse", "matcher": "Bash|apply_patch",
|
|
26
26
|
"after_install": "Codex only runs new hooks after the user approves the definition in /hooks."},
|
|
27
27
|
"cursor": {"label": "Cursor", "prefix": "cursor", "files": [os.path.join(HOME, ".cursor", "hooks.json")],
|
package/endpoint_sync.py
CHANGED
|
@@ -27,6 +27,11 @@ def claude_dir():
|
|
|
27
27
|
return os.environ.get("CLAUDE_CONFIG_DIR") or _home(".claude")
|
|
28
28
|
|
|
29
29
|
|
|
30
|
+
def codex_dir():
|
|
31
|
+
"""Codex keeps config.toml and hooks.json in CODEX_HOME when set, else ~/.codex."""
|
|
32
|
+
return os.environ.get("CODEX_HOME") or _home(".codex")
|
|
33
|
+
|
|
34
|
+
|
|
30
35
|
def claude_json():
|
|
31
36
|
"""User-scope MCP servers: $CLAUDE_CONFIG_DIR/.claude.json when set, else ~/.claude.json."""
|
|
32
37
|
custom = os.environ.get("CLAUDE_CONFIG_DIR")
|
|
@@ -72,7 +77,7 @@ def _codex_mcp(text, base):
|
|
|
72
77
|
|
|
73
78
|
def targets():
|
|
74
79
|
return ((claude_json(), _claude_mcp), (os.path.join(claude_dir(), "settings.json"), _claude_hook),
|
|
75
|
-
(
|
|
80
|
+
(os.path.join(codex_dir(), "config.toml"), _codex_mcp))
|
|
76
81
|
|
|
77
82
|
|
|
78
83
|
_CODEX_ANY_URL_RE = re.compile(r'\[mcp_servers\.smart-tool\][^\[]*?\burl\s*=\s*"([^"]+)"')
|
|
@@ -93,7 +98,7 @@ def registrations():
|
|
|
93
98
|
except (OSError, AttributeError):
|
|
94
99
|
break
|
|
95
100
|
try:
|
|
96
|
-
with open(
|
|
101
|
+
with open(os.path.join(codex_dir(), "config.toml"), encoding="utf-8") as stream:
|
|
97
102
|
match = _CODEX_ANY_URL_RE.search(stream.read())
|
|
98
103
|
found["codex"] = match.group(1) if match else None
|
|
99
104
|
except OSError:
|
package/indexer.py
CHANGED
|
@@ -843,6 +843,32 @@ def overlay_chunks(root, rel_paths):
|
|
|
843
843
|
return chunks
|
|
844
844
|
|
|
845
845
|
|
|
846
|
+
def keyword_chunks(root, rel_paths, queries, budget_s):
|
|
847
|
+
"""`(chunks, scanned)`: the chunks, read from disk right now, of the files among `rel_paths` that contain at least
|
|
848
|
+
one term of `queries` (the file name counts), for a project that has no index yet. The scan stops after `budget_s`
|
|
849
|
+
seconds; `scanned` is how many files it reached. Reading and skipping rules are the index's (`_stable_read`)."""
|
|
850
|
+
terms = [term.casefold() for term in dict.fromkeys(term for query in queries for term in _query_terms(query))]
|
|
851
|
+
deadline = time.monotonic() + budget_s
|
|
852
|
+
chunks, scanned = [], 0
|
|
853
|
+
for rel in rel_paths:
|
|
854
|
+
if time.monotonic() > deadline:
|
|
855
|
+
break
|
|
856
|
+
scanned += 1
|
|
857
|
+
data, encoding, _stat = _stable_read(root, rel)
|
|
858
|
+
if data is None:
|
|
859
|
+
continue
|
|
860
|
+
try:
|
|
861
|
+
text = data.decode(encoding or "utf-8", "ignore").casefold()
|
|
862
|
+
except LookupError:
|
|
863
|
+
continue
|
|
864
|
+
name = rel.casefold()
|
|
865
|
+
if not any(term in text or term in name for term in terms):
|
|
866
|
+
continue
|
|
867
|
+
pieces = chunk_file(root, rel, CHUNK_LINES, data=data, encoding=encoding)
|
|
868
|
+
chunks.extend((rel, start, end, piece) for start, end, piece in pieces or [])
|
|
869
|
+
return chunks, scanned
|
|
870
|
+
|
|
871
|
+
|
|
846
872
|
def _overlay_scored(chunks, query, alpha):
|
|
847
873
|
"""`[(score, path, start, end, text)]` for overlay chunks matching the query terms by keyword. A chunk that holds
|
|
848
874
|
every term scores 1.0, the top of the fused index scores (`search` ranks it first on a tie): the edited file is read
|
|
@@ -874,7 +900,10 @@ def search(root, query_vector=None, top_k=20, query=None, model_id=None, include
|
|
|
874
900
|
files) are ignored and the overlay chunks, read from disk, take part by keyword instead."""
|
|
875
901
|
path = existing_db_path(root)
|
|
876
902
|
if not path:
|
|
877
|
-
|
|
903
|
+
if not (overlay and overlay["chunks"] and query):
|
|
904
|
+
return []
|
|
905
|
+
extra = _overlay_scored([chunk for chunk in overlay["chunks"] if include is None or include(chunk[0])], query, 0.0)
|
|
906
|
+
return sorted(extra, key=lambda item: item[0], reverse=True)[:top_k]
|
|
878
907
|
kind_filter = include
|
|
879
908
|
if overlay and overlay["drop"]:
|
|
880
909
|
dropped = set(overlay["drop"])
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.20",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Allan Santos",
|
package/setup_audit.py
CHANGED
package/smart_tool_daemon.py
CHANGED
|
@@ -176,6 +176,7 @@ _MAX_JOBS_RETAINED = 200
|
|
|
176
176
|
# nearly every call and costs the agent extra smart_search_result round trips. Codex gives an MCP call 60 s by default.
|
|
177
177
|
SMART_SEARCH_SYNC_WAIT_S = 25
|
|
178
178
|
OVERLAY_MAX_FILES = 40
|
|
179
|
+
COLD_SCAN_BUDGET_S = 8
|
|
179
180
|
EDIT_INDEX_DELAY_S = 8
|
|
180
181
|
# smart_search_result waits for the job this long before answering pending: answering at once had an agent poll the
|
|
181
182
|
# same job 7 times in 16 s and then give up on Smart Tool (claude -p session, 2026-10-09; job durations median 6 s,
|
|
@@ -875,6 +876,29 @@ def _index_lag(root, model, cfg, problem, job_id):
|
|
|
875
876
|
"refresh": bool(changed or removed or rescan or _profile_due(scope, scope_model))}
|
|
876
877
|
|
|
877
878
|
|
|
879
|
+
def _cold_index(root, cfg, problem):
|
|
880
|
+
"""Whether this search is the first one of a project with no index, and the index can be built in the background:
|
|
881
|
+
the gateway answers and the scope is saved or a model can decide it. Otherwise the search keeps its old course
|
|
882
|
+
(build first, or ask the agent for the scope)."""
|
|
883
|
+
if indexer.info(root).get("chunks") or problem:
|
|
884
|
+
return False
|
|
885
|
+
return bool(index_scope.valid_scope(index_scope.load_scope(root)) or cfg.get("scope_model") or cfg.get("router_model"))
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def _cold_overlay(root, queries):
|
|
889
|
+
"""`(overlay, warning)` for a project with no index: the in-scope files (the saved scope, else the whole folder minus
|
|
890
|
+
the always-excluded names) holding a query term, read from disk and ranked by keyword, plus the notice that the
|
|
891
|
+
index is being built. The scan stops after `COLD_SCAN_BUDGET_S`."""
|
|
892
|
+
scope = index_scope.load_scope(root)
|
|
893
|
+
files = index_scope.resolve_included_files(root, scope if index_scope.valid_scope(scope) else {"include": [""]})
|
|
894
|
+
chunks, scanned = indexer.keyword_chunks(root, files, queries, COLD_SCAN_BUDGET_S)
|
|
895
|
+
reached = f"{scanned} of {len(files)} files" if scanned < len(files) else f"all {len(files)} files"
|
|
896
|
+
warning = ("Indexing in progress: this folder has no index yet, so the results are keyword matches read straight "
|
|
897
|
+
f"from disk in {reached}, without semantic ranking. The index is being built in the background: repeat "
|
|
898
|
+
"the search in a few minutes for the full ranking.")
|
|
899
|
+
return {"drop": [], "chunks": chunks}, warning
|
|
900
|
+
|
|
901
|
+
|
|
878
902
|
def _lag_overlay(root, lag):
|
|
879
903
|
"""`(overlay, warning)` for `indexer.search`: the stale files read from disk, searched by keyword in place of their
|
|
880
904
|
outdated chunks, and the notice telling the agent that indexing is in progress. `(None, "")` when the index is
|
|
@@ -1086,6 +1110,10 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1086
1110
|
raise
|
|
1087
1111
|
warnings.append("Lexical index not updated; results from the last ready index. " + _project_error(exc))
|
|
1088
1112
|
search_model = None
|
|
1113
|
+
elif _cold_index(root, cfg, problem):
|
|
1114
|
+
overlay, notice = _cold_overlay(root, texts)
|
|
1115
|
+
search_model = None
|
|
1116
|
+
warnings += [text for text in (notice, _refresh_in_background(root, {"refresh": True})) if text]
|
|
1089
1117
|
elif problem and not local_embedder.is_local(model):
|
|
1090
1118
|
if not indexer.info(root).get("chunks"):
|
|
1091
1119
|
raise RuntimeError(problem)
|
|
@@ -1434,13 +1462,13 @@ def _handle_smart_search(arguments):
|
|
|
1434
1462
|
job_id, future = _submit_project_job("smart_search", arguments)
|
|
1435
1463
|
if future is None:
|
|
1436
1464
|
_metric("pending", reason="queued")
|
|
1437
|
-
return
|
|
1465
|
+
return _agent_search_answer(_smart_search_job_payload(job_id))
|
|
1438
1466
|
try:
|
|
1439
1467
|
result = future.result(timeout=SMART_SEARCH_SYNC_WAIT_S)
|
|
1440
1468
|
except concurrent.futures.TimeoutError:
|
|
1441
1469
|
if not future.done():
|
|
1442
1470
|
_metric("pending", reason="sync_timeout")
|
|
1443
|
-
return
|
|
1471
|
+
return _agent_search_answer(_smart_search_job_payload(job_id))
|
|
1444
1472
|
result = future.result()
|
|
1445
1473
|
_mark_notice_delivered(job_id)
|
|
1446
1474
|
return result
|
|
@@ -1546,9 +1574,27 @@ def _handle_smart_search_result(arguments):
|
|
|
1546
1574
|
time.sleep(0.25)
|
|
1547
1575
|
payload = _smart_search_job_payload(job_id)
|
|
1548
1576
|
_mark_notice_delivered(job_id)
|
|
1577
|
+
if payload["kind"] == "smart_search":
|
|
1578
|
+
return _agent_search_answer(payload)
|
|
1549
1579
|
return _to_json_sanitized(payload)
|
|
1550
1580
|
|
|
1551
1581
|
|
|
1582
|
+
_AGENT_SEARCH_DROPPED = ("kind", "project_id", "updated_at", "stats", "view")
|
|
1583
|
+
|
|
1584
|
+
|
|
1585
|
+
def _agent_search_answer(payload):
|
|
1586
|
+
"""What smart_search_result hands the agent for a smart_search job: the result text itself when done (the same text
|
|
1587
|
+
a search that finishes within the wait returns, no JSON wrapper that escapes it and repeats the index stats and
|
|
1588
|
+
the git view the setup page shows), otherwise the payload without those fields."""
|
|
1589
|
+
if payload["status"] == "done":
|
|
1590
|
+
warning = payload.get("warning")
|
|
1591
|
+
return f"{payload['result']}\nwarning: {warning}" if warning else payload["result"]
|
|
1592
|
+
slim = {key: value for key, value in payload.items() if key not in _AGENT_SEARCH_DROPPED}
|
|
1593
|
+
if not slim.get("cancel_requested"):
|
|
1594
|
+
slim.pop("cancel_requested", None)
|
|
1595
|
+
return _to_json_sanitized(slim)
|
|
1596
|
+
|
|
1597
|
+
|
|
1552
1598
|
def _project_payload(project):
|
|
1553
1599
|
data = dict(project)
|
|
1554
1600
|
data['dismissed_duplicates'] = len(data.get('dismissed_duplicates') or {})
|
package/version.py
CHANGED