@allansantos-dev/smart-tool 0.9.18 → 0.9.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/README.md +1 -1
- package/affected_tests.py +38 -8
- package/agents/smart-scout.md +13 -0
- package/browser_control.py +2 -2
- package/client_hooks.py +1 -1
- package/endpoint_sync.py +7 -2
- package/hook_decision.py +2 -4
- package/indexer.py +107 -5
- package/package.json +1 -1
- package/project_store.py +9 -2
- package/setup_audit.py +1 -1
- package/smart_tool_daemon.py +223 -24
- package/version.py +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,43 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.20 - beta
|
|
4
|
+
|
|
5
|
+
- The first search of a folder that has no index yet no longer waits for the index. It answers at once with the files
|
|
6
|
+
that hold a query term, read from disk and ranked by keyword (up to 8 s of scanning, and it says how many files it
|
|
7
|
+
reached), and builds the index in the background; the answer says "Indexing in progress: this folder has no index
|
|
8
|
+
yet". A folder with no saved scope and no model to decide it still asks the agent for the scope, as before.
|
|
9
|
+
- `smart_search_result` hands the agent the result text itself when the job is done, the same text a search that
|
|
10
|
+
finishes within the wait returns. Before, it was a JSON wrapper that escaped the text and repeated the index stats
|
|
11
|
+
and the git view (~1.5 thousand characters of metadata per answer). A job still pending or failed comes without
|
|
12
|
+
those fields too.
|
|
13
|
+
- Playwright MCP starts with `--caps=` so a `PLAYWRIGHT_MCP_CAPS=network` in the user's environment cannot turn on
|
|
14
|
+
`browser_route` (measured on 0.0.83: with the variable set, 29 tools including `browser_route`; with `--caps=`, 25
|
|
15
|
+
and no `browser_route`).
|
|
16
|
+
- `CODEX_HOME` is followed everywhere the Codex config is read or written: the MCP registration check, the endpoint
|
|
17
|
+
sync after a port change and the Codex hook file (before, only the browser control and the setup audit followed it).
|
|
18
|
+
- `affected_tests` links a Python test to the modules it imports inside a string, as in
|
|
19
|
+
`subprocess.run([sys.executable, "-c", "import click"])`, so a change to that module selects the test.
|
|
20
|
+
|
|
21
|
+
## 0.9.19 - beta
|
|
22
|
+
|
|
23
|
+
- A search no longer waits for the index to catch up. It answers from the existing index at once, searches the files
|
|
24
|
+
edited since the last index by keyword straight from disk, and says "Indexing in progress: N file(s) changed..." in
|
|
25
|
+
the answer while the update runs in the background. An index that does not exist yet, or was built by another model,
|
|
26
|
+
is still built first.
|
|
27
|
+
- A search waits up to 25 s for its own result before answering with a `job_id` (it was 3 s, below the 3.3 s a warm
|
|
28
|
+
search takes at best, so nearly every call needed a second `smart_search_result`).
|
|
29
|
+
- The scope and the language profile are no longer asked of the model inside a search that has an index: the saved ones
|
|
30
|
+
are used and refreshed in the background.
|
|
31
|
+
- New folders are registered watched and eager: the index follows edits by itself. A PreToolUse Edit/Write of an eager
|
|
32
|
+
folder without a running watcher queues one debounced update with the edited paths. Folders already registered keep
|
|
33
|
+
their policy.
|
|
34
|
+
- A `top_k` whose total is above 20 is scaled down in proportion instead of rejecting the call (the rejection cost the
|
|
35
|
+
agent a turn on its first search).
|
|
36
|
+
- A search made right after an edit keeps the just-edited chunk in the answer: the reranker can no longer push it out.
|
|
37
|
+
- An edit whose path comes in the Windows 8.3 short form (`ALLANS~1`-style folder names) is matched to its registered
|
|
38
|
+
project; before, the project lookup compared it with the long form and found nothing.
|
|
39
|
+
- The daemon log prints the seconds spent in each phase of a job (`phases=queued:0.02,auth:0.01,...`).
|
|
40
|
+
|
|
3
41
|
## 0.9.18 - beta
|
|
4
42
|
|
|
5
43
|
- Updates are no longer applied only at logon. The tray checks npm at start and every 24 h and installs a newer
|
package/README.md
CHANGED
|
@@ -15,7 +15,7 @@ registered as the `playwright` server where this machine lets a browser be drive
|
|
|
15
15
|
An optional hook routes the agent's native Grep/Glob/Read/Bash/WebSearch/WebFetch calls to these tools when they
|
|
16
16
|
are cheaper, or just tells the agent what Smart Tool would do.
|
|
17
17
|
|
|
18
|
-
**Status:** 0.9.
|
|
18
|
+
**Status:** 0.9.20 beta. Windows only. What changed in each version: [CHANGELOG.md](CHANGELOG.md).
|
|
19
19
|
|
|
20
20
|
## How it works
|
|
21
21
|
|
package/affected_tests.py
CHANGED
|
@@ -33,6 +33,8 @@ NOTE = ("Selection from the indexed import graph: tests that load code through s
|
|
|
33
33
|
"files and imports count before reindexing; pytest fixtures link a test to the conftest.py that defines them.")
|
|
34
34
|
TEST_SOURCES = (".py", ".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".mts", ".cts", ".java")
|
|
35
35
|
_PATH_LITERAL = re.compile(r"""['"`]((?:\.{0,2}/)?[\w.-]+(?:/[\w.-]+)*\.(?:py|js|mjs|cjs|ts|mts|cts|java))['"`]""")
|
|
36
|
+
_STRING_LITERAL = re.compile(r"""'([^'\n]*)'|"([^"\n]*)\"""")
|
|
37
|
+
_IMPORT_STATEMENT = re.compile(r"(?:^|;)\s*(?:import|from)\s+([A-Za-z_]\w*(?:\.\w+)*)")
|
|
36
38
|
_HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@", re.M)
|
|
37
39
|
|
|
38
40
|
|
|
@@ -213,12 +215,7 @@ def _path_literal_users(root, test_files, known):
|
|
|
213
215
|
if not test.lower().endswith(TEST_SOURCES):
|
|
214
216
|
continue
|
|
215
217
|
source = _read(os.path.join(root, test))
|
|
216
|
-
folders
|
|
217
|
-
while True:
|
|
218
|
-
folders.append(folder)
|
|
219
|
-
if not folder:
|
|
220
|
-
break
|
|
221
|
-
folder = os.path.dirname(folder)
|
|
218
|
+
folders = _folders_above(test)
|
|
222
219
|
for literal in set(_PATH_LITERAL.findall(source)):
|
|
223
220
|
for base in folders:
|
|
224
221
|
candidate = posixpath.normpath(posixpath.join(base, literal.removeprefix("./")))
|
|
@@ -228,6 +225,37 @@ def _path_literal_users(root, test_files, known):
|
|
|
228
225
|
return users
|
|
229
226
|
|
|
230
227
|
|
|
228
|
+
def _import_string_users(root, test_files, known):
|
|
229
|
+
"""Python modules a test imports inside a string (subprocess.run([sys.executable, "-c", "import click"])), resolved
|
|
230
|
+
to the package or module file under the project root, a `src` folder or a folder above the test: the code runs in
|
|
231
|
+
another process, so no import statement of the test links it."""
|
|
232
|
+
users = defaultdict(set)
|
|
233
|
+
for test in test_files:
|
|
234
|
+
if not test.lower().endswith(".py"):
|
|
235
|
+
continue
|
|
236
|
+
folders = ["src", *_folders_above(test)]
|
|
237
|
+
literals = (single or double for single, double in _STRING_LITERAL.findall(_read(os.path.join(root, test))))
|
|
238
|
+
for module in {name for text in literals for name in _IMPORT_STATEMENT.findall(text)}:
|
|
239
|
+
stem = module.replace(".", "/")
|
|
240
|
+
for base in folders:
|
|
241
|
+
target = next((candidate for candidate in (posixpath.join(base, stem + ".py"),
|
|
242
|
+
posixpath.join(base, stem, "__init__.py"))
|
|
243
|
+
if candidate in known and candidate != test), None)
|
|
244
|
+
if target:
|
|
245
|
+
users[target].add(test)
|
|
246
|
+
break
|
|
247
|
+
return users
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _folders_above(path):
|
|
251
|
+
"""The folder of `path` and every folder above it up to the project root (`""`), nearest first."""
|
|
252
|
+
folders, folder = [], os.path.dirname(path)
|
|
253
|
+
while folder:
|
|
254
|
+
folders.append(folder)
|
|
255
|
+
folder = os.path.dirname(folder)
|
|
256
|
+
return folders + [""]
|
|
257
|
+
|
|
258
|
+
|
|
231
259
|
_TEST_CASES = {
|
|
232
260
|
".py": re.compile(r"^\s*(async\s+)?def\s+test\w*\s*\(|^\s*class\s+Test\w*|unittest\.TestCase", re.M),
|
|
233
261
|
".java": re.compile(r"@(Test|ParameterizedTest|RepeatedTest|TestFactory|TestTemplate|RunWith|Suite)\b|"
|
|
@@ -296,8 +324,10 @@ def affected(root, base="HEAD", limit=30, view_id=None):
|
|
|
296
324
|
callers[call["target"]].add(call["source"])
|
|
297
325
|
for conftest, users in _fixture_users(root, test_files).items():
|
|
298
326
|
importers[conftest] |= users
|
|
299
|
-
|
|
300
|
-
|
|
327
|
+
known_files = {f["path"] for f in data.get("files") or []}
|
|
328
|
+
for linked in (_path_literal_users(root, test_files, known_files), _import_string_users(root, test_files, known_files)):
|
|
329
|
+
for target, users in linked.items():
|
|
330
|
+
importers[target] |= users
|
|
301
331
|
by_id = {s["id"]: s for s in data.get("symbols") or []}
|
|
302
332
|
run_all, not_analyzed, changed_code, changed_tests, conftests = [], [], [], [], []
|
|
303
333
|
for path, change in sorted(changes.items()):
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: smart-scout
|
|
3
|
+
description: Optional helper for the Smart Tool. Use it only for a task that chains several Smart Tool calls (search, then graph, then affected_tests) or while the index is still loading. A plain search is faster called directly.
|
|
4
|
+
model: haiku
|
|
5
|
+
tools: mcp__smart-tool__smart_search, mcp__smart-tool__smart_search_result, mcp__smart-tool__project_manage, Read
|
|
6
|
+
---
|
|
7
|
+
You run Smart Tool calls for another session and return only the result.
|
|
8
|
+
|
|
9
|
+
Use smart_search with project_root equal to the working directory (forward slashes), query_identifiers in English identifier terms, query_comments in the language of the comments, and top_k {"code": 6, "test": 2, "doc": 2}. When it answers pending, call smart_search_result with the job_id until it finishes. project_manage action=graph (symbol) and action=affected_tests are yours to chain when the task asks for them. Open at most 3 files with Read and make at most 8 tool calls in total.
|
|
10
|
+
|
|
11
|
+
Reply ONLY with a list of up to 8 items, each `path:first_line-last_line: reason in up to 15 words`, no code excerpts and no introduction.
|
|
12
|
+
|
|
13
|
+
If nothing relevant came back after two different queries, reply exactly `EMPTY: <the two queries you tried>`. Never invent a path. Never use Grep, shell search or Read of folders to compensate: an empty answer is a valid answer, and the session that called you decides whether to retry with smart_search or to report that the code was not found.
|
package/browser_control.py
CHANGED
|
@@ -62,7 +62,7 @@ def server_command():
|
|
|
62
62
|
raise RuntimeError("Supported Node.js not found for Playwright MCP.")
|
|
63
63
|
if not CLI.is_file():
|
|
64
64
|
raise RuntimeError(f"Playwright MCP is not installed ({CLI}); reinstall Smart Tool.")
|
|
65
|
-
return [node, str(CLI), "--isolated", "--headless", "--no-webmcp", "--config", write_config(),
|
|
65
|
+
return [node, str(CLI), "--isolated", "--headless", "--no-webmcp", "--caps=", "--config", write_config(),
|
|
66
66
|
"--executable-path", chromium_executable(), "--output-dir", OUTPUT_DIR,
|
|
67
67
|
"--output-max-size", str(OUTPUT_MAX_BYTES)]
|
|
68
68
|
|
|
@@ -159,7 +159,7 @@ def registered():
|
|
|
159
159
|
|
|
160
160
|
def _codex_config():
|
|
161
161
|
"""Codex keeps config.toml in CODEX_HOME when set, else ~/.codex."""
|
|
162
|
-
return os.path.join(
|
|
162
|
+
return os.path.join(endpoint_sync.codex_dir(), "config.toml")
|
|
163
163
|
|
|
164
164
|
|
|
165
165
|
def _replace_text(path, text):
|
package/client_hooks.py
CHANGED
|
@@ -21,7 +21,7 @@ PREVIEW_TTL_S = 600
|
|
|
21
21
|
CLIENTS = {
|
|
22
22
|
"claude": {"label": "Claude Code", "names": {"claude-code"}, "files": [os.path.join(endpoint_sync.claude_dir(), "settings.json")],
|
|
23
23
|
"docs": "https://code.claude.com/docs/en/hooks", "event": "PreToolUse", "matcher": "Grep|Glob|Read|Bash|PowerShell|WebSearch|WebFetch|Edit|MultiEdit|Write|mcp__smart-tool__smart_search", "http": True},
|
|
24
|
-
"codex": {"label": "Codex", "prefix": "codex", "files": [os.path.join(
|
|
24
|
+
"codex": {"label": "Codex", "prefix": "codex", "files": [os.path.join(endpoint_sync.codex_dir(), "hooks.json"), os.path.join(endpoint_sync.codex_dir(), "config.toml")],
|
|
25
25
|
"docs": "https://learn.chatgpt.com/docs/hooks", "event": "PreToolUse", "matcher": "Bash|apply_patch",
|
|
26
26
|
"after_install": "Codex only runs new hooks after the user approves the definition in /hooks."},
|
|
27
27
|
"cursor": {"label": "Cursor", "prefix": "cursor", "files": [os.path.join(HOME, ".cursor", "hooks.json")],
|
package/endpoint_sync.py
CHANGED
|
@@ -27,6 +27,11 @@ def claude_dir():
|
|
|
27
27
|
return os.environ.get("CLAUDE_CONFIG_DIR") or _home(".claude")
|
|
28
28
|
|
|
29
29
|
|
|
30
|
+
def codex_dir():
|
|
31
|
+
"""Codex keeps config.toml and hooks.json in CODEX_HOME when set, else ~/.codex."""
|
|
32
|
+
return os.environ.get("CODEX_HOME") or _home(".codex")
|
|
33
|
+
|
|
34
|
+
|
|
30
35
|
def claude_json():
|
|
31
36
|
"""User-scope MCP servers: $CLAUDE_CONFIG_DIR/.claude.json when set, else ~/.claude.json."""
|
|
32
37
|
custom = os.environ.get("CLAUDE_CONFIG_DIR")
|
|
@@ -72,7 +77,7 @@ def _codex_mcp(text, base):
|
|
|
72
77
|
|
|
73
78
|
def targets():
|
|
74
79
|
return ((claude_json(), _claude_mcp), (os.path.join(claude_dir(), "settings.json"), _claude_hook),
|
|
75
|
-
(
|
|
80
|
+
(os.path.join(codex_dir(), "config.toml"), _codex_mcp))
|
|
76
81
|
|
|
77
82
|
|
|
78
83
|
_CODEX_ANY_URL_RE = re.compile(r'\[mcp_servers\.smart-tool\][^\[]*?\burl\s*=\s*"([^"]+)"')
|
|
@@ -93,7 +98,7 @@ def registrations():
|
|
|
93
98
|
except (OSError, AttributeError):
|
|
94
99
|
break
|
|
95
100
|
try:
|
|
96
|
-
with open(
|
|
101
|
+
with open(os.path.join(codex_dir(), "config.toml"), encoding="utf-8") as stream:
|
|
97
102
|
match = _CODEX_ANY_URL_RE.search(stream.read())
|
|
98
103
|
found["codex"] = match.group(1) if match else None
|
|
99
104
|
except OSError:
|
package/hook_decision.py
CHANGED
|
@@ -171,10 +171,8 @@ def advise(reason):
|
|
|
171
171
|
|
|
172
172
|
|
|
173
173
|
def _project_of(path):
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
if target.startswith(os.path.normcase(os.path.abspath(p["root"])) + os.sep)]
|
|
177
|
-
return max(roots, key=len) if roots else None
|
|
174
|
+
project = project_store.containing(path)
|
|
175
|
+
return project["root"] if project else None
|
|
178
176
|
|
|
179
177
|
|
|
180
178
|
def _impact_line(root, rel, name):
|
package/indexer.py
CHANGED
|
@@ -779,9 +779,14 @@ def _vector_rows(root, path):
|
|
|
779
779
|
return rows, meta, norms
|
|
780
780
|
|
|
781
781
|
|
|
782
|
+
def _query_terms(query):
|
|
783
|
+
"""Up to 24 distinct word terms of the query, in order."""
|
|
784
|
+
return list(dict.fromkeys(re.findall(r"[\w]{2,}", query or "", re.UNICODE)))[:24]
|
|
785
|
+
|
|
786
|
+
|
|
782
787
|
def _lexical_scored(path, query, rows, limit, offset=0):
|
|
783
788
|
"""(chunk_id, score) best first; higher score is a better lexical match."""
|
|
784
|
-
terms =
|
|
789
|
+
terms = _query_terms(query)
|
|
785
790
|
if not terms:
|
|
786
791
|
return []
|
|
787
792
|
expression = " OR ".join('"' + term.replace('"', '""') + '"' for term in terms)
|
|
@@ -808,11 +813,101 @@ def _min_max(pairs):
|
|
|
808
813
|
return {key: (value - low) / (high - low) if high > low else 1.0 for key, value in pairs}
|
|
809
814
|
|
|
810
815
|
|
|
811
|
-
def
|
|
812
|
-
"""
|
|
816
|
+
def stale_files(root, included_paths):
|
|
817
|
+
"""`(changed, removed)` against the indexed state: `changed` lists in-scope files whose `(mtime_ns, size)` differs
|
|
818
|
+
from the stored one (or that were never indexed), `removed` lists indexed files that left the scope. It only
|
|
819
|
+
stats files, so it is the cheap answer to "what would a reindex have to touch"."""
|
|
820
|
+
snapshot = manifest_snapshot(root)
|
|
821
|
+
changed, removed = [], set(snapshot)
|
|
822
|
+
for rel in sorted(set(included_paths)):
|
|
823
|
+
stat, _reason = _stat_candidate(root, rel)
|
|
824
|
+
if stat is None:
|
|
825
|
+
continue
|
|
826
|
+
removed.discard(rel)
|
|
827
|
+
if snapshot.get(rel) != stat:
|
|
828
|
+
changed.append(rel)
|
|
829
|
+
return changed, sorted(removed)
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def overlay_chunks(root, rel_paths):
|
|
833
|
+
"""`[(path, start_line, end_line, text)]` read from disk right now for the given files, chunked like the index.
|
|
834
|
+
Files that cannot be read consistently are skipped: the overlay only complements the index, it never fails a
|
|
835
|
+
search."""
|
|
836
|
+
chunks = []
|
|
837
|
+
for rel in rel_paths:
|
|
838
|
+
data, encoding, _stat = _stable_read(root, rel)
|
|
839
|
+
if data is None:
|
|
840
|
+
continue
|
|
841
|
+
pieces = chunk_file(root, rel, CHUNK_LINES, data=data, encoding=encoding)
|
|
842
|
+
chunks.extend((rel, start, end, text) for start, end, text in pieces or [])
|
|
843
|
+
return chunks
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def keyword_chunks(root, rel_paths, queries, budget_s):
|
|
847
|
+
"""`(chunks, scanned)`: the chunks, read from disk right now, of the files among `rel_paths` that contain at least
|
|
848
|
+
one term of `queries` (the file name counts), for a project that has no index yet. The scan stops after `budget_s`
|
|
849
|
+
seconds; `scanned` is how many files it reached. Reading and skipping rules are the index's (`_stable_read`)."""
|
|
850
|
+
terms = [term.casefold() for term in dict.fromkeys(term for query in queries for term in _query_terms(query))]
|
|
851
|
+
deadline = time.monotonic() + budget_s
|
|
852
|
+
chunks, scanned = [], 0
|
|
853
|
+
for rel in rel_paths:
|
|
854
|
+
if time.monotonic() > deadline:
|
|
855
|
+
break
|
|
856
|
+
scanned += 1
|
|
857
|
+
data, encoding, _stat = _stable_read(root, rel)
|
|
858
|
+
if data is None:
|
|
859
|
+
continue
|
|
860
|
+
try:
|
|
861
|
+
text = data.decode(encoding or "utf-8", "ignore").casefold()
|
|
862
|
+
except LookupError:
|
|
863
|
+
continue
|
|
864
|
+
name = rel.casefold()
|
|
865
|
+
if not any(term in text or term in name for term in terms):
|
|
866
|
+
continue
|
|
867
|
+
pieces = chunk_file(root, rel, CHUNK_LINES, data=data, encoding=encoding)
|
|
868
|
+
chunks.extend((rel, start, end, piece) for start, end, piece in pieces or [])
|
|
869
|
+
return chunks, scanned
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
def _overlay_scored(chunks, query, alpha):
|
|
873
|
+
"""`[(score, path, start, end, text)]` for overlay chunks matching the query terms by keyword. A chunk that holds
|
|
874
|
+
every term scores 1.0, the top of the fused index scores (`search` ranks it first on a tie): the edited file is read
|
|
875
|
+
from disk, so it is the current answer for an exact match. A partial match stays on the 0-1 scale of the lexical
|
|
876
|
+
share of the fused score."""
|
|
877
|
+
terms = _query_terms(query)
|
|
878
|
+
scored = []
|
|
879
|
+
for path, start, end, text in chunks:
|
|
880
|
+
haystack = (path + " " + text).casefold()
|
|
881
|
+
matched = sum(term.casefold() in haystack for term in terms)
|
|
882
|
+
if matched:
|
|
883
|
+
score = 1.0 if matched == len(terms) else (1 - alpha) * matched / len(terms)
|
|
884
|
+
scored.append((score, path, start, end, text))
|
|
885
|
+
return scored
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def overlay_exact_keys(overlay, query):
|
|
889
|
+
"""`{(path, start, end)}` of the overlay chunks that hold every query term. A reranker must not push them below
|
|
890
|
+
indexed chunks: they come from the file as it is on disk now."""
|
|
891
|
+
if not overlay or not overlay["chunks"] or not query:
|
|
892
|
+
return set()
|
|
893
|
+
return {(path, start, end) for score, path, start, end, _text in _overlay_scored(overlay["chunks"], query, 0.0)
|
|
894
|
+
if score == 1.0}
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
def search(root, query_vector=None, top_k=20, query=None, model_id=None, include=None, overlay=None):
|
|
898
|
+
"""Vector plus lexical retrieval by fusion of relative scores, without network. `overlay` is
|
|
899
|
+
`{"drop": paths, "chunks": overlay_chunks(...)}`: the indexed chunks of the `drop` paths (stale or removed
|
|
900
|
+
files) are ignored and the overlay chunks, read from disk, take part by keyword instead."""
|
|
813
901
|
path = existing_db_path(root)
|
|
814
902
|
if not path:
|
|
815
|
-
|
|
903
|
+
if not (overlay and overlay["chunks"] and query):
|
|
904
|
+
return []
|
|
905
|
+
extra = _overlay_scored([chunk for chunk in overlay["chunks"] if include is None or include(chunk[0])], query, 0.0)
|
|
906
|
+
return sorted(extra, key=lambda item: item[0], reverse=True)[:top_k]
|
|
907
|
+
kind_filter = include
|
|
908
|
+
if overlay and overlay["drop"]:
|
|
909
|
+
dropped = set(overlay["drop"])
|
|
910
|
+
include = lambda rel: rel not in dropped and (kind_filter is None or kind_filter(rel))
|
|
816
911
|
rows, meta, norms = _vector_rows(root, path)
|
|
817
912
|
if include is not None:
|
|
818
913
|
rows = [row for row in rows if include(row[1])]
|
|
@@ -854,7 +949,14 @@ def search(root, query_vector=None, top_k=20, query=None, model_id=None, include
|
|
|
854
949
|
else:
|
|
855
950
|
scores, ids = vector_scores, vector_ids
|
|
856
951
|
by_id = {row[0]: row for row in rows}
|
|
857
|
-
|
|
952
|
+
found = [(scores[chunk_id], *by_id[chunk_id][1:5]) for chunk_id in ids[:top_k]]
|
|
953
|
+
if overlay and overlay["chunks"]:
|
|
954
|
+
extra = _overlay_scored([chunk for chunk in overlay["chunks"] if kind_filter is None or kind_filter(chunk[0])],
|
|
955
|
+
query, FUSION_ALPHA if query_vector is not None else 0.0)
|
|
956
|
+
found = [item for _score, item in sorted(
|
|
957
|
+
[((item[0], 1), item) for item in extra] + [((item[0], 0), item) for item in found],
|
|
958
|
+
key=lambda pair: pair[0], reverse=True)][:top_k]
|
|
959
|
+
return found
|
|
858
960
|
|
|
859
961
|
|
|
860
962
|
def graph_cache_path(view_path):
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.20",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Allan Santos",
|
package/project_store.py
CHANGED
|
@@ -40,8 +40,8 @@ def register(root, watch=None):
|
|
|
40
40
|
row = db.execute("SELECT data FROM projects WHERE id=?", (key,)).fetchone()
|
|
41
41
|
data = json.loads(row[0]) if row else {
|
|
42
42
|
"id": key, "root": display, "name": os.path.basename(display) or display,
|
|
43
|
-
"status": "registered", "watch":
|
|
44
|
-
"update_mode": "
|
|
43
|
+
"status": "registered", "watch": True, "paused": False, "enabled": False,
|
|
44
|
+
"update_mode": "eager",
|
|
45
45
|
"dirty_seq": 0, "indexed_seq": 0, "created_at": time.time(),
|
|
46
46
|
"last_indexed": None, "last_checked": None, "last_error": "",
|
|
47
47
|
"last_job_id": None, "watch_backend": "off", "next_retry_at": 0,
|
|
@@ -60,6 +60,13 @@ def get(project_id):
|
|
|
60
60
|
return json.loads(row[0]) if row else None
|
|
61
61
|
|
|
62
62
|
|
|
63
|
+
def containing(path):
|
|
64
|
+
"""The registered project whose folder holds `path` (the deepest one when projects nest), or `None`."""
|
|
65
|
+
target = project_identity.canonical_root(path)
|
|
66
|
+
inside = [p for p in all_projects() if target.startswith(project_identity.canonical_root(p["root"]) + os.sep)]
|
|
67
|
+
return max(inside, key=lambda p: len(p["root"])) if inside else None
|
|
68
|
+
|
|
69
|
+
|
|
63
70
|
def all_projects():
|
|
64
71
|
with _LOCK, _db() as db:
|
|
65
72
|
rows = db.execute("SELECT data FROM projects ORDER BY root COLLATE NOCASE").fetchall()
|
package/setup_audit.py
CHANGED
package/smart_tool_daemon.py
CHANGED
|
@@ -172,7 +172,12 @@ _capabilities_cache = None
|
|
|
172
172
|
_jobs = {}
|
|
173
173
|
_JOBS_LOCK = threading.Lock()
|
|
174
174
|
_MAX_JOBS_RETAINED = 200
|
|
175
|
-
|
|
175
|
+
# A warm search never took less than 3.3 s at its fastest 5% and the median is 8.6 s: waiting less answers pending to
|
|
176
|
+
# nearly every call and costs the agent extra smart_search_result round trips. Codex gives an MCP call 60 s by default.
|
|
177
|
+
SMART_SEARCH_SYNC_WAIT_S = 25
|
|
178
|
+
OVERLAY_MAX_FILES = 40
|
|
179
|
+
COLD_SCAN_BUDGET_S = 8
|
|
180
|
+
EDIT_INDEX_DELAY_S = 8
|
|
176
181
|
# smart_search_result waits for the job this long before answering pending: answering at once had an agent poll the
|
|
177
182
|
# same job 7 times in 16 s and then give up on Smart Tool (claude -p session, 2026-10-09; job durations median 6 s,
|
|
178
183
|
# p75 10.7 s).
|
|
@@ -232,6 +237,8 @@ def _update_job(job_id, **fields):
|
|
|
232
237
|
current = _jobs.get(job_id)
|
|
233
238
|
if current is not None and current.get("status") == "pending":
|
|
234
239
|
updated = {**current, **fields, "updated_at": time.time()}
|
|
240
|
+
if fields.get("phase") and fields["phase"] != current.get("phase"):
|
|
241
|
+
updated["phases"] = [*current.get("phases", []), [fields["phase"], time.time()]]
|
|
235
242
|
_jobs[job_id] = updated
|
|
236
243
|
if (fields.get("phase") != current.get("phase") or
|
|
237
244
|
time.monotonic() - _PERSIST_LAST.get(job_id, 0) > .5 or fields.get("cancel_requested")):
|
|
@@ -241,6 +248,20 @@ def _update_job(job_id, **fields):
|
|
|
241
248
|
_persist_project_job(job_id, persistent)
|
|
242
249
|
|
|
243
250
|
|
|
251
|
+
def _phase_timings(job):
|
|
252
|
+
""""phase:seconds" for each phase of a job in the order it ran; the last phase lasts until the job finished."""
|
|
253
|
+
marks = [("queued", job.get("created_at") or 0), *[tuple(mark) for mark in job.get("phases", [])]]
|
|
254
|
+
end = job.get("finished_at") or time.time()
|
|
255
|
+
spans = {}
|
|
256
|
+
for (name, start), following in zip(marks, [*marks[1:], ("", end)]):
|
|
257
|
+
spans[name] = spans.get(name, 0.0) + max(following[1] - start, 0.0)
|
|
258
|
+
return ",".join(f"{name}:{seconds:.2f}" for name, seconds in spans.items())
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
_EDIT_PENDING = {}
|
|
262
|
+
_EDIT_LOCK = threading.Lock()
|
|
263
|
+
|
|
264
|
+
|
|
244
265
|
def _project_lock(root):
|
|
245
266
|
key = project_identity.canonical_root(root)
|
|
246
267
|
with _PROJECT_LOCKS_LOCK:
|
|
@@ -322,7 +343,10 @@ SERVER_INSTRUCTIONS = (
|
|
|
322
343
|
"with query). Never retry the same search or download through Bash, python, node, PowerShell, curl or wget: that "
|
|
323
344
|
"bypasses the rule the user set. Reading one known file, or searching inside one known file, is never blocked. "
|
|
324
345
|
"Before changing a function, project_manage action=graph with symbol lists its callers and tests; after editing, "
|
|
325
|
-
"project_manage action=affected_tests lists the tests to run."
|
|
346
|
+
"project_manage action=affected_tests lists the tests to run. When smart_search returns nothing, retry smart_search with "
|
|
347
|
+
"other terms; do not switch to Grep, Read of folders or shell search. A helper agent (smart-scout, a cheap model) is "
|
|
348
|
+
"optional: use it only for a task that chains several Smart Tool calls or while the index is still loading; a plain "
|
|
349
|
+
"search is faster called directly. A helper that returns empty means retry with the tool or report not found."
|
|
326
350
|
)
|
|
327
351
|
|
|
328
352
|
TOOLS = [
|
|
@@ -377,7 +401,10 @@ TOOLS = [
|
|
|
377
401
|
"query_comments with the same request in the comments' language; when the profile shows two "
|
|
378
402
|
"languages both are required, or explain in single_language_reason. Put documentation searches in "
|
|
379
403
|
"doc_query. Code and test results list the likeliest functions as name:line in symbols. Tune top_k per block for the project (project_manage search_limits). Fast searches return results directly. If work takes longer than a few seconds, "
|
|
380
|
-
"a pending job_id is returned; call smart_search_result with that ID until done."
|
|
404
|
+
"a pending job_id is returned; call smart_search_result with that ID until done. A warning that starts "
|
|
405
|
+
"with 'Indexing in progress' means the files named there changed after the last index: they were "
|
|
406
|
+
"searched by keyword from disk and the rest of the answer is current; repeat the search later for their "
|
|
407
|
+
"semantic ranking."
|
|
381
408
|
),
|
|
382
409
|
"inputSchema": {
|
|
383
410
|
"type": "object",
|
|
@@ -814,6 +841,133 @@ def _agent_scope_request(root, scope, problem):
|
|
|
814
841
|
"(always included unless listed in exclude).")
|
|
815
842
|
|
|
816
843
|
|
|
844
|
+
def _profile_due(scope, scope_model):
|
|
845
|
+
"""Whether the saved scope lacks a current language profile and the model may be asked for one now, that is, a
|
|
846
|
+
scope model is configured and the pause after a failed attempt has passed."""
|
|
847
|
+
failed_at = (scope.get("profile_error") or {}).get("at", 0)
|
|
848
|
+
return bool(scope_model and (scope.get("profile") or {}).get("version") != index_profile.PROFILE_VERSION
|
|
849
|
+
and time.time() - failed_at >= index_profile.RETRY_AFTER_S)
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def _index_lag(root, model, cfg, problem, job_id):
|
|
853
|
+
"""What a search can serve without waiting for the index. `None` when the existing index cannot be used as it is
|
|
854
|
+
(missing, built by another model, chunker or view, no saved scope, or a scope that needs the model while it is
|
|
855
|
+
unavailable): the caller then updates it before searching. Otherwise `{"changed", "removed", "refresh"}` with the
|
|
856
|
+
in-scope files that differ from the indexed state, the indexed files that left the scope, and whether a
|
|
857
|
+
background update is due (stale files, or a scope or profile the model has to refresh)."""
|
|
858
|
+
project = project_store.get(project_identity.project_id(root)) or {}
|
|
859
|
+
info = indexer.info(root)
|
|
860
|
+
view = index_views.describe(root)
|
|
861
|
+
scope = index_scope.load_scope(root)
|
|
862
|
+
if (not info.get("chunks") or info.get("model_id") != model or not index_scope.valid_scope(scope)
|
|
863
|
+
or info.get("chunker_version") != indexer.CHUNKER_VERSION
|
|
864
|
+
or info.get("schema_version") != indexer.SCHEMA_VERSION
|
|
865
|
+
or info.get("chunk_lines") != indexer.CHUNK_LINES
|
|
866
|
+
or (view.get("git") and info.get("view_id") != view.get("view_id"))
|
|
867
|
+
or project.get("scope_dirty")):
|
|
868
|
+
return None
|
|
869
|
+
scope_model = cfg.get("scope_model") or cfg.get("router_model")
|
|
870
|
+
rescan = index_scope.needs_rescan(root, scope)
|
|
871
|
+
if rescan and (not scope_model or problem):
|
|
872
|
+
return None
|
|
873
|
+
_update_job(job_id, phase="listing")
|
|
874
|
+
changed, removed = indexer.stale_files(root, index_scope.resolve_included_files(root, scope))
|
|
875
|
+
return {"changed": changed, "removed": removed,
|
|
876
|
+
"refresh": bool(changed or removed or rescan or _profile_due(scope, scope_model))}
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
def _cold_index(root, cfg, problem):
|
|
880
|
+
"""Whether this search is the first one of a project with no index, and the index can be built in the background:
|
|
881
|
+
the gateway answers and the scope is saved or a model can decide it. Otherwise the search keeps its old course
|
|
882
|
+
(build first, or ask the agent for the scope)."""
|
|
883
|
+
if indexer.info(root).get("chunks") or problem:
|
|
884
|
+
return False
|
|
885
|
+
return bool(index_scope.valid_scope(index_scope.load_scope(root)) or cfg.get("scope_model") or cfg.get("router_model"))
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def _cold_overlay(root, queries):
|
|
889
|
+
"""`(overlay, warning)` for a project with no index: the in-scope files (the saved scope, else the whole folder minus
|
|
890
|
+
the always-excluded names) holding a query term, read from disk and ranked by keyword, plus the notice that the
|
|
891
|
+
index is being built. The scan stops after `COLD_SCAN_BUDGET_S`."""
|
|
892
|
+
scope = index_scope.load_scope(root)
|
|
893
|
+
files = index_scope.resolve_included_files(root, scope if index_scope.valid_scope(scope) else {"include": [""]})
|
|
894
|
+
chunks, scanned = indexer.keyword_chunks(root, files, queries, COLD_SCAN_BUDGET_S)
|
|
895
|
+
reached = f"{scanned} of {len(files)} files" if scanned < len(files) else f"all {len(files)} files"
|
|
896
|
+
warning = ("Indexing in progress: this folder has no index yet, so the results are keyword matches read straight "
|
|
897
|
+
f"from disk in {reached}, without semantic ranking. The index is being built in the background: repeat "
|
|
898
|
+
"the search in a few minutes for the full ranking.")
|
|
899
|
+
return {"drop": [], "chunks": chunks}, warning
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
def _lag_overlay(root, lag):
|
|
903
|
+
"""`(overlay, warning)` for `indexer.search`: the stale files read from disk, searched by keyword in place of their
|
|
904
|
+
outdated chunks, and the notice telling the agent that indexing is in progress. `(None, "")` when the index is
|
|
905
|
+
current."""
|
|
906
|
+
stale = lag["changed"] + lag["removed"]
|
|
907
|
+
if not stale:
|
|
908
|
+
return None, ""
|
|
909
|
+
shown = lag["changed"][:OVERLAY_MAX_FILES]
|
|
910
|
+
skipped = len(lag["changed"]) - len(shown)
|
|
911
|
+
names = ", ".join(stale[:3]) + (f" (+{len(stale) - 3})" if len(stale) > 3 else "")
|
|
912
|
+
warning = (f"Indexing in progress: {len(stale)} file(s) changed after the last index ({names}). Results come from "
|
|
913
|
+
"the existing index; those files were searched by keyword straight from disk, without semantic "
|
|
914
|
+
"ranking" + (f", and {skipped} of them were not searched" if skipped else "") +
|
|
915
|
+
". The index updates in the background: repeat the search later for the updated ranking.")
|
|
916
|
+
return {"drop": stale, "chunks": indexer.overlay_chunks(root, shown)}, warning
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
def _refresh_in_background(root, lag):
|
|
920
|
+
"""Queues the index update that the search did not wait for. Returns the notice to show when it could not be
|
|
921
|
+
queued, `""` when it was queued or none was due."""
|
|
922
|
+
if not lag["refresh"]:
|
|
923
|
+
return ""
|
|
924
|
+
try:
|
|
925
|
+
_submit_project_job("project_index", {"project_root": root, "_origin": "search"})
|
|
926
|
+
except RuntimeError as exc:
|
|
927
|
+
return f"The background index update could not start ({exc}); the next search tries again."
|
|
928
|
+
return ""
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
def _note_edit(payload):
|
|
932
|
+
"""Queues a debounced background index update for a file an agent is about to write, in an eager project whose
|
|
933
|
+
watcher is not running. The PreToolUse hook sees the edit before it lands, so the update waits
|
|
934
|
+
`EDIT_INDEX_DELAY_S` and groups the edits of a burst into one job with `changed_paths`."""
|
|
935
|
+
if not isinstance(payload, dict) or payload.get("tool_name") not in ("Edit", "MultiEdit", "Write"):
|
|
936
|
+
return
|
|
937
|
+
target = (payload.get("tool_input") or {}).get("file_path")
|
|
938
|
+
if not isinstance(target, str) or not os.path.isabs(target):
|
|
939
|
+
return
|
|
940
|
+
project = project_store.containing(target)
|
|
941
|
+
if (not project or not project.get("enabled") or project.get("paused") or project.get("update_mode") != "eager"
|
|
942
|
+
or project.get("watch_backend", "off") != "off"):
|
|
943
|
+
return
|
|
944
|
+
rel = os.path.relpath(project_identity.display_root(target), project["root"]).replace(os.sep, "/")
|
|
945
|
+
with _EDIT_LOCK:
|
|
946
|
+
pending = _EDIT_PENDING.get(project["id"])
|
|
947
|
+
if pending is not None:
|
|
948
|
+
pending.add(rel)
|
|
949
|
+
return
|
|
950
|
+
_EDIT_PENDING[project["id"]] = {rel}
|
|
951
|
+
timer = threading.Timer(EDIT_INDEX_DELAY_S, _flush_edits, args=(project["id"],))
|
|
952
|
+
timer.daemon = True
|
|
953
|
+
timer.start()
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def _flush_edits(project_id):
|
|
957
|
+
"""Submits the index update for the files `_note_edit` collected; a failure is logged because it runs on a timer
|
|
958
|
+
thread nobody waits for."""
|
|
959
|
+
with _EDIT_LOCK:
|
|
960
|
+
paths = _EDIT_PENDING.pop(project_id, set())
|
|
961
|
+
project = project_store.get(project_id)
|
|
962
|
+
if not paths or not project or project.get("paused") or not project.get("enabled"):
|
|
963
|
+
return
|
|
964
|
+
try:
|
|
965
|
+
_submit_project_job("project_index", {"project_root": project["root"], "changed_paths": sorted(paths),
|
|
966
|
+
"_origin": "edit"})
|
|
967
|
+
except Exception as exc:
|
|
968
|
+
_log(f"edit index update not queued: {type(exc).__name__}: {exc}")
|
|
969
|
+
|
|
970
|
+
|
|
817
971
|
def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, preview=False, model=None):
|
|
818
972
|
root = arguments["project_root"]
|
|
819
973
|
project = project_store.get(project_identity.project_id(root)) or {}
|
|
@@ -835,9 +989,7 @@ def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, pre
|
|
|
835
989
|
deadline=_job_deadline(job_id), cancel_check=checkpoint)
|
|
836
990
|
_update_job(job_id, phase="listing")
|
|
837
991
|
included = index_scope.resolve_included_files(root, scope, cancel_check=checkpoint)
|
|
838
|
-
|
|
839
|
-
if ((scope.get("profile") or {}).get("version") != index_profile.PROFILE_VERSION
|
|
840
|
-
and time.time() - failed_at >= index_profile.RETRY_AFTER_S and scope_model):
|
|
992
|
+
if _profile_due(scope, scope_model):
|
|
841
993
|
_update_job(job_id, phase="profile")
|
|
842
994
|
try:
|
|
843
995
|
profile = index_profile.decide(root, included, scope_model, deadline=_job_deadline(job_id))
|
|
@@ -918,6 +1070,8 @@ def _grouped_yaml(profile, blocks, warning):
|
|
|
918
1070
|
|
|
919
1071
|
LOCAL_EMBEDDER_DOWN_WARNING = ("The mcp-memory local embedder is configured but not responding: index and search are lexical only "
|
|
920
1072
|
"until it is back (check the mcp-memory embedding service).")
|
|
1073
|
+
EMPTY_RESULT_NOTICE = ("No matches. Do not fall back to Grep, Read of folders or shell search: retry smart_search with other "
|
|
1074
|
+
"identifier terms, another group, or a wider top_k, and if it stays empty report that the code was not found.")
|
|
921
1075
|
LEXICAL_WARNING = ("No embedding model: index and search are lexical only (exact words, no synonyms or cross-language matches). "
|
|
922
1076
|
"Configure an embedding_model for semantic search.")
|
|
923
1077
|
|
|
@@ -939,6 +1093,7 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
939
1093
|
model = local_embedder.resolve(cfg.get("embedding_model"), indexer.info(root).get("model_id"))
|
|
940
1094
|
warnings = []
|
|
941
1095
|
vectors = {}
|
|
1096
|
+
overlay = None
|
|
942
1097
|
search_model = model
|
|
943
1098
|
if problem:
|
|
944
1099
|
model_client.OFFLINE.set(problem)
|
|
@@ -955,6 +1110,10 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
955
1110
|
raise
|
|
956
1111
|
warnings.append("Lexical index not updated; results from the last ready index. " + _project_error(exc))
|
|
957
1112
|
search_model = None
|
|
1113
|
+
elif _cold_index(root, cfg, problem):
|
|
1114
|
+
overlay, notice = _cold_overlay(root, texts)
|
|
1115
|
+
search_model = None
|
|
1116
|
+
warnings += [text for text in (notice, _refresh_in_background(root, {"refresh": True})) if text]
|
|
958
1117
|
elif problem and not local_embedder.is_local(model):
|
|
959
1118
|
if not indexer.info(root).get("chunks"):
|
|
960
1119
|
raise RuntimeError(problem)
|
|
@@ -975,18 +1134,23 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
975
1134
|
warnings.append("Embedding unavailable; lexical results from the last ready index. The update was not confirmed. " + _project_error(exc))
|
|
976
1135
|
search_model = None
|
|
977
1136
|
if vectors:
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
1137
|
+
lag = _index_lag(root, model, cfg, problem, job_id)
|
|
1138
|
+
if lag is not None:
|
|
1139
|
+
overlay, notice = _lag_overlay(root, lag)
|
|
1140
|
+
warnings += [text for text in (notice, _refresh_in_background(root, lag)) if text]
|
|
1141
|
+
else:
|
|
1142
|
+
try:
|
|
1143
|
+
_prepare_project_index(arguments, job_id, cfg, token, query_vector=next(iter(vectors.values())),
|
|
1144
|
+
model=model)
|
|
1145
|
+
except ProjectJobCancelled:
|
|
985
1146
|
raise
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
1147
|
+
except Exception as exc:
|
|
1148
|
+
if not indexer.info(root).get("chunks"):
|
|
1149
|
+
raise
|
|
1150
|
+
warnings.append("Index update failed; results from the last ready index. " + _project_error(exc))
|
|
1151
|
+
search_model = None
|
|
1152
|
+
if indexer.info(root).get("model_id") != model:
|
|
1153
|
+
vectors = {}
|
|
990
1154
|
if vectors and not cfg.get("rerank_model"):
|
|
991
1155
|
warnings.append("Rerank not configured: hybrid lexical/vector order, less precise at the top. Configure a rerank_model.")
|
|
992
1156
|
_job_checkpoint(job_id)
|
|
@@ -1015,7 +1179,7 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1015
1179
|
query_terms = _symbol_terms(" ".join(queries[group] for group in groups if group != "doc"))
|
|
1016
1180
|
_update_job(job_id, phase="searching")
|
|
1017
1181
|
found = {group: indexer.search(root, vectors.get(queries[group]), top_k=max(limits[group] * 4, candidate_pool),
|
|
1018
|
-
query=queries[group], model_id=search_model,
|
|
1182
|
+
query=queries[group], model_id=search_model, overlay=overlay,
|
|
1019
1183
|
include=lambda path, group=group: index_profile.kind(path, profile) == group)
|
|
1020
1184
|
for group in groups}
|
|
1021
1185
|
|
|
@@ -1035,7 +1199,12 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1035
1199
|
ordered = [(float(item["relevance_score"]), "rerank", candidates[item["index"]][1:]) for item in reranked]
|
|
1036
1200
|
except Exception as exc:
|
|
1037
1201
|
problem = f"Rerank unavailable for {group}; hybrid lexical/vector order kept. " + _project_error(exc)
|
|
1038
|
-
|
|
1202
|
+
exact = indexer.overlay_exact_keys(overlay, query)
|
|
1203
|
+
if exact:
|
|
1204
|
+
pinned = [(c[0], "overlay", c[1:]) for c in candidates if tuple(c[1:4]) in exact][:2]
|
|
1205
|
+
pinned_keys = {tuple(item[2][:3]) for item in pinned}
|
|
1206
|
+
ordered = pinned + [item for item in ordered if tuple(item[2][:3]) not in pinned_keys]
|
|
1207
|
+
rows =[{"path": path, "lines": f"{start}-{end}", "score": round(score, 6), "snippet": text[:500],
|
|
1039
1208
|
"ranking": ranking, "location_kind": document_text.location_kind(path),
|
|
1040
1209
|
**({"symbols": _chunk_symbols(symbol_index, path, start, end, text, query_terms)}
|
|
1041
1210
|
if symbol_index and group != "doc" else {})}
|
|
@@ -1050,6 +1219,8 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1050
1219
|
if problem:
|
|
1051
1220
|
warnings.append(problem)
|
|
1052
1221
|
_job_checkpoint(job_id)
|
|
1222
|
+
if not any(blocks.values()):
|
|
1223
|
+
notes.append(EMPTY_RESULT_NOTICE)
|
|
1053
1224
|
warning = " ".join(warnings)
|
|
1054
1225
|
if warning:
|
|
1055
1226
|
_update_job(job_id, warning=warning)
|
|
@@ -1120,7 +1291,8 @@ def _run_project_job_in_view(job_id, arguments, kind):
|
|
|
1120
1291
|
if updated:
|
|
1121
1292
|
fields.update(indexed_seq=captured_seq, last_indexed=time.time())
|
|
1122
1293
|
project_store.update(project_id, **fields)
|
|
1123
|
-
_log(f"project_job {job_id} {project_id} {kind} done elapsed={time.time()-job['created_at']:.2f}s"
|
|
1294
|
+
_log(f"project_job {job_id} {project_id} {kind} done elapsed={time.time()-job['created_at']:.2f}s "
|
|
1295
|
+
f"phases={_phase_timings(final)}")
|
|
1124
1296
|
return result
|
|
1125
1297
|
except Exception as exc:
|
|
1126
1298
|
with _JOBS_LOCK:
|
|
@@ -1215,8 +1387,13 @@ def _search_limits(value, groups, saved=None):
|
|
|
1215
1387
|
for group in groups:
|
|
1216
1388
|
raw = value if not isinstance(value, dict) else value.get(group, (saved or {}).get(group, SEARCH_LIMIT_DEFAULT))
|
|
1217
1389
|
limits[group] = _limit_value(raw, f"top_k.{group}")
|
|
1218
|
-
|
|
1219
|
-
|
|
1390
|
+
total = sum(limits.values())
|
|
1391
|
+
if total > SEARCH_LIMIT_TOTAL:
|
|
1392
|
+
# A first call that asks for more than the total is scaled down in proportion (a rejection costs the agent a turn).
|
|
1393
|
+
scaled = {group: max(1, limit * SEARCH_LIMIT_TOTAL // total) for group, limit in limits.items()}
|
|
1394
|
+
while sum(scaled.values()) > SEARCH_LIMIT_TOTAL:
|
|
1395
|
+
scaled[max(scaled, key=scaled.get)] -= 1
|
|
1396
|
+
limits = scaled
|
|
1220
1397
|
return limits
|
|
1221
1398
|
|
|
1222
1399
|
|
|
@@ -1285,13 +1462,13 @@ def _handle_smart_search(arguments):
|
|
|
1285
1462
|
job_id, future = _submit_project_job("smart_search", arguments)
|
|
1286
1463
|
if future is None:
|
|
1287
1464
|
_metric("pending", reason="queued")
|
|
1288
|
-
return
|
|
1465
|
+
return _agent_search_answer(_smart_search_job_payload(job_id))
|
|
1289
1466
|
try:
|
|
1290
1467
|
result = future.result(timeout=SMART_SEARCH_SYNC_WAIT_S)
|
|
1291
1468
|
except concurrent.futures.TimeoutError:
|
|
1292
1469
|
if not future.done():
|
|
1293
1470
|
_metric("pending", reason="sync_timeout")
|
|
1294
|
-
return
|
|
1471
|
+
return _agent_search_answer(_smart_search_job_payload(job_id))
|
|
1295
1472
|
result = future.result()
|
|
1296
1473
|
_mark_notice_delivered(job_id)
|
|
1297
1474
|
return result
|
|
@@ -1397,9 +1574,27 @@ def _handle_smart_search_result(arguments):
|
|
|
1397
1574
|
time.sleep(0.25)
|
|
1398
1575
|
payload = _smart_search_job_payload(job_id)
|
|
1399
1576
|
_mark_notice_delivered(job_id)
|
|
1577
|
+
if payload["kind"] == "smart_search":
|
|
1578
|
+
return _agent_search_answer(payload)
|
|
1400
1579
|
return _to_json_sanitized(payload)
|
|
1401
1580
|
|
|
1402
1581
|
|
|
1582
|
+
_AGENT_SEARCH_DROPPED = ("kind", "project_id", "updated_at", "stats", "view")
|
|
1583
|
+
|
|
1584
|
+
|
|
1585
|
+
def _agent_search_answer(payload):
|
|
1586
|
+
"""What smart_search_result hands the agent for a smart_search job: the result text itself when done (the same text
|
|
1587
|
+
a search that finishes within the wait returns, no JSON wrapper that escapes it and repeats the index stats and
|
|
1588
|
+
the git view the setup page shows), otherwise the payload without those fields."""
|
|
1589
|
+
if payload["status"] == "done":
|
|
1590
|
+
warning = payload.get("warning")
|
|
1591
|
+
return f"{payload['result']}\nwarning: {warning}" if warning else payload["result"]
|
|
1592
|
+
slim = {key: value for key, value in payload.items() if key not in _AGENT_SEARCH_DROPPED}
|
|
1593
|
+
if not slim.get("cancel_requested"):
|
|
1594
|
+
slim.pop("cancel_requested", None)
|
|
1595
|
+
return _to_json_sanitized(slim)
|
|
1596
|
+
|
|
1597
|
+
|
|
1403
1598
|
def _project_payload(project):
|
|
1404
1599
|
data = dict(project)
|
|
1405
1600
|
data['dismissed_duplicates'] = len(data.get('dismissed_duplicates') or {})
|
|
@@ -3388,6 +3583,10 @@ def _pretool_hook(handler):
|
|
|
3388
3583
|
client = (urllib.parse.parse_qs(urllib.parse.urlsplit(handler.path).query).get("client") or [""])[0]
|
|
3389
3584
|
try:
|
|
3390
3585
|
payload = json.loads(handler._read_body(HOOK_MAX_BODY_BYTES) or b"{}")
|
|
3586
|
+
try:
|
|
3587
|
+
_note_edit(payload)
|
|
3588
|
+
except Exception as exc:
|
|
3589
|
+
_log(f"edit index update skipped: {type(exc).__name__}: {exc}")
|
|
3391
3590
|
output = hook_decision.decide(payload, client or None, _web_route)
|
|
3392
3591
|
except Exception as exc:
|
|
3393
3592
|
_log(f"PreToolUse hook without a decision: {type(exc).__name__}: {exc}")
|
package/version.py
CHANGED