@allansantos-dev/smart-tool 0.9.18 → 0.9.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +1 -1
- package/agents/smart-scout.md +13 -0
- package/hook_decision.py +2 -4
- package/indexer.py +77 -4
- package/package.json +1 -1
- package/project_store.py +9 -2
- package/smart_tool_daemon.py +175 -22
- package/version.py +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.19 - beta
|
|
4
|
+
|
|
5
|
+
- A search no longer waits for the index to catch up. It answers from the existing index at once, searches the files
|
|
6
|
+
edited since the last index by keyword straight from disk, and says "Indexing in progress: N file(s) changed..." in
|
|
7
|
+
the answer while the update runs in the background. An index that does not exist yet, or was built by another model,
|
|
8
|
+
is still built first.
|
|
9
|
+
- A search waits up to 25 s for its own result before answering with a `job_id` (it was 3 s, below the 3.3 s a warm
|
|
10
|
+
search takes at best, so nearly every call needed a second `smart_search_result`).
|
|
11
|
+
- The scope and the language profile are no longer asked of the model inside a search that has an index: the saved ones
|
|
12
|
+
are used and refreshed in the background.
|
|
13
|
+
- New folders are registered watched and eager: the index follows edits by itself. A PreToolUse Edit/Write of an eager
|
|
14
|
+
folder without a running watcher queues one debounced update with the edited paths. Folders already registered keep
|
|
15
|
+
their policy.
|
|
16
|
+
- A `top_k` whose total is above 20 is scaled down in proportion instead of rejecting the call (the rejection cost the
|
|
17
|
+
agent a turn on its first search).
|
|
18
|
+
- A search made right after an edit keeps the just-edited chunk in the answer: the reranker can no longer push it out.
|
|
19
|
+
- An edit whose path comes in the Windows 8.3 short form (`ALLANS~1`-style folder names) is matched to its registered
|
|
20
|
+
project; before, the project lookup compared it with the long form and found nothing.
|
|
21
|
+
- The daemon log prints the seconds spent in each phase of a job (`phases=queued:0.02,auth:0.01,...`).
|
|
22
|
+
|
|
3
23
|
## 0.9.18 - beta
|
|
4
24
|
|
|
5
25
|
- Updates are no longer applied only at logon. The tray checks npm at start and every 24 h and installs a newer
|
package/README.md
CHANGED
|
@@ -15,7 +15,7 @@ registered as the `playwright` server where this machine lets a browser be drive
|
|
|
15
15
|
An optional hook routes the agent's native Grep/Glob/Read/Bash/WebSearch/WebFetch calls to these tools when they
|
|
16
16
|
are cheaper, or just tells the agent what Smart Tool would do.
|
|
17
17
|
|
|
18
|
-
**Status:** 0.9.
|
|
18
|
+
**Status:** 0.9.19 beta. Windows only. What changed in each version: [CHANGELOG.md](CHANGELOG.md).
|
|
19
19
|
|
|
20
20
|
## How it works
|
|
21
21
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: smart-scout
|
|
3
|
+
description: Optional helper for the Smart Tool. Use it only for a task that chains several Smart Tool calls (search, then graph, then affected_tests) or while the index is still loading. A plain search is faster called directly.
|
|
4
|
+
model: haiku
|
|
5
|
+
tools: mcp__smart-tool__smart_search, mcp__smart-tool__smart_search_result, mcp__smart-tool__project_manage, Read
|
|
6
|
+
---
|
|
7
|
+
You run Smart Tool calls for another session and return only the result.
|
|
8
|
+
|
|
9
|
+
Use smart_search with project_root equal to the working directory (forward slashes), query_identifiers in English identifier terms, query_comments in the language of the comments, and top_k {"code": 6, "test": 2, "doc": 2}. When it answers pending, call smart_search_result with the job_id until it finishes. project_manage action=graph (symbol) and action=affected_tests are yours to chain when the task asks for them. Open at most 3 files with Read and make at most 8 tool calls in total.
|
|
10
|
+
|
|
11
|
+
Reply ONLY with a list of up to 8 items, each `path:first_line-last_line: reason in up to 15 words`, no code excerpts and no introduction.
|
|
12
|
+
|
|
13
|
+
If nothing relevant came back after two different queries, reply exactly `EMPTY: <the two queries you tried>`. Never invent a path. Never use Grep, shell search or Read of folders to compensate: an empty answer is a valid answer, and the session that called you decides whether to retry with smart_search or to report that the code was not found.
|
package/hook_decision.py
CHANGED
|
@@ -171,10 +171,8 @@ def advise(reason):
|
|
|
171
171
|
|
|
172
172
|
|
|
173
173
|
def _project_of(path):
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
if target.startswith(os.path.normcase(os.path.abspath(p["root"])) + os.sep)]
|
|
177
|
-
return max(roots, key=len) if roots else None
|
|
174
|
+
project = project_store.containing(path)
|
|
175
|
+
return project["root"] if project else None
|
|
178
176
|
|
|
179
177
|
|
|
180
178
|
def _impact_line(root, rel, name):
|
package/indexer.py
CHANGED
|
@@ -779,9 +779,14 @@ def _vector_rows(root, path):
|
|
|
779
779
|
return rows, meta, norms
|
|
780
780
|
|
|
781
781
|
|
|
782
|
+
def _query_terms(query):
|
|
783
|
+
"""Up to 24 distinct word terms of the query, in order."""
|
|
784
|
+
return list(dict.fromkeys(re.findall(r"[\w]{2,}", query or "", re.UNICODE)))[:24]
|
|
785
|
+
|
|
786
|
+
|
|
782
787
|
def _lexical_scored(path, query, rows, limit, offset=0):
|
|
783
788
|
"""(chunk_id, score) best first; higher score is a better lexical match."""
|
|
784
|
-
terms =
|
|
789
|
+
terms = _query_terms(query)
|
|
785
790
|
if not terms:
|
|
786
791
|
return []
|
|
787
792
|
expression = " OR ".join('"' + term.replace('"', '""') + '"' for term in terms)
|
|
@@ -808,11 +813,72 @@ def _min_max(pairs):
|
|
|
808
813
|
return {key: (value - low) / (high - low) if high > low else 1.0 for key, value in pairs}
|
|
809
814
|
|
|
810
815
|
|
|
811
|
-
def
|
|
812
|
-
"""
|
|
816
|
+
def stale_files(root, included_paths):
|
|
817
|
+
"""`(changed, removed)` against the indexed state: `changed` lists in-scope files whose `(mtime_ns, size)` differs
|
|
818
|
+
from the stored one (or that were never indexed), `removed` lists indexed files that left the scope. It only
|
|
819
|
+
stats files, so it is the cheap answer to "what would a reindex have to touch"."""
|
|
820
|
+
snapshot = manifest_snapshot(root)
|
|
821
|
+
changed, removed = [], set(snapshot)
|
|
822
|
+
for rel in sorted(set(included_paths)):
|
|
823
|
+
stat, _reason = _stat_candidate(root, rel)
|
|
824
|
+
if stat is None:
|
|
825
|
+
continue
|
|
826
|
+
removed.discard(rel)
|
|
827
|
+
if snapshot.get(rel) != stat:
|
|
828
|
+
changed.append(rel)
|
|
829
|
+
return changed, sorted(removed)
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def overlay_chunks(root, rel_paths):
|
|
833
|
+
"""`[(path, start_line, end_line, text)]` read from disk right now for the given files, chunked like the index.
|
|
834
|
+
Files that cannot be read consistently are skipped: the overlay only complements the index, it never fails a
|
|
835
|
+
search."""
|
|
836
|
+
chunks = []
|
|
837
|
+
for rel in rel_paths:
|
|
838
|
+
data, encoding, _stat = _stable_read(root, rel)
|
|
839
|
+
if data is None:
|
|
840
|
+
continue
|
|
841
|
+
pieces = chunk_file(root, rel, CHUNK_LINES, data=data, encoding=encoding)
|
|
842
|
+
chunks.extend((rel, start, end, text) for start, end, text in pieces or [])
|
|
843
|
+
return chunks
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def _overlay_scored(chunks, query, alpha):
|
|
847
|
+
"""`[(score, path, start, end, text)]` for overlay chunks matching the query terms by keyword. A chunk that holds
|
|
848
|
+
every term scores 1.0, the top of the fused index scores (`search` ranks it first on a tie): the edited file is read
|
|
849
|
+
from disk, so it is the current answer for an exact match. A partial match stays on the 0-1 scale of the lexical
|
|
850
|
+
share of the fused score."""
|
|
851
|
+
terms = _query_terms(query)
|
|
852
|
+
scored = []
|
|
853
|
+
for path, start, end, text in chunks:
|
|
854
|
+
haystack = (path + " " + text).casefold()
|
|
855
|
+
matched = sum(term.casefold() in haystack for term in terms)
|
|
856
|
+
if matched:
|
|
857
|
+
score = 1.0 if matched == len(terms) else (1 - alpha) * matched / len(terms)
|
|
858
|
+
scored.append((score, path, start, end, text))
|
|
859
|
+
return scored
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def overlay_exact_keys(overlay, query):
|
|
863
|
+
"""`{(path, start, end)}` of the overlay chunks that hold every query term. A reranker must not push them below
|
|
864
|
+
indexed chunks: they come from the file as it is on disk now."""
|
|
865
|
+
if not overlay or not overlay["chunks"] or not query:
|
|
866
|
+
return set()
|
|
867
|
+
return {(path, start, end) for score, path, start, end, _text in _overlay_scored(overlay["chunks"], query, 0.0)
|
|
868
|
+
if score == 1.0}
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
def search(root, query_vector=None, top_k=20, query=None, model_id=None, include=None, overlay=None):
|
|
872
|
+
"""Vector plus lexical retrieval by fusion of relative scores, without network. `overlay` is
|
|
873
|
+
`{"drop": paths, "chunks": overlay_chunks(...)}`: the indexed chunks of the `drop` paths (stale or removed
|
|
874
|
+
files) are ignored and the overlay chunks, read from disk, take part by keyword instead."""
|
|
813
875
|
path = existing_db_path(root)
|
|
814
876
|
if not path:
|
|
815
877
|
return []
|
|
878
|
+
kind_filter = include
|
|
879
|
+
if overlay and overlay["drop"]:
|
|
880
|
+
dropped = set(overlay["drop"])
|
|
881
|
+
include = lambda rel: rel not in dropped and (kind_filter is None or kind_filter(rel))
|
|
816
882
|
rows, meta, norms = _vector_rows(root, path)
|
|
817
883
|
if include is not None:
|
|
818
884
|
rows = [row for row in rows if include(row[1])]
|
|
@@ -854,7 +920,14 @@ def search(root, query_vector=None, top_k=20, query=None, model_id=None, include
|
|
|
854
920
|
else:
|
|
855
921
|
scores, ids = vector_scores, vector_ids
|
|
856
922
|
by_id = {row[0]: row for row in rows}
|
|
857
|
-
|
|
923
|
+
found = [(scores[chunk_id], *by_id[chunk_id][1:5]) for chunk_id in ids[:top_k]]
|
|
924
|
+
if overlay and overlay["chunks"]:
|
|
925
|
+
extra = _overlay_scored([chunk for chunk in overlay["chunks"] if kind_filter is None or kind_filter(chunk[0])],
|
|
926
|
+
query, FUSION_ALPHA if query_vector is not None else 0.0)
|
|
927
|
+
found = [item for _score, item in sorted(
|
|
928
|
+
[((item[0], 1), item) for item in extra] + [((item[0], 0), item) for item in found],
|
|
929
|
+
key=lambda pair: pair[0], reverse=True)][:top_k]
|
|
930
|
+
return found
|
|
858
931
|
|
|
859
932
|
|
|
860
933
|
def graph_cache_path(view_path):
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.19",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Allan Santos",
|
package/project_store.py
CHANGED
|
@@ -40,8 +40,8 @@ def register(root, watch=None):
|
|
|
40
40
|
row = db.execute("SELECT data FROM projects WHERE id=?", (key,)).fetchone()
|
|
41
41
|
data = json.loads(row[0]) if row else {
|
|
42
42
|
"id": key, "root": display, "name": os.path.basename(display) or display,
|
|
43
|
-
"status": "registered", "watch":
|
|
44
|
-
"update_mode": "
|
|
43
|
+
"status": "registered", "watch": True, "paused": False, "enabled": False,
|
|
44
|
+
"update_mode": "eager",
|
|
45
45
|
"dirty_seq": 0, "indexed_seq": 0, "created_at": time.time(),
|
|
46
46
|
"last_indexed": None, "last_checked": None, "last_error": "",
|
|
47
47
|
"last_job_id": None, "watch_backend": "off", "next_retry_at": 0,
|
|
@@ -60,6 +60,13 @@ def get(project_id):
|
|
|
60
60
|
return json.loads(row[0]) if row else None
|
|
61
61
|
|
|
62
62
|
|
|
63
|
+
def containing(path):
|
|
64
|
+
"""The registered project whose folder holds `path` (the deepest one when projects nest), or `None`."""
|
|
65
|
+
target = project_identity.canonical_root(path)
|
|
66
|
+
inside = [p for p in all_projects() if target.startswith(project_identity.canonical_root(p["root"]) + os.sep)]
|
|
67
|
+
return max(inside, key=lambda p: len(p["root"])) if inside else None
|
|
68
|
+
|
|
69
|
+
|
|
63
70
|
def all_projects():
|
|
64
71
|
with _LOCK, _db() as db:
|
|
65
72
|
rows = db.execute("SELECT data FROM projects ORDER BY root COLLATE NOCASE").fetchall()
|
package/smart_tool_daemon.py
CHANGED
|
@@ -172,7 +172,11 @@ _capabilities_cache = None
|
|
|
172
172
|
_jobs = {}
|
|
173
173
|
_JOBS_LOCK = threading.Lock()
|
|
174
174
|
_MAX_JOBS_RETAINED = 200
|
|
175
|
-
|
|
175
|
+
# A warm search never took less than 3.3 s at its fastest 5% and the median is 8.6 s: waiting less answers pending to
|
|
176
|
+
# nearly every call and costs the agent extra smart_search_result round trips. Codex gives an MCP call 60 s by default.
|
|
177
|
+
SMART_SEARCH_SYNC_WAIT_S = 25
|
|
178
|
+
OVERLAY_MAX_FILES = 40
|
|
179
|
+
EDIT_INDEX_DELAY_S = 8
|
|
176
180
|
# smart_search_result waits for the job this long before answering pending: answering at once had an agent poll the
|
|
177
181
|
# same job 7 times in 16 s and then give up on Smart Tool (claude -p session, 2026-10-09; job durations median 6 s,
|
|
178
182
|
# p75 10.7 s).
|
|
@@ -232,6 +236,8 @@ def _update_job(job_id, **fields):
|
|
|
232
236
|
current = _jobs.get(job_id)
|
|
233
237
|
if current is not None and current.get("status") == "pending":
|
|
234
238
|
updated = {**current, **fields, "updated_at": time.time()}
|
|
239
|
+
if fields.get("phase") and fields["phase"] != current.get("phase"):
|
|
240
|
+
updated["phases"] = [*current.get("phases", []), [fields["phase"], time.time()]]
|
|
235
241
|
_jobs[job_id] = updated
|
|
236
242
|
if (fields.get("phase") != current.get("phase") or
|
|
237
243
|
time.monotonic() - _PERSIST_LAST.get(job_id, 0) > .5 or fields.get("cancel_requested")):
|
|
@@ -241,6 +247,20 @@ def _update_job(job_id, **fields):
|
|
|
241
247
|
_persist_project_job(job_id, persistent)
|
|
242
248
|
|
|
243
249
|
|
|
250
|
+
def _phase_timings(job):
|
|
251
|
+
""""phase:seconds" for each phase of a job in the order it ran; the last phase lasts until the job finished."""
|
|
252
|
+
marks = [("queued", job.get("created_at") or 0), *[tuple(mark) for mark in job.get("phases", [])]]
|
|
253
|
+
end = job.get("finished_at") or time.time()
|
|
254
|
+
spans = {}
|
|
255
|
+
for (name, start), following in zip(marks, [*marks[1:], ("", end)]):
|
|
256
|
+
spans[name] = spans.get(name, 0.0) + max(following[1] - start, 0.0)
|
|
257
|
+
return ",".join(f"{name}:{seconds:.2f}" for name, seconds in spans.items())
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
_EDIT_PENDING = {}
|
|
261
|
+
_EDIT_LOCK = threading.Lock()
|
|
262
|
+
|
|
263
|
+
|
|
244
264
|
def _project_lock(root):
|
|
245
265
|
key = project_identity.canonical_root(root)
|
|
246
266
|
with _PROJECT_LOCKS_LOCK:
|
|
@@ -322,7 +342,10 @@ SERVER_INSTRUCTIONS = (
|
|
|
322
342
|
"with query). Never retry the same search or download through Bash, python, node, PowerShell, curl or wget: that "
|
|
323
343
|
"bypasses the rule the user set. Reading one known file, or searching inside one known file, is never blocked. "
|
|
324
344
|
"Before changing a function, project_manage action=graph with symbol lists its callers and tests; after editing, "
|
|
325
|
-
"project_manage action=affected_tests lists the tests to run."
|
|
345
|
+
"project_manage action=affected_tests lists the tests to run. When smart_search returns nothing, retry smart_search with "
|
|
346
|
+
"other terms; do not switch to Grep, Read of folders or shell search. A helper agent (smart-scout, a cheap model) is "
|
|
347
|
+
"optional: use it only for a task that chains several Smart Tool calls or while the index is still loading; a plain "
|
|
348
|
+
"search is faster called directly. A helper that returns empty means retry with the tool or report not found."
|
|
326
349
|
)
|
|
327
350
|
|
|
328
351
|
TOOLS = [
|
|
@@ -377,7 +400,10 @@ TOOLS = [
|
|
|
377
400
|
"query_comments with the same request in the comments' language; when the profile shows two "
|
|
378
401
|
"languages both are required, or explain in single_language_reason. Put documentation searches in "
|
|
379
402
|
"doc_query. Code and test results list the likeliest functions as name:line in symbols. Tune top_k per block for the project (project_manage search_limits). Fast searches return results directly. If work takes longer than a few seconds, "
|
|
380
|
-
"a pending job_id is returned; call smart_search_result with that ID until done."
|
|
403
|
+
"a pending job_id is returned; call smart_search_result with that ID until done. A warning that starts "
|
|
404
|
+
"with 'Indexing in progress' means the files named there changed after the last index: they were "
|
|
405
|
+
"searched by keyword from disk and the rest of the answer is current; repeat the search later for their "
|
|
406
|
+
"semantic ranking."
|
|
381
407
|
),
|
|
382
408
|
"inputSchema": {
|
|
383
409
|
"type": "object",
|
|
@@ -814,6 +840,110 @@ def _agent_scope_request(root, scope, problem):
|
|
|
814
840
|
"(always included unless listed in exclude).")
|
|
815
841
|
|
|
816
842
|
|
|
843
|
+
def _profile_due(scope, scope_model):
|
|
844
|
+
"""Whether the saved scope lacks a current language profile and the model may be asked for one now, that is, a
|
|
845
|
+
scope model is configured and the pause after a failed attempt has passed."""
|
|
846
|
+
failed_at = (scope.get("profile_error") or {}).get("at", 0)
|
|
847
|
+
return bool(scope_model and (scope.get("profile") or {}).get("version") != index_profile.PROFILE_VERSION
|
|
848
|
+
and time.time() - failed_at >= index_profile.RETRY_AFTER_S)
|
|
849
|
+
|
|
850
|
+
|
|
851
|
+
def _index_lag(root, model, cfg, problem, job_id):
|
|
852
|
+
"""What a search can serve without waiting for the index. `None` when the existing index cannot be used as it is
|
|
853
|
+
(missing, built by another model, chunker or view, no saved scope, or a scope that needs the model while it is
|
|
854
|
+
unavailable): the caller then updates it before searching. Otherwise `{"changed", "removed", "refresh"}` with the
|
|
855
|
+
in-scope files that differ from the indexed state, the indexed files that left the scope, and whether a
|
|
856
|
+
background update is due (stale files, or a scope or profile the model has to refresh)."""
|
|
857
|
+
project = project_store.get(project_identity.project_id(root)) or {}
|
|
858
|
+
info = indexer.info(root)
|
|
859
|
+
view = index_views.describe(root)
|
|
860
|
+
scope = index_scope.load_scope(root)
|
|
861
|
+
if (not info.get("chunks") or info.get("model_id") != model or not index_scope.valid_scope(scope)
|
|
862
|
+
or info.get("chunker_version") != indexer.CHUNKER_VERSION
|
|
863
|
+
or info.get("schema_version") != indexer.SCHEMA_VERSION
|
|
864
|
+
or info.get("chunk_lines") != indexer.CHUNK_LINES
|
|
865
|
+
or (view.get("git") and info.get("view_id") != view.get("view_id"))
|
|
866
|
+
or project.get("scope_dirty")):
|
|
867
|
+
return None
|
|
868
|
+
scope_model = cfg.get("scope_model") or cfg.get("router_model")
|
|
869
|
+
rescan = index_scope.needs_rescan(root, scope)
|
|
870
|
+
if rescan and (not scope_model or problem):
|
|
871
|
+
return None
|
|
872
|
+
_update_job(job_id, phase="listing")
|
|
873
|
+
changed, removed = indexer.stale_files(root, index_scope.resolve_included_files(root, scope))
|
|
874
|
+
return {"changed": changed, "removed": removed,
|
|
875
|
+
"refresh": bool(changed or removed or rescan or _profile_due(scope, scope_model))}
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def _lag_overlay(root, lag):
|
|
879
|
+
"""`(overlay, warning)` for `indexer.search`: the stale files read from disk, searched by keyword in place of their
|
|
880
|
+
outdated chunks, and the notice telling the agent that indexing is in progress. `(None, "")` when the index is
|
|
881
|
+
current."""
|
|
882
|
+
stale = lag["changed"] + lag["removed"]
|
|
883
|
+
if not stale:
|
|
884
|
+
return None, ""
|
|
885
|
+
shown = lag["changed"][:OVERLAY_MAX_FILES]
|
|
886
|
+
skipped = len(lag["changed"]) - len(shown)
|
|
887
|
+
names = ", ".join(stale[:3]) + (f" (+{len(stale) - 3})" if len(stale) > 3 else "")
|
|
888
|
+
warning = (f"Indexing in progress: {len(stale)} file(s) changed after the last index ({names}). Results come from "
|
|
889
|
+
"the existing index; those files were searched by keyword straight from disk, without semantic "
|
|
890
|
+
"ranking" + (f", and {skipped} of them were not searched" if skipped else "") +
|
|
891
|
+
". The index updates in the background: repeat the search later for the updated ranking.")
|
|
892
|
+
return {"drop": stale, "chunks": indexer.overlay_chunks(root, shown)}, warning
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
def _refresh_in_background(root, lag):
|
|
896
|
+
"""Queues the index update that the search did not wait for. Returns the notice to show when it could not be
|
|
897
|
+
queued, `""` when it was queued or none was due."""
|
|
898
|
+
if not lag["refresh"]:
|
|
899
|
+
return ""
|
|
900
|
+
try:
|
|
901
|
+
_submit_project_job("project_index", {"project_root": root, "_origin": "search"})
|
|
902
|
+
except RuntimeError as exc:
|
|
903
|
+
return f"The background index update could not start ({exc}); the next search tries again."
|
|
904
|
+
return ""
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
def _note_edit(payload):
|
|
908
|
+
"""Queues a debounced background index update for a file an agent is about to write, in an eager project whose
|
|
909
|
+
watcher is not running. The PreToolUse hook sees the edit before it lands, so the update waits
|
|
910
|
+
`EDIT_INDEX_DELAY_S` and groups the edits of a burst into one job with `changed_paths`."""
|
|
911
|
+
if not isinstance(payload, dict) or payload.get("tool_name") not in ("Edit", "MultiEdit", "Write"):
|
|
912
|
+
return
|
|
913
|
+
target = (payload.get("tool_input") or {}).get("file_path")
|
|
914
|
+
if not isinstance(target, str) or not os.path.isabs(target):
|
|
915
|
+
return
|
|
916
|
+
project = project_store.containing(target)
|
|
917
|
+
if (not project or not project.get("enabled") or project.get("paused") or project.get("update_mode") != "eager"
|
|
918
|
+
or project.get("watch_backend", "off") != "off"):
|
|
919
|
+
return
|
|
920
|
+
rel = os.path.relpath(project_identity.display_root(target), project["root"]).replace(os.sep, "/")
|
|
921
|
+
with _EDIT_LOCK:
|
|
922
|
+
pending = _EDIT_PENDING.get(project["id"])
|
|
923
|
+
if pending is not None:
|
|
924
|
+
pending.add(rel)
|
|
925
|
+
return
|
|
926
|
+
_EDIT_PENDING[project["id"]] = {rel}
|
|
927
|
+
timer = threading.Timer(EDIT_INDEX_DELAY_S, _flush_edits, args=(project["id"],))
|
|
928
|
+
timer.daemon = True
|
|
929
|
+
timer.start()
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
def _flush_edits(project_id):
|
|
933
|
+
"""Submits the index update for the files `_note_edit` collected; a failure is logged because it runs on a timer
|
|
934
|
+
thread nobody waits for."""
|
|
935
|
+
with _EDIT_LOCK:
|
|
936
|
+
paths = _EDIT_PENDING.pop(project_id, set())
|
|
937
|
+
project = project_store.get(project_id)
|
|
938
|
+
if not paths or not project or project.get("paused") or not project.get("enabled"):
|
|
939
|
+
return
|
|
940
|
+
try:
|
|
941
|
+
_submit_project_job("project_index", {"project_root": project["root"], "changed_paths": sorted(paths),
|
|
942
|
+
"_origin": "edit"})
|
|
943
|
+
except Exception as exc:
|
|
944
|
+
_log(f"edit index update not queued: {type(exc).__name__}: {exc}")
|
|
945
|
+
|
|
946
|
+
|
|
817
947
|
def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, preview=False, model=None):
|
|
818
948
|
root = arguments["project_root"]
|
|
819
949
|
project = project_store.get(project_identity.project_id(root)) or {}
|
|
@@ -835,9 +965,7 @@ def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, pre
|
|
|
835
965
|
deadline=_job_deadline(job_id), cancel_check=checkpoint)
|
|
836
966
|
_update_job(job_id, phase="listing")
|
|
837
967
|
included = index_scope.resolve_included_files(root, scope, cancel_check=checkpoint)
|
|
838
|
-
|
|
839
|
-
if ((scope.get("profile") or {}).get("version") != index_profile.PROFILE_VERSION
|
|
840
|
-
and time.time() - failed_at >= index_profile.RETRY_AFTER_S and scope_model):
|
|
968
|
+
if _profile_due(scope, scope_model):
|
|
841
969
|
_update_job(job_id, phase="profile")
|
|
842
970
|
try:
|
|
843
971
|
profile = index_profile.decide(root, included, scope_model, deadline=_job_deadline(job_id))
|
|
@@ -918,6 +1046,8 @@ def _grouped_yaml(profile, blocks, warning):
|
|
|
918
1046
|
|
|
919
1047
|
LOCAL_EMBEDDER_DOWN_WARNING = ("The mcp-memory local embedder is configured but not responding: index and search are lexical only "
|
|
920
1048
|
"until it is back (check the mcp-memory embedding service).")
|
|
1049
|
+
EMPTY_RESULT_NOTICE = ("No matches. Do not fall back to Grep, Read of folders or shell search: retry smart_search with other "
|
|
1050
|
+
"identifier terms, another group, or a wider top_k, and if it stays empty report that the code was not found.")
|
|
921
1051
|
LEXICAL_WARNING = ("No embedding model: index and search are lexical only (exact words, no synonyms or cross-language matches). "
|
|
922
1052
|
"Configure an embedding_model for semantic search.")
|
|
923
1053
|
|
|
@@ -939,6 +1069,7 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
939
1069
|
model = local_embedder.resolve(cfg.get("embedding_model"), indexer.info(root).get("model_id"))
|
|
940
1070
|
warnings = []
|
|
941
1071
|
vectors = {}
|
|
1072
|
+
overlay = None
|
|
942
1073
|
search_model = model
|
|
943
1074
|
if problem:
|
|
944
1075
|
model_client.OFFLINE.set(problem)
|
|
@@ -975,18 +1106,23 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
975
1106
|
warnings.append("Embedding unavailable; lexical results from the last ready index. The update was not confirmed. " + _project_error(exc))
|
|
976
1107
|
search_model = None
|
|
977
1108
|
if vectors:
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
1109
|
+
lag = _index_lag(root, model, cfg, problem, job_id)
|
|
1110
|
+
if lag is not None:
|
|
1111
|
+
overlay, notice = _lag_overlay(root, lag)
|
|
1112
|
+
warnings += [text for text in (notice, _refresh_in_background(root, lag)) if text]
|
|
1113
|
+
else:
|
|
1114
|
+
try:
|
|
1115
|
+
_prepare_project_index(arguments, job_id, cfg, token, query_vector=next(iter(vectors.values())),
|
|
1116
|
+
model=model)
|
|
1117
|
+
except ProjectJobCancelled:
|
|
985
1118
|
raise
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
1119
|
+
except Exception as exc:
|
|
1120
|
+
if not indexer.info(root).get("chunks"):
|
|
1121
|
+
raise
|
|
1122
|
+
warnings.append("Index update failed; results from the last ready index. " + _project_error(exc))
|
|
1123
|
+
search_model = None
|
|
1124
|
+
if indexer.info(root).get("model_id") != model:
|
|
1125
|
+
vectors = {}
|
|
990
1126
|
if vectors and not cfg.get("rerank_model"):
|
|
991
1127
|
warnings.append("Rerank not configured: hybrid lexical/vector order, less precise at the top. Configure a rerank_model.")
|
|
992
1128
|
_job_checkpoint(job_id)
|
|
@@ -1015,7 +1151,7 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1015
1151
|
query_terms = _symbol_terms(" ".join(queries[group] for group in groups if group != "doc"))
|
|
1016
1152
|
_update_job(job_id, phase="searching")
|
|
1017
1153
|
found = {group: indexer.search(root, vectors.get(queries[group]), top_k=max(limits[group] * 4, candidate_pool),
|
|
1018
|
-
query=queries[group], model_id=search_model,
|
|
1154
|
+
query=queries[group], model_id=search_model, overlay=overlay,
|
|
1019
1155
|
include=lambda path, group=group: index_profile.kind(path, profile) == group)
|
|
1020
1156
|
for group in groups}
|
|
1021
1157
|
|
|
@@ -1035,7 +1171,12 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1035
1171
|
ordered = [(float(item["relevance_score"]), "rerank", candidates[item["index"]][1:]) for item in reranked]
|
|
1036
1172
|
except Exception as exc:
|
|
1037
1173
|
problem = f"Rerank unavailable for {group}; hybrid lexical/vector order kept. " + _project_error(exc)
|
|
1038
|
-
|
|
1174
|
+
exact = indexer.overlay_exact_keys(overlay, query)
|
|
1175
|
+
if exact:
|
|
1176
|
+
pinned = [(c[0], "overlay", c[1:]) for c in candidates if tuple(c[1:4]) in exact][:2]
|
|
1177
|
+
pinned_keys = {tuple(item[2][:3]) for item in pinned}
|
|
1178
|
+
ordered = pinned + [item for item in ordered if tuple(item[2][:3]) not in pinned_keys]
|
|
1179
|
+
rows =[{"path": path, "lines": f"{start}-{end}", "score": round(score, 6), "snippet": text[:500],
|
|
1039
1180
|
"ranking": ranking, "location_kind": document_text.location_kind(path),
|
|
1040
1181
|
**({"symbols": _chunk_symbols(symbol_index, path, start, end, text, query_terms)}
|
|
1041
1182
|
if symbol_index and group != "doc" else {})}
|
|
@@ -1050,6 +1191,8 @@ def _perform_smart_search_isolated(arguments, job_id):
|
|
|
1050
1191
|
if problem:
|
|
1051
1192
|
warnings.append(problem)
|
|
1052
1193
|
_job_checkpoint(job_id)
|
|
1194
|
+
if not any(blocks.values()):
|
|
1195
|
+
notes.append(EMPTY_RESULT_NOTICE)
|
|
1053
1196
|
warning = " ".join(warnings)
|
|
1054
1197
|
if warning:
|
|
1055
1198
|
_update_job(job_id, warning=warning)
|
|
@@ -1120,7 +1263,8 @@ def _run_project_job_in_view(job_id, arguments, kind):
|
|
|
1120
1263
|
if updated:
|
|
1121
1264
|
fields.update(indexed_seq=captured_seq, last_indexed=time.time())
|
|
1122
1265
|
project_store.update(project_id, **fields)
|
|
1123
|
-
_log(f"project_job {job_id} {project_id} {kind} done elapsed={time.time()-job['created_at']:.2f}s"
|
|
1266
|
+
_log(f"project_job {job_id} {project_id} {kind} done elapsed={time.time()-job['created_at']:.2f}s "
|
|
1267
|
+
f"phases={_phase_timings(final)}")
|
|
1124
1268
|
return result
|
|
1125
1269
|
except Exception as exc:
|
|
1126
1270
|
with _JOBS_LOCK:
|
|
@@ -1215,8 +1359,13 @@ def _search_limits(value, groups, saved=None):
|
|
|
1215
1359
|
for group in groups:
|
|
1216
1360
|
raw = value if not isinstance(value, dict) else value.get(group, (saved or {}).get(group, SEARCH_LIMIT_DEFAULT))
|
|
1217
1361
|
limits[group] = _limit_value(raw, f"top_k.{group}")
|
|
1218
|
-
|
|
1219
|
-
|
|
1362
|
+
total = sum(limits.values())
|
|
1363
|
+
if total > SEARCH_LIMIT_TOTAL:
|
|
1364
|
+
# A first call that asks for more than the total is scaled down in proportion (a rejection costs the agent a turn).
|
|
1365
|
+
scaled = {group: max(1, limit * SEARCH_LIMIT_TOTAL // total) for group, limit in limits.items()}
|
|
1366
|
+
while sum(scaled.values()) > SEARCH_LIMIT_TOTAL:
|
|
1367
|
+
scaled[max(scaled, key=scaled.get)] -= 1
|
|
1368
|
+
limits = scaled
|
|
1220
1369
|
return limits
|
|
1221
1370
|
|
|
1222
1371
|
|
|
@@ -3388,6 +3537,10 @@ def _pretool_hook(handler):
|
|
|
3388
3537
|
client = (urllib.parse.parse_qs(urllib.parse.urlsplit(handler.path).query).get("client") or [""])[0]
|
|
3389
3538
|
try:
|
|
3390
3539
|
payload = json.loads(handler._read_body(HOOK_MAX_BODY_BYTES) or b"{}")
|
|
3540
|
+
try:
|
|
3541
|
+
_note_edit(payload)
|
|
3542
|
+
except Exception as exc:
|
|
3543
|
+
_log(f"edit index update skipped: {type(exc).__name__}: {exc}")
|
|
3391
3544
|
output = hook_decision.decide(payload, client or None, _web_route)
|
|
3392
3545
|
except Exception as exc:
|
|
3393
3546
|
_log(f"PreToolUse hook without a decision: {type(exc).__name__}: {exc}")
|
package/version.py
CHANGED