@allansantos-dev/smart-tool 0.9.18 → 0.9.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,43 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.9.20 - beta
4
+
5
+ - The first search of a folder that has no index yet no longer waits for the index. It answers at once with the files
6
+ that hold a query term, read from disk and ranked by keyword (up to 8 s of scanning, and it says how many files it
7
+ reached), and builds the index in the background; the answer says "Indexing in progress: this folder has no index
8
+ yet". A folder with no saved scope and no model to decide it still asks the agent for the scope, as before.
9
+ - `smart_search_result` hands the agent the result text itself when the job is done, the same text a search that
10
+ finishes within the wait returns. Before, it was a JSON wrapper that escaped the text and repeated the index stats
11
+ and the git view (~1.5 thousand characters of metadata per answer). A job still pending or failed comes without
12
+ those fields too.
13
+ - Playwright MCP starts with `--caps=` so a `PLAYWRIGHT_MCP_CAPS=network` in the user's environment cannot turn on
14
+ `browser_route` (measured on 0.0.83: with the variable set, 29 tools including `browser_route`; with `--caps=`, 25
15
+ and no `browser_route`).
16
+ - `CODEX_HOME` is followed everywhere the Codex config is read or written: the MCP registration check, the endpoint
17
+ sync after a port change and the Codex hook file (before, only the browser control and the setup audit followed it).
18
+ - `affected_tests` links a Python test to the modules it imports inside a string, as in
19
+ `subprocess.run([sys.executable, "-c", "import click"])`, so a change to that module selects the test.
20
+
21
+ ## 0.9.19 - beta
22
+
23
+ - A search no longer waits for the index to catch up. It answers from the existing index at once, searches the files
24
+ edited since the last index by keyword straight from disk, and says "Indexing in progress: N file(s) changed..." in
25
+ the answer while the update runs in the background. An index that does not exist yet, or was built by another model,
26
+ is still built first.
27
+ - A search waits up to 25 s for its own result before answering with a `job_id` (it was 3 s, below the 3.3 s a warm
28
+ search takes at best, so nearly every call needed a second `smart_search_result`).
29
+ - The scope and the language profile are no longer asked of the model inside a search that has an index: the saved ones
30
+ are used and refreshed in the background.
31
+ - New folders are registered watched and eager: the index follows edits by itself. A PreToolUse Edit/Write of an eager
32
+ folder without a running watcher queues one debounced update with the edited paths. Folders already registered keep
33
+ their policy.
34
+ - A `top_k` whose total is above 20 is scaled down in proportion instead of rejecting the call (the rejection cost the
35
+ agent a turn on its first search).
36
+ - A search made right after an edit keeps the just-edited chunk in the answer: the reranker can no longer push it out.
37
+ - An edit whose path comes in the Windows 8.3 short form (`ALLANS~1`-style folder names) is matched to its registered
38
+ project; before, the project lookup compared it with the long form and found nothing.
39
+ - The daemon log prints the seconds spent in each phase of a job (`phases=queued:0.02,auth:0.01,...`).
40
+
3
41
  ## 0.9.18 - beta
4
42
 
5
43
  - Updates are no longer applied only at logon. The tray checks npm at start and every 24 h and installs a newer
package/README.md CHANGED
@@ -15,7 +15,7 @@ registered as the `playwright` server where this machine lets a browser be drive
15
15
  An optional hook routes the agent's native Grep/Glob/Read/Bash/WebSearch/WebFetch calls to these tools when they
16
16
  are cheaper, or just tells the agent what Smart Tool would do.
17
17
 
18
- **Status:** 0.9.18 beta. Windows only. What changed in each version: [CHANGELOG.md](CHANGELOG.md).
18
+ **Status:** 0.9.20 beta. Windows only. What changed in each version: [CHANGELOG.md](CHANGELOG.md).
19
19
 
20
20
  ## How it works
21
21
 
package/affected_tests.py CHANGED
@@ -33,6 +33,8 @@ NOTE = ("Selection from the indexed import graph: tests that load code through s
33
33
  "files and imports count before reindexing; pytest fixtures link a test to the conftest.py that defines them.")
34
34
  TEST_SOURCES = (".py", ".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".mts", ".cts", ".java")
35
35
  _PATH_LITERAL = re.compile(r"""['"`]((?:\.{0,2}/)?[\w.-]+(?:/[\w.-]+)*\.(?:py|js|mjs|cjs|ts|mts|cts|java))['"`]""")
36
+ _STRING_LITERAL = re.compile(r"""'([^'\n]*)'|"([^"\n]*)\"""")
37
+ _IMPORT_STATEMENT = re.compile(r"(?:^|;)\s*(?:import|from)\s+([A-Za-z_]\w*(?:\.\w+)*)")
36
38
  _HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@", re.M)
37
39
 
38
40
 
@@ -213,12 +215,7 @@ def _path_literal_users(root, test_files, known):
213
215
  if not test.lower().endswith(TEST_SOURCES):
214
216
  continue
215
217
  source = _read(os.path.join(root, test))
216
- folders, folder = [], os.path.dirname(test)
217
- while True:
218
- folders.append(folder)
219
- if not folder:
220
- break
221
- folder = os.path.dirname(folder)
218
+ folders = _folders_above(test)
222
219
  for literal in set(_PATH_LITERAL.findall(source)):
223
220
  for base in folders:
224
221
  candidate = posixpath.normpath(posixpath.join(base, literal.removeprefix("./")))
@@ -228,6 +225,37 @@ def _path_literal_users(root, test_files, known):
228
225
  return users
229
226
 
230
227
 
228
+ def _import_string_users(root, test_files, known):
229
+ """Python modules a test imports inside a string (subprocess.run([sys.executable, "-c", "import click"])), resolved
230
+ to the package or module file under the project root, a `src` folder or a folder above the test: the code runs in
231
+ another process, so no import statement of the test links it."""
232
+ users = defaultdict(set)
233
+ for test in test_files:
234
+ if not test.lower().endswith(".py"):
235
+ continue
236
+ folders = ["src", *_folders_above(test)]
237
+ literals = (single or double for single, double in _STRING_LITERAL.findall(_read(os.path.join(root, test))))
238
+ for module in {name for text in literals for name in _IMPORT_STATEMENT.findall(text)}:
239
+ stem = module.replace(".", "/")
240
+ for base in folders:
241
+ target = next((candidate for candidate in (posixpath.join(base, stem + ".py"),
242
+ posixpath.join(base, stem, "__init__.py"))
243
+ if candidate in known and candidate != test), None)
244
+ if target:
245
+ users[target].add(test)
246
+ break
247
+ return users
248
+
249
+
250
+ def _folders_above(path):
251
+ """The folder of `path` and every folder above it up to the project root (`""`), nearest first."""
252
+ folders, folder = [], os.path.dirname(path)
253
+ while folder:
254
+ folders.append(folder)
255
+ folder = os.path.dirname(folder)
256
+ return folders + [""]
257
+
258
+
231
259
  _TEST_CASES = {
232
260
  ".py": re.compile(r"^\s*(async\s+)?def\s+test\w*\s*\(|^\s*class\s+Test\w*|unittest\.TestCase", re.M),
233
261
  ".java": re.compile(r"@(Test|ParameterizedTest|RepeatedTest|TestFactory|TestTemplate|RunWith|Suite)\b|"
@@ -296,8 +324,10 @@ def affected(root, base="HEAD", limit=30, view_id=None):
296
324
  callers[call["target"]].add(call["source"])
297
325
  for conftest, users in _fixture_users(root, test_files).items():
298
326
  importers[conftest] |= users
299
- for target, users in _path_literal_users(root, test_files, {f["path"] for f in data.get("files") or []}).items():
300
- importers[target] |= users
327
+ known_files = {f["path"] for f in data.get("files") or []}
328
+ for linked in (_path_literal_users(root, test_files, known_files), _import_string_users(root, test_files, known_files)):
329
+ for target, users in linked.items():
330
+ importers[target] |= users
301
331
  by_id = {s["id"]: s for s in data.get("symbols") or []}
302
332
  run_all, not_analyzed, changed_code, changed_tests, conftests = [], [], [], [], []
303
333
  for path, change in sorted(changes.items()):
@@ -0,0 +1,13 @@
1
+ ---
2
+ name: smart-scout
3
+ description: Optional helper for the Smart Tool. Use it only for a task that chains several Smart Tool calls (search, then graph, then affected_tests) or while the index is still loading. A plain search is faster called directly.
4
+ model: haiku
5
+ tools: mcp__smart-tool__smart_search, mcp__smart-tool__smart_search_result, mcp__smart-tool__project_manage, Read
6
+ ---
7
+ You run Smart Tool calls for another session and return only the result.
8
+
9
+ Use smart_search with project_root equal to the working directory (forward slashes), query_identifiers in English identifier terms, query_comments in the language of the comments, and top_k {"code": 6, "test": 2, "doc": 2}. When it answers pending, call smart_search_result with the job_id until it finishes. project_manage action=graph (symbol) and action=affected_tests are yours to chain when the task asks for them. Open at most 3 files with Read and make at most 8 tool calls in total.
10
+
11
+ Reply ONLY with a list of up to 8 items, each `path:first_line-last_line: reason in up to 15 words`, no code excerpts and no introduction.
12
+
13
+ If nothing relevant came back after two different queries, reply exactly `EMPTY: <the two queries you tried>`. Never invent a path. Never use Grep, shell search or Read of folders to compensate: an empty answer is a valid answer, and the session that called you decides whether to retry with smart_search or to report that the code was not found.
@@ -62,7 +62,7 @@ def server_command():
62
62
  raise RuntimeError("Supported Node.js not found for Playwright MCP.")
63
63
  if not CLI.is_file():
64
64
  raise RuntimeError(f"Playwright MCP is not installed ({CLI}); reinstall Smart Tool.")
65
- return [node, str(CLI), "--isolated", "--headless", "--no-webmcp", "--config", write_config(),
65
+ return [node, str(CLI), "--isolated", "--headless", "--no-webmcp", "--caps=", "--config", write_config(),
66
66
  "--executable-path", chromium_executable(), "--output-dir", OUTPUT_DIR,
67
67
  "--output-max-size", str(OUTPUT_MAX_BYTES)]
68
68
 
@@ -159,7 +159,7 @@ def registered():
159
159
 
160
160
  def _codex_config():
161
161
  """Codex keeps config.toml in CODEX_HOME when set, else ~/.codex."""
162
- return os.path.join(os.environ.get("CODEX_HOME") or os.path.join(os.path.expanduser("~"), ".codex"), "config.toml")
162
+ return os.path.join(endpoint_sync.codex_dir(), "config.toml")
163
163
 
164
164
 
165
165
  def _replace_text(path, text):
package/client_hooks.py CHANGED
@@ -21,7 +21,7 @@ PREVIEW_TTL_S = 600
21
21
  CLIENTS = {
22
22
  "claude": {"label": "Claude Code", "names": {"claude-code"}, "files": [os.path.join(endpoint_sync.claude_dir(), "settings.json")],
23
23
  "docs": "https://code.claude.com/docs/en/hooks", "event": "PreToolUse", "matcher": "Grep|Glob|Read|Bash|PowerShell|WebSearch|WebFetch|Edit|MultiEdit|Write|mcp__smart-tool__smart_search", "http": True},
24
- "codex": {"label": "Codex", "prefix": "codex", "files": [os.path.join(HOME, ".codex", "hooks.json"), os.path.join(HOME, ".codex", "config.toml")],
24
+ "codex": {"label": "Codex", "prefix": "codex", "files": [os.path.join(endpoint_sync.codex_dir(), "hooks.json"), os.path.join(endpoint_sync.codex_dir(), "config.toml")],
25
25
  "docs": "https://learn.chatgpt.com/docs/hooks", "event": "PreToolUse", "matcher": "Bash|apply_patch",
26
26
  "after_install": "Codex only runs new hooks after the user approves the definition in /hooks."},
27
27
  "cursor": {"label": "Cursor", "prefix": "cursor", "files": [os.path.join(HOME, ".cursor", "hooks.json")],
package/endpoint_sync.py CHANGED
@@ -27,6 +27,11 @@ def claude_dir():
27
27
  return os.environ.get("CLAUDE_CONFIG_DIR") or _home(".claude")
28
28
 
29
29
 
30
+ def codex_dir():
31
+ """Codex keeps config.toml and hooks.json in CODEX_HOME when set, else ~/.codex."""
32
+ return os.environ.get("CODEX_HOME") or _home(".codex")
33
+
34
+
30
35
  def claude_json():
31
36
  """User-scope MCP servers: $CLAUDE_CONFIG_DIR/.claude.json when set, else ~/.claude.json."""
32
37
  custom = os.environ.get("CLAUDE_CONFIG_DIR")
@@ -72,7 +77,7 @@ def _codex_mcp(text, base):
72
77
 
73
78
  def targets():
74
79
  return ((claude_json(), _claude_mcp), (os.path.join(claude_dir(), "settings.json"), _claude_hook),
75
- (_home(".codex", "config.toml"), _codex_mcp))
80
+ (os.path.join(codex_dir(), "config.toml"), _codex_mcp))
76
81
 
77
82
 
78
83
  _CODEX_ANY_URL_RE = re.compile(r'\[mcp_servers\.smart-tool\][^\[]*?\burl\s*=\s*"([^"]+)"')
@@ -93,7 +98,7 @@ def registrations():
93
98
  except (OSError, AttributeError):
94
99
  break
95
100
  try:
96
- with open(_home(".codex", "config.toml"), encoding="utf-8") as stream:
101
+ with open(os.path.join(codex_dir(), "config.toml"), encoding="utf-8") as stream:
97
102
  match = _CODEX_ANY_URL_RE.search(stream.read())
98
103
  found["codex"] = match.group(1) if match else None
99
104
  except OSError:
package/hook_decision.py CHANGED
@@ -171,10 +171,8 @@ def advise(reason):
171
171
 
172
172
 
173
173
  def _project_of(path):
174
- target = os.path.normcase(os.path.abspath(path))
175
- roots = [p["root"] for p in project_store.all_projects()
176
- if target.startswith(os.path.normcase(os.path.abspath(p["root"])) + os.sep)]
177
- return max(roots, key=len) if roots else None
174
+ project = project_store.containing(path)
175
+ return project["root"] if project else None
178
176
 
179
177
 
180
178
  def _impact_line(root, rel, name):
package/indexer.py CHANGED
@@ -779,9 +779,14 @@ def _vector_rows(root, path):
779
779
  return rows, meta, norms
780
780
 
781
781
 
782
+ def _query_terms(query):
783
+ """Up to 24 distinct word terms of the query, in order."""
784
+ return list(dict.fromkeys(re.findall(r"[\w]{2,}", query or "", re.UNICODE)))[:24]
785
+
786
+
782
787
  def _lexical_scored(path, query, rows, limit, offset=0):
783
788
  """(chunk_id, score) best first; higher score is a better lexical match."""
784
- terms = list(dict.fromkeys(re.findall(r"[\w]{2,}", query or "", re.UNICODE)))[:24]
789
+ terms = _query_terms(query)
785
790
  if not terms:
786
791
  return []
787
792
  expression = " OR ".join('"' + term.replace('"', '""') + '"' for term in terms)
@@ -808,11 +813,101 @@ def _min_max(pairs):
808
813
  return {key: (value - low) / (high - low) if high > low else 1.0 for key, value in pairs}
809
814
 
810
815
 
811
- def search(root, query_vector=None, top_k=20, query=None, model_id=None, include=None):
812
- """Recuperação vetorial + lexical por fusão de notas relativas, sem rede."""
816
+ def stale_files(root, included_paths):
817
+ """`(changed, removed)` against the indexed state: `changed` lists in-scope files whose `(mtime_ns, size)` differs
818
+ from the stored one (or that were never indexed), `removed` lists indexed files that left the scope. It only
819
+ stats files, so it is the cheap answer to "what would a reindex have to touch"."""
820
+ snapshot = manifest_snapshot(root)
821
+ changed, removed = [], set(snapshot)
822
+ for rel in sorted(set(included_paths)):
823
+ stat, _reason = _stat_candidate(root, rel)
824
+ if stat is None:
825
+ continue
826
+ removed.discard(rel)
827
+ if snapshot.get(rel) != stat:
828
+ changed.append(rel)
829
+ return changed, sorted(removed)
830
+
831
+
832
+ def overlay_chunks(root, rel_paths):
833
+ """`[(path, start_line, end_line, text)]` read from disk right now for the given files, chunked like the index.
834
+ Files that cannot be read consistently are skipped: the overlay only complements the index, it never fails a
835
+ search."""
836
+ chunks = []
837
+ for rel in rel_paths:
838
+ data, encoding, _stat = _stable_read(root, rel)
839
+ if data is None:
840
+ continue
841
+ pieces = chunk_file(root, rel, CHUNK_LINES, data=data, encoding=encoding)
842
+ chunks.extend((rel, start, end, text) for start, end, text in pieces or [])
843
+ return chunks
844
+
845
+
846
+ def keyword_chunks(root, rel_paths, queries, budget_s):
847
+ """`(chunks, scanned)`: the chunks, read from disk right now, of the files among `rel_paths` that contain at least
848
+ one term of `queries` (the file name counts), for a project that has no index yet. The scan stops after `budget_s`
849
+ seconds; `scanned` is how many files it reached. Reading and skipping rules are the index's (`_stable_read`)."""
850
+ terms = [term.casefold() for term in dict.fromkeys(term for query in queries for term in _query_terms(query))]
851
+ deadline = time.monotonic() + budget_s
852
+ chunks, scanned = [], 0
853
+ for rel in rel_paths:
854
+ if time.monotonic() > deadline:
855
+ break
856
+ scanned += 1
857
+ data, encoding, _stat = _stable_read(root, rel)
858
+ if data is None:
859
+ continue
860
+ try:
861
+ text = data.decode(encoding or "utf-8", "ignore").casefold()
862
+ except LookupError:
863
+ continue
864
+ name = rel.casefold()
865
+ if not any(term in text or term in name for term in terms):
866
+ continue
867
+ pieces = chunk_file(root, rel, CHUNK_LINES, data=data, encoding=encoding)
868
+ chunks.extend((rel, start, end, piece) for start, end, piece in pieces or [])
869
+ return chunks, scanned
870
+
871
+
872
+ def _overlay_scored(chunks, query, alpha):
873
+ """`[(score, path, start, end, text)]` for overlay chunks matching the query terms by keyword. A chunk that holds
874
+ every term scores 1.0, the top of the fused index scores (`search` ranks it first on a tie): the edited file is read
875
+ from disk, so it is the current answer for an exact match. A partial match stays on the 0-1 scale of the lexical
876
+ share of the fused score."""
877
+ terms = _query_terms(query)
878
+ scored = []
879
+ for path, start, end, text in chunks:
880
+ haystack = (path + " " + text).casefold()
881
+ matched = sum(term.casefold() in haystack for term in terms)
882
+ if matched:
883
+ score = 1.0 if matched == len(terms) else (1 - alpha) * matched / len(terms)
884
+ scored.append((score, path, start, end, text))
885
+ return scored
886
+
887
+
888
+ def overlay_exact_keys(overlay, query):
889
+ """`{(path, start, end)}` of the overlay chunks that hold every query term. A reranker must not push them below
890
+ indexed chunks: they come from the file as it is on disk now."""
891
+ if not overlay or not overlay["chunks"] or not query:
892
+ return set()
893
+ return {(path, start, end) for score, path, start, end, _text in _overlay_scored(overlay["chunks"], query, 0.0)
894
+ if score == 1.0}
895
+
896
+
897
+ def search(root, query_vector=None, top_k=20, query=None, model_id=None, include=None, overlay=None):
898
+ """Vector plus lexical retrieval by fusion of relative scores, without network. `overlay` is
899
+ `{"drop": paths, "chunks": overlay_chunks(...)}`: the indexed chunks of the `drop` paths (stale or removed
900
+ files) are ignored and the overlay chunks, read from disk, take part by keyword instead."""
813
901
  path = existing_db_path(root)
814
902
  if not path:
815
- return []
903
+ if not (overlay and overlay["chunks"] and query):
904
+ return []
905
+ extra = _overlay_scored([chunk for chunk in overlay["chunks"] if include is None or include(chunk[0])], query, 0.0)
906
+ return sorted(extra, key=lambda item: item[0], reverse=True)[:top_k]
907
+ kind_filter = include
908
+ if overlay and overlay["drop"]:
909
+ dropped = set(overlay["drop"])
910
+ include = lambda rel: rel not in dropped and (kind_filter is None or kind_filter(rel))
816
911
  rows, meta, norms = _vector_rows(root, path)
817
912
  if include is not None:
818
913
  rows = [row for row in rows if include(row[1])]
@@ -854,7 +949,14 @@ def search(root, query_vector=None, top_k=20, query=None, model_id=None, include
854
949
  else:
855
950
  scores, ids = vector_scores, vector_ids
856
951
  by_id = {row[0]: row for row in rows}
857
- return [(scores[chunk_id], *by_id[chunk_id][1:5]) for chunk_id in ids[:top_k]]
952
+ found = [(scores[chunk_id], *by_id[chunk_id][1:5]) for chunk_id in ids[:top_k]]
953
+ if overlay and overlay["chunks"]:
954
+ extra = _overlay_scored([chunk for chunk in overlay["chunks"] if kind_filter is None or kind_filter(chunk[0])],
955
+ query, FUSION_ALPHA if query_vector is not None else 0.0)
956
+ found = [item for _score, item in sorted(
957
+ [((item[0], 1), item) for item in extra] + [((item[0], 0), item) for item in found],
958
+ key=lambda pair: pair[0], reverse=True)][:top_k]
959
+ return found
858
960
 
859
961
 
860
962
  def graph_cache_path(view_path):
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@allansantos-dev/smart-tool",
3
- "version": "0.9.18",
3
+ "version": "0.9.20",
4
4
  "description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Allan Santos",
package/project_store.py CHANGED
@@ -40,8 +40,8 @@ def register(root, watch=None):
40
40
  row = db.execute("SELECT data FROM projects WHERE id=?", (key,)).fetchone()
41
41
  data = json.loads(row[0]) if row else {
42
42
  "id": key, "root": display, "name": os.path.basename(display) or display,
43
- "status": "registered", "watch": False, "paused": False, "enabled": False,
44
- "update_mode": "on_search",
43
+ "status": "registered", "watch": True, "paused": False, "enabled": False,
44
+ "update_mode": "eager",
45
45
  "dirty_seq": 0, "indexed_seq": 0, "created_at": time.time(),
46
46
  "last_indexed": None, "last_checked": None, "last_error": "",
47
47
  "last_job_id": None, "watch_backend": "off", "next_retry_at": 0,
@@ -60,6 +60,13 @@ def get(project_id):
60
60
  return json.loads(row[0]) if row else None
61
61
 
62
62
 
63
+ def containing(path):
64
+ """The registered project whose folder holds `path` (the deepest one when projects nest), or `None`."""
65
+ target = project_identity.canonical_root(path)
66
+ inside = [p for p in all_projects() if target.startswith(project_identity.canonical_root(p["root"]) + os.sep)]
67
+ return max(inside, key=lambda p: len(p["root"])) if inside else None
68
+
69
+
63
70
  def all_projects():
64
71
  with _LOCK, _db() as db:
65
72
  rows = db.execute("SELECT data FROM projects ORDER BY root COLLATE NOCASE").fetchall()
package/setup_audit.py CHANGED
@@ -64,7 +64,7 @@ def _home(*parts):
64
64
 
65
65
  def codex_dir():
66
66
  """Codex's home: CODEX_HOME when set, else ~/.codex."""
67
- return os.environ.get("CODEX_HOME") or _home(".codex")
67
+ return endpoint_sync.codex_dir()
68
68
 
69
69
 
70
70
  def _managed_settings():
@@ -172,7 +172,12 @@ _capabilities_cache = None
172
172
  _jobs = {}
173
173
  _JOBS_LOCK = threading.Lock()
174
174
  _MAX_JOBS_RETAINED = 200
175
- SMART_SEARCH_SYNC_WAIT_S = 3
175
+ # A warm search never took less than 3.3 s at its fastest 5% and the median is 8.6 s: waiting less answers pending to
176
+ # nearly every call and costs the agent extra smart_search_result round trips. Codex gives an MCP call 60 s by default.
177
+ SMART_SEARCH_SYNC_WAIT_S = 25
178
+ OVERLAY_MAX_FILES = 40
179
+ COLD_SCAN_BUDGET_S = 8
180
+ EDIT_INDEX_DELAY_S = 8
176
181
  # smart_search_result waits for the job this long before answering pending: answering at once had an agent poll the
177
182
  # same job 7 times in 16 s and then give up on Smart Tool (claude -p session, 2026-10-09; job durations median 6 s,
178
183
  # p75 10.7 s).
@@ -232,6 +237,8 @@ def _update_job(job_id, **fields):
232
237
  current = _jobs.get(job_id)
233
238
  if current is not None and current.get("status") == "pending":
234
239
  updated = {**current, **fields, "updated_at": time.time()}
240
+ if fields.get("phase") and fields["phase"] != current.get("phase"):
241
+ updated["phases"] = [*current.get("phases", []), [fields["phase"], time.time()]]
235
242
  _jobs[job_id] = updated
236
243
  if (fields.get("phase") != current.get("phase") or
237
244
  time.monotonic() - _PERSIST_LAST.get(job_id, 0) > .5 or fields.get("cancel_requested")):
@@ -241,6 +248,20 @@ def _update_job(job_id, **fields):
241
248
  _persist_project_job(job_id, persistent)
242
249
 
243
250
 
251
+ def _phase_timings(job):
252
+ """"phase:seconds" for each phase of a job in the order it ran; the last phase lasts until the job finished."""
253
+ marks = [("queued", job.get("created_at") or 0), *[tuple(mark) for mark in job.get("phases", [])]]
254
+ end = job.get("finished_at") or time.time()
255
+ spans = {}
256
+ for (name, start), following in zip(marks, [*marks[1:], ("", end)]):
257
+ spans[name] = spans.get(name, 0.0) + max(following[1] - start, 0.0)
258
+ return ",".join(f"{name}:{seconds:.2f}" for name, seconds in spans.items())
259
+
260
+
261
+ _EDIT_PENDING = {}
262
+ _EDIT_LOCK = threading.Lock()
263
+
264
+
244
265
  def _project_lock(root):
245
266
  key = project_identity.canonical_root(root)
246
267
  with _PROJECT_LOCKS_LOCK:
@@ -322,7 +343,10 @@ SERVER_INSTRUCTIONS = (
322
343
  "with query). Never retry the same search or download through Bash, python, node, PowerShell, curl or wget: that "
323
344
  "bypasses the rule the user set. Reading one known file, or searching inside one known file, is never blocked. "
324
345
  "Before changing a function, project_manage action=graph with symbol lists its callers and tests; after editing, "
325
- "project_manage action=affected_tests lists the tests to run."
346
+ "project_manage action=affected_tests lists the tests to run. When smart_search returns nothing, retry smart_search with "
347
+ "other terms; do not switch to Grep, Read of folders or shell search. A helper agent (smart-scout, a cheap model) is "
348
+ "optional: use it only for a task that chains several Smart Tool calls or while the index is still loading; a plain "
349
+ "search is faster called directly. A helper that returns empty means retry with the tool or report not found."
326
350
  )
327
351
 
328
352
  TOOLS = [
@@ -377,7 +401,10 @@ TOOLS = [
377
401
  "query_comments with the same request in the comments' language; when the profile shows two "
378
402
  "languages both are required, or explain in single_language_reason. Put documentation searches in "
379
403
  "doc_query. Code and test results list the likeliest functions as name:line in symbols. Tune top_k per block for the project (project_manage search_limits). Fast searches return results directly. If work takes longer than a few seconds, "
380
- "a pending job_id is returned; call smart_search_result with that ID until done."
404
+ "a pending job_id is returned; call smart_search_result with that ID until done. A warning that starts "
405
+ "with 'Indexing in progress' means the files named there changed after the last index: they were "
406
+ "searched by keyword from disk and the rest of the answer is current; repeat the search later for their "
407
+ "semantic ranking."
381
408
  ),
382
409
  "inputSchema": {
383
410
  "type": "object",
@@ -814,6 +841,133 @@ def _agent_scope_request(root, scope, problem):
814
841
  "(always included unless listed in exclude).")
815
842
 
816
843
 
844
+ def _profile_due(scope, scope_model):
845
+ """Whether the saved scope lacks a current language profile and the model may be asked for one now, that is, a
846
+ scope model is configured and the pause after a failed attempt has passed."""
847
+ failed_at = (scope.get("profile_error") or {}).get("at", 0)
848
+ return bool(scope_model and (scope.get("profile") or {}).get("version") != index_profile.PROFILE_VERSION
849
+ and time.time() - failed_at >= index_profile.RETRY_AFTER_S)
850
+
851
+
852
+ def _index_lag(root, model, cfg, problem, job_id):
853
+ """What a search can serve without waiting for the index. `None` when the existing index cannot be used as it is
854
+ (missing, built by another model, chunker or view, no saved scope, or a scope that needs the model while it is
855
+ unavailable): the caller then updates it before searching. Otherwise `{"changed", "removed", "refresh"}` with the
856
+ in-scope files that differ from the indexed state, the indexed files that left the scope, and whether a
857
+ background update is due (stale files, or a scope or profile the model has to refresh)."""
858
+ project = project_store.get(project_identity.project_id(root)) or {}
859
+ info = indexer.info(root)
860
+ view = index_views.describe(root)
861
+ scope = index_scope.load_scope(root)
862
+ if (not info.get("chunks") or info.get("model_id") != model or not index_scope.valid_scope(scope)
863
+ or info.get("chunker_version") != indexer.CHUNKER_VERSION
864
+ or info.get("schema_version") != indexer.SCHEMA_VERSION
865
+ or info.get("chunk_lines") != indexer.CHUNK_LINES
866
+ or (view.get("git") and info.get("view_id") != view.get("view_id"))
867
+ or project.get("scope_dirty")):
868
+ return None
869
+ scope_model = cfg.get("scope_model") or cfg.get("router_model")
870
+ rescan = index_scope.needs_rescan(root, scope)
871
+ if rescan and (not scope_model or problem):
872
+ return None
873
+ _update_job(job_id, phase="listing")
874
+ changed, removed = indexer.stale_files(root, index_scope.resolve_included_files(root, scope))
875
+ return {"changed": changed, "removed": removed,
876
+ "refresh": bool(changed or removed or rescan or _profile_due(scope, scope_model))}
877
+
878
+
879
+ def _cold_index(root, cfg, problem):
880
+ """Whether this search is the first one of a project with no index, and the index can be built in the background:
881
+ the gateway answers and the scope is saved or a model can decide it. Otherwise the search keeps its old course
882
+ (build first, or ask the agent for the scope)."""
883
+ if indexer.info(root).get("chunks") or problem:
884
+ return False
885
+ return bool(index_scope.valid_scope(index_scope.load_scope(root)) or cfg.get("scope_model") or cfg.get("router_model"))
886
+
887
+
888
+ def _cold_overlay(root, queries):
889
+ """`(overlay, warning)` for a project with no index: the in-scope files (the saved scope, else the whole folder minus
890
+ the always-excluded names) holding a query term, read from disk and ranked by keyword, plus the notice that the
891
+ index is being built. The scan stops after `COLD_SCAN_BUDGET_S`."""
892
+ scope = index_scope.load_scope(root)
893
+ files = index_scope.resolve_included_files(root, scope if index_scope.valid_scope(scope) else {"include": [""]})
894
+ chunks, scanned = indexer.keyword_chunks(root, files, queries, COLD_SCAN_BUDGET_S)
895
+ reached = f"{scanned} of {len(files)} files" if scanned < len(files) else f"all {len(files)} files"
896
+ warning = ("Indexing in progress: this folder has no index yet, so the results are keyword matches read straight "
897
+ f"from disk in {reached}, without semantic ranking. The index is being built in the background: repeat "
898
+ "the search in a few minutes for the full ranking.")
899
+ return {"drop": [], "chunks": chunks}, warning
900
+
901
+
902
+ def _lag_overlay(root, lag):
903
+ """`(overlay, warning)` for `indexer.search`: the stale files read from disk, searched by keyword in place of their
904
+ outdated chunks, and the notice telling the agent that indexing is in progress. `(None, "")` when the index is
905
+ current."""
906
+ stale = lag["changed"] + lag["removed"]
907
+ if not stale:
908
+ return None, ""
909
+ shown = lag["changed"][:OVERLAY_MAX_FILES]
910
+ skipped = len(lag["changed"]) - len(shown)
911
+ names = ", ".join(stale[:3]) + (f" (+{len(stale) - 3})" if len(stale) > 3 else "")
912
+ warning = (f"Indexing in progress: {len(stale)} file(s) changed after the last index ({names}). Results come from "
913
+ "the existing index; those files were searched by keyword straight from disk, without semantic "
914
+ "ranking" + (f", and {skipped} of them were not searched" if skipped else "") +
915
+ ". The index updates in the background: repeat the search later for the updated ranking.")
916
+ return {"drop": stale, "chunks": indexer.overlay_chunks(root, shown)}, warning
917
+
918
+
919
+ def _refresh_in_background(root, lag):
920
+ """Queues the index update that the search did not wait for. Returns the notice to show when it could not be
921
+ queued, `""` when it was queued or none was due."""
922
+ if not lag["refresh"]:
923
+ return ""
924
+ try:
925
+ _submit_project_job("project_index", {"project_root": root, "_origin": "search"})
926
+ except RuntimeError as exc:
927
+ return f"The background index update could not start ({exc}); the next search tries again."
928
+ return ""
929
+
930
+
931
+ def _note_edit(payload):
932
+ """Queues a debounced background index update for a file an agent is about to write, in an eager project whose
933
+ watcher is not running. The PreToolUse hook sees the edit before it lands, so the update waits
934
+ `EDIT_INDEX_DELAY_S` and groups the edits of a burst into one job with `changed_paths`."""
935
+ if not isinstance(payload, dict) or payload.get("tool_name") not in ("Edit", "MultiEdit", "Write"):
936
+ return
937
+ target = (payload.get("tool_input") or {}).get("file_path")
938
+ if not isinstance(target, str) or not os.path.isabs(target):
939
+ return
940
+ project = project_store.containing(target)
941
+ if (not project or not project.get("enabled") or project.get("paused") or project.get("update_mode") != "eager"
942
+ or project.get("watch_backend", "off") != "off"):
943
+ return
944
+ rel = os.path.relpath(project_identity.display_root(target), project["root"]).replace(os.sep, "/")
945
+ with _EDIT_LOCK:
946
+ pending = _EDIT_PENDING.get(project["id"])
947
+ if pending is not None:
948
+ pending.add(rel)
949
+ return
950
+ _EDIT_PENDING[project["id"]] = {rel}
951
+ timer = threading.Timer(EDIT_INDEX_DELAY_S, _flush_edits, args=(project["id"],))
952
+ timer.daemon = True
953
+ timer.start()
954
+
955
+
956
+ def _flush_edits(project_id):
957
+ """Submits the index update for the files `_note_edit` collected; a failure is logged because it runs on a timer
958
+ thread nobody waits for."""
959
+ with _EDIT_LOCK:
960
+ paths = _EDIT_PENDING.pop(project_id, set())
961
+ project = project_store.get(project_id)
962
+ if not paths or not project or project.get("paused") or not project.get("enabled"):
963
+ return
964
+ try:
965
+ _submit_project_job("project_index", {"project_root": project["root"], "changed_paths": sorted(paths),
966
+ "_origin": "edit"})
967
+ except Exception as exc:
968
+ _log(f"edit index update not queued: {type(exc).__name__}: {exc}")
969
+
970
+
817
971
  def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, preview=False, model=None):
818
972
  root = arguments["project_root"]
819
973
  project = project_store.get(project_identity.project_id(root)) or {}
@@ -835,9 +989,7 @@ def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, pre
835
989
  deadline=_job_deadline(job_id), cancel_check=checkpoint)
836
990
  _update_job(job_id, phase="listing")
837
991
  included = index_scope.resolve_included_files(root, scope, cancel_check=checkpoint)
838
- failed_at = (scope.get("profile_error") or {}).get("at", 0)
839
- if ((scope.get("profile") or {}).get("version") != index_profile.PROFILE_VERSION
840
- and time.time() - failed_at >= index_profile.RETRY_AFTER_S and scope_model):
992
+ if _profile_due(scope, scope_model):
841
993
  _update_job(job_id, phase="profile")
842
994
  try:
843
995
  profile = index_profile.decide(root, included, scope_model, deadline=_job_deadline(job_id))
@@ -918,6 +1070,8 @@ def _grouped_yaml(profile, blocks, warning):
918
1070
 
919
1071
  LOCAL_EMBEDDER_DOWN_WARNING = ("The mcp-memory local embedder is configured but not responding: index and search are lexical only "
920
1072
  "until it is back (check the mcp-memory embedding service).")
1073
+ EMPTY_RESULT_NOTICE = ("No matches. Do not fall back to Grep, Read of folders or shell search: retry smart_search with other "
1074
+ "identifier terms, another group, or a wider top_k, and if it stays empty report that the code was not found.")
921
1075
  LEXICAL_WARNING = ("No embedding model: index and search are lexical only (exact words, no synonyms or cross-language matches). "
922
1076
  "Configure an embedding_model for semantic search.")
923
1077
 
@@ -939,6 +1093,7 @@ def _perform_smart_search_isolated(arguments, job_id):
939
1093
  model = local_embedder.resolve(cfg.get("embedding_model"), indexer.info(root).get("model_id"))
940
1094
  warnings = []
941
1095
  vectors = {}
1096
+ overlay = None
942
1097
  search_model = model
943
1098
  if problem:
944
1099
  model_client.OFFLINE.set(problem)
@@ -955,6 +1110,10 @@ def _perform_smart_search_isolated(arguments, job_id):
955
1110
  raise
956
1111
  warnings.append("Lexical index not updated; results from the last ready index. " + _project_error(exc))
957
1112
  search_model = None
1113
+ elif _cold_index(root, cfg, problem):
1114
+ overlay, notice = _cold_overlay(root, texts)
1115
+ search_model = None
1116
+ warnings += [text for text in (notice, _refresh_in_background(root, {"refresh": True})) if text]
958
1117
  elif problem and not local_embedder.is_local(model):
959
1118
  if not indexer.info(root).get("chunks"):
960
1119
  raise RuntimeError(problem)
@@ -975,18 +1134,23 @@ def _perform_smart_search_isolated(arguments, job_id):
975
1134
  warnings.append("Embedding unavailable; lexical results from the last ready index. The update was not confirmed. " + _project_error(exc))
976
1135
  search_model = None
977
1136
  if vectors:
978
- try:
979
- _prepare_project_index(arguments, job_id, cfg, token, query_vector=next(iter(vectors.values())),
980
- model=model)
981
- except ProjectJobCancelled:
982
- raise
983
- except Exception as exc:
984
- if not indexer.info(root).get("chunks"):
1137
+ lag = _index_lag(root, model, cfg, problem, job_id)
1138
+ if lag is not None:
1139
+ overlay, notice = _lag_overlay(root, lag)
1140
+ warnings += [text for text in (notice, _refresh_in_background(root, lag)) if text]
1141
+ else:
1142
+ try:
1143
+ _prepare_project_index(arguments, job_id, cfg, token, query_vector=next(iter(vectors.values())),
1144
+ model=model)
1145
+ except ProjectJobCancelled:
985
1146
  raise
986
- warnings.append("Index update failed; results from the last ready index. " + _project_error(exc))
987
- search_model = None
988
- if indexer.info(root).get("model_id") != model:
989
- vectors = {}
1147
+ except Exception as exc:
1148
+ if not indexer.info(root).get("chunks"):
1149
+ raise
1150
+ warnings.append("Index update failed; results from the last ready index. " + _project_error(exc))
1151
+ search_model = None
1152
+ if indexer.info(root).get("model_id") != model:
1153
+ vectors = {}
990
1154
  if vectors and not cfg.get("rerank_model"):
991
1155
  warnings.append("Rerank not configured: hybrid lexical/vector order, less precise at the top. Configure a rerank_model.")
992
1156
  _job_checkpoint(job_id)
@@ -1015,7 +1179,7 @@ def _perform_smart_search_isolated(arguments, job_id):
1015
1179
  query_terms = _symbol_terms(" ".join(queries[group] for group in groups if group != "doc"))
1016
1180
  _update_job(job_id, phase="searching")
1017
1181
  found = {group: indexer.search(root, vectors.get(queries[group]), top_k=max(limits[group] * 4, candidate_pool),
1018
- query=queries[group], model_id=search_model,
1182
+ query=queries[group], model_id=search_model, overlay=overlay,
1019
1183
  include=lambda path, group=group: index_profile.kind(path, profile) == group)
1020
1184
  for group in groups}
1021
1185
 
@@ -1035,7 +1199,12 @@ def _perform_smart_search_isolated(arguments, job_id):
1035
1199
  ordered = [(float(item["relevance_score"]), "rerank", candidates[item["index"]][1:]) for item in reranked]
1036
1200
  except Exception as exc:
1037
1201
  problem = f"Rerank unavailable for {group}; hybrid lexical/vector order kept. " + _project_error(exc)
1038
- rows = [{"path": path, "lines": f"{start}-{end}", "score": round(score, 6), "snippet": text[:500],
1202
+ exact = indexer.overlay_exact_keys(overlay, query)
1203
+ if exact:
1204
+ pinned = [(c[0], "overlay", c[1:]) for c in candidates if tuple(c[1:4]) in exact][:2]
1205
+ pinned_keys = {tuple(item[2][:3]) for item in pinned}
1206
+ ordered = pinned + [item for item in ordered if tuple(item[2][:3]) not in pinned_keys]
1207
+ rows =[{"path": path, "lines": f"{start}-{end}", "score": round(score, 6), "snippet": text[:500],
1039
1208
  "ranking": ranking, "location_kind": document_text.location_kind(path),
1040
1209
  **({"symbols": _chunk_symbols(symbol_index, path, start, end, text, query_terms)}
1041
1210
  if symbol_index and group != "doc" else {})}
@@ -1050,6 +1219,8 @@ def _perform_smart_search_isolated(arguments, job_id):
1050
1219
  if problem:
1051
1220
  warnings.append(problem)
1052
1221
  _job_checkpoint(job_id)
1222
+ if not any(blocks.values()):
1223
+ notes.append(EMPTY_RESULT_NOTICE)
1053
1224
  warning = " ".join(warnings)
1054
1225
  if warning:
1055
1226
  _update_job(job_id, warning=warning)
@@ -1120,7 +1291,8 @@ def _run_project_job_in_view(job_id, arguments, kind):
1120
1291
  if updated:
1121
1292
  fields.update(indexed_seq=captured_seq, last_indexed=time.time())
1122
1293
  project_store.update(project_id, **fields)
1123
- _log(f"project_job {job_id} {project_id} {kind} done elapsed={time.time()-job['created_at']:.2f}s")
1294
+ _log(f"project_job {job_id} {project_id} {kind} done elapsed={time.time()-job['created_at']:.2f}s "
1295
+ f"phases={_phase_timings(final)}")
1124
1296
  return result
1125
1297
  except Exception as exc:
1126
1298
  with _JOBS_LOCK:
@@ -1215,8 +1387,13 @@ def _search_limits(value, groups, saved=None):
1215
1387
  for group in groups:
1216
1388
  raw = value if not isinstance(value, dict) else value.get(group, (saved or {}).get(group, SEARCH_LIMIT_DEFAULT))
1217
1389
  limits[group] = _limit_value(raw, f"top_k.{group}")
1218
- if sum(limits.values()) > SEARCH_LIMIT_TOTAL:
1219
- raise ValueError(f"The top_k total across the requested blocks is {sum(limits.values())}; the maximum is {SEARCH_LIMIT_TOTAL}.")
1390
+ total = sum(limits.values())
1391
+ if total > SEARCH_LIMIT_TOTAL:
1392
+ # A first call that asks for more than the total is scaled down in proportion (a rejection costs the agent a turn).
1393
+ scaled = {group: max(1, limit * SEARCH_LIMIT_TOTAL // total) for group, limit in limits.items()}
1394
+ while sum(scaled.values()) > SEARCH_LIMIT_TOTAL:
1395
+ scaled[max(scaled, key=scaled.get)] -= 1
1396
+ limits = scaled
1220
1397
  return limits
1221
1398
 
1222
1399
 
@@ -1285,13 +1462,13 @@ def _handle_smart_search(arguments):
1285
1462
  job_id, future = _submit_project_job("smart_search", arguments)
1286
1463
  if future is None:
1287
1464
  _metric("pending", reason="queued")
1288
- return _to_json_sanitized(_smart_search_job_payload(job_id))
1465
+ return _agent_search_answer(_smart_search_job_payload(job_id))
1289
1466
  try:
1290
1467
  result = future.result(timeout=SMART_SEARCH_SYNC_WAIT_S)
1291
1468
  except concurrent.futures.TimeoutError:
1292
1469
  if not future.done():
1293
1470
  _metric("pending", reason="sync_timeout")
1294
- return _to_json_sanitized(_smart_search_job_payload(job_id))
1471
+ return _agent_search_answer(_smart_search_job_payload(job_id))
1295
1472
  result = future.result()
1296
1473
  _mark_notice_delivered(job_id)
1297
1474
  return result
@@ -1397,9 +1574,27 @@ def _handle_smart_search_result(arguments):
1397
1574
  time.sleep(0.25)
1398
1575
  payload = _smart_search_job_payload(job_id)
1399
1576
  _mark_notice_delivered(job_id)
1577
+ if payload["kind"] == "smart_search":
1578
+ return _agent_search_answer(payload)
1400
1579
  return _to_json_sanitized(payload)
1401
1580
 
1402
1581
 
1582
+ _AGENT_SEARCH_DROPPED = ("kind", "project_id", "updated_at", "stats", "view")
1583
+
1584
+
1585
+ def _agent_search_answer(payload):
1586
+ """What smart_search_result hands the agent for a smart_search job: the result text itself when done (the same text
1587
+ a search that finishes within the wait returns, no JSON wrapper that escapes it and repeats the index stats and
1588
+ the git view the setup page shows), otherwise the payload without those fields."""
1589
+ if payload["status"] == "done":
1590
+ warning = payload.get("warning")
1591
+ return f"{payload['result']}\nwarning: {warning}" if warning else payload["result"]
1592
+ slim = {key: value for key, value in payload.items() if key not in _AGENT_SEARCH_DROPPED}
1593
+ if not slim.get("cancel_requested"):
1594
+ slim.pop("cancel_requested", None)
1595
+ return _to_json_sanitized(slim)
1596
+
1597
+
1403
1598
  def _project_payload(project):
1404
1599
  data = dict(project)
1405
1600
  data['dismissed_duplicates'] = len(data.get('dismissed_duplicates') or {})
@@ -3388,6 +3583,10 @@ def _pretool_hook(handler):
3388
3583
  client = (urllib.parse.parse_qs(urllib.parse.urlsplit(handler.path).query).get("client") or [""])[0]
3389
3584
  try:
3390
3585
  payload = json.loads(handler._read_body(HOOK_MAX_BODY_BYTES) or b"{}")
3586
+ try:
3587
+ _note_edit(payload)
3588
+ except Exception as exc:
3589
+ _log(f"edit index update skipped: {type(exc).__name__}: {exc}")
3391
3590
  output = hook_decision.decide(payload, client or None, _web_route)
3392
3591
  except Exception as exc:
3393
3592
  _log(f"PreToolUse hook without a decision: {type(exc).__name__}: {exc}")
package/version.py CHANGED
@@ -1,7 +1,7 @@
1
1
  """Single source of the Smart Tool version and of the User-Agent sent to public APIs."""
2
2
  import re
3
3
 
4
- VERSION = "0.9.18"
4
+ VERSION = "0.9.20"
5
5
  _CONTACT_RE = re.compile(r"^(?:[^\s@()<>;]+@[^\s@()<>;]+\.[^\s@()<>;]+|https?://[^\s()<>;]+)$")
6
6
 
7
7