@allansantos-dev/smart-tool 0.9.81 → 0.9.83

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +115 -0
  2. package/NOTICE +1 -1
  3. package/README.md +73 -14
  4. package/affected_tests.py +9 -8
  5. package/agent_redirect.py +32 -0
  6. package/autostart.py +32 -25
  7. package/browser_bridge.py +320 -0
  8. package/browser_control.py +76 -39
  9. package/browser_service_client.py +1 -7
  10. package/client_hooks.py +41 -42
  11. package/code_graph.py +66 -21
  12. package/code_graph_java.mjs +21 -12
  13. package/code_health.py +14 -9
  14. package/codex_config.py +3 -0
  15. package/commit_check.py +184 -0
  16. package/daemon_launcher.py +37 -5
  17. package/doc_candidates.py +63 -0
  18. package/doc_check.py +9 -3
  19. package/doc_extras.py +2 -3
  20. package/doc_insights.py +26 -2
  21. package/doc_prose.py +59 -11
  22. package/doc_registry.py +241 -0
  23. package/doc_scope.py +205 -39
  24. package/doc_turn.py +205 -0
  25. package/doc_validate.py +52 -9
  26. package/document_converter.py +4 -4
  27. package/document_read.py +19 -11
  28. package/document_url.py +107 -0
  29. package/edit_shape.py +119 -0
  30. package/endpoint_sync.py +3 -21
  31. package/engine_home.py +95 -0
  32. package/hook_call.py +133 -0
  33. package/hook_decision.py +142 -7
  34. package/hooks/pre_commit.py +45 -0
  35. package/hooks/pretool_router.py +36 -30
  36. package/hooks/turn_end.py +40 -0
  37. package/index_views.py +5 -0
  38. package/install.py +24 -9
  39. package/lazy_mcp_proxy.py +10 -9
  40. package/local_embedder.py +8 -1
  41. package/locales/pt-BR.json +1 -2
  42. package/model_breaker.py +6 -5
  43. package/package.json +2 -2
  44. package/process_limits.py +17 -3
  45. package/project_identity.py +19 -1
  46. package/project_scan.py +98 -0
  47. package/release_notes.py +125 -0
  48. package/rename_preview.py +95 -0
  49. package/requester.py +1 -1
  50. package/scout_access.py +48 -0
  51. package/shell_reads.py +67 -0
  52. package/smart_tool_daemon.py +245 -50
  53. package/tray.py +31 -0
  54. package/venv_launch.py +67 -0
  55. package/version.py +1 -1
  56. package/web_adapters/node/engine-home.mjs +42 -0
  57. package/web_adapters/node/package-lock.json +13 -0
  58. package/web_adapters/node/package.json +1 -0
  59. package/web_fetch_many.py +61 -0
@@ -19,8 +19,14 @@ import urllib.error
19
19
  import urllib.request
20
20
  from pathlib import Path
21
21
 
22
+ import venv_launch
23
+
22
24
  RUN_DIR = paths.DATA_DIR
23
25
  REGISTRY_PATH = os.path.join(RUN_DIR, "daemon.json")
26
+ LOG_PATH = os.path.join(RUN_DIR, "daemon.log")
27
+ SYNCHRONIZE = 0x00100000
28
+ PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
29
+ INFINITE = 0xFFFFFFFF
24
30
  LOCK_PATH = os.path.join(RUN_DIR, "daemon.lock")
25
31
  LOCK_STALE_SECONDS = 30
26
32
  UPDATE_MARKER = os.path.join(RUN_DIR, "updating")
@@ -85,12 +91,8 @@ def _spawn_daemon():
85
91
  executable = executable.with_name("python.exe")
86
92
  fd, launcher = tempfile.mkstemp(prefix="smart-tool-hidden-", suffix=".vbs")
87
93
  try:
88
- command = (f'Chr(34) & "{executable}" & Chr(34) & " " & '
89
- f'Chr(34) & "{DAEMON_SCRIPT}" & Chr(34)')
90
94
  with os.fdopen(fd, "w", encoding="utf-16") as stream:
91
- stream.write('Set sh = CreateObject("WScript.Shell")\n')
92
- stream.write(f'sh.CurrentDirectory = "{Path(DAEMON_SCRIPT).parent}"\n')
93
- stream.write(f'sh.Run {command}, 0, False\n')
95
+ stream.write(venv_launch.hidden_run_vbs(executable, DAEMON_SCRIPT, Path(DAEMON_SCRIPT).parent))
94
96
  result = subprocess.run(
95
97
  ["wscript.exe", "//B", "//NoLogo", launcher],
96
98
  stdin=subprocess.DEVNULL, capture_output=True, timeout=10,
@@ -170,6 +172,36 @@ def start_daemon_if_down():
170
172
  return False
171
173
 
172
174
 
175
+ def note_stop(message):
176
+ """Appends a stop line to the daemon log, for stops the daemon cannot write itself: the installer closing it and
177
+ the supervisor seeing it end without a word (killed, crashed)."""
178
+ os.makedirs(RUN_DIR, exist_ok=True)
179
+ with open(LOG_PATH, "a", encoding="utf-8") as stream:
180
+ stream.write(f"{time.strftime('%Y-%m-%d %H:%M:%S')} {message}\n")
181
+
182
+
183
+ def wait_for_exit(pid):
184
+ """Blocks until process `pid` ends and returns its exit code; None when it cannot be watched (already gone, not
185
+ ours to open, or not Windows), so the caller falls back to polling."""
186
+ if os.name != "nt" or not pid:
187
+ return None
188
+ import ctypes
189
+ from ctypes import wintypes
190
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
191
+ kernel32.OpenProcess.restype = wintypes.HANDLE
192
+ handle = kernel32.OpenProcess(SYNCHRONIZE | PROCESS_QUERY_LIMITED_INFORMATION, False, int(pid))
193
+ if not handle:
194
+ return None
195
+ try:
196
+ kernel32.WaitForSingleObject(wintypes.HANDLE(handle), INFINITE)
197
+ code = wintypes.DWORD()
198
+ if not kernel32.GetExitCodeProcess(wintypes.HANDLE(handle), ctypes.byref(code)):
199
+ return None
200
+ return code.value
201
+ finally:
202
+ kernel32.CloseHandle(wintypes.HANDLE(handle))
203
+
204
+
173
205
  def _wait_for_daemon(wait_seconds, poll_interval=0.1):
174
206
  deadline = time.time() + wait_seconds
175
207
  while time.time() < deadline:
@@ -0,0 +1,63 @@
1
+ """Feature candidates from the embeddings home for public functions no entry point reaches through the call graph.
2
+
3
+ The graph links a function to a feature when a call path exists; for a public function with no path at all (code the
4
+ framework calls, configuration, a handler), the cosine between its text (name, signature, first docstring line) and
5
+ each feature's text proposes at most MAX_CANDIDATES features. They are recorded as status `candidate` with their
6
+ score and shown as "possible" on both sides of the chain; the embedder never confirms a link (measured: right on the
7
+ first place 5 times in 11, margins of 0.01 to 0.03 between first and second).
8
+ """
9
+ import math
10
+
11
+ import local_embedder
12
+
13
+ MAX_CANDIDATES = 2
14
+ MAX_FLOW_NAMES = 15
15
+
16
+
17
+ def orphans(model, reached):
18
+ """Public code functions that are not an entry point and that no entry point's flow reaches.
19
+
20
+ Parameters: model (doc_scope._Model); reached (dict anchor -> features, from doc_scope.features).
21
+ """
22
+ return [anchor for anchor in model.public() if anchor not in reached]
23
+
24
+
25
+ def _function_text(model, anchor):
26
+ function = model.function_of[anchor]
27
+ return " ".join(part for part in (function["name"], function["signature"], function["doc"]) if part)
28
+
29
+
30
+ def _feature_text(model, entry):
31
+ names = [model.function_of[a]["name"] for a in model.flow(entry)[:MAX_FLOW_NAMES]]
32
+ return _function_text(model, entry) + (" uses " + ", ".join(names) if names else "")
33
+
34
+
35
+ def _cosine(a, b):
36
+ norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b))
37
+ return sum(x * y for x, y in zip(a, b)) / norm if norm else 0.0
38
+
39
+
40
+ def propose(model, reached, skip=()):
41
+ """Candidate link rows for the orphan functions not in `skip` (those that already have candidates).
42
+
43
+ Returns: list of link dicts (origin embedder, status candidate, score, hops None); [] when there is nothing to
44
+ propose or no embeddings home is installed. Raises RuntimeError when the home is announced but does not answer, and
45
+ lets the home's own errors through: a candidate is never invented without vectors.
46
+ """
47
+ todo = [anchor for anchor in orphans(model, reached) if anchor not in set(skip)]
48
+ entries = model.entry_points()
49
+ if not todo or not entries or not local_embedder.configured():
50
+ return []
51
+ info = local_embedder.info()
52
+ if not info:
53
+ raise RuntimeError("The embeddings home is announced but does not answer; no feature candidate was proposed.")
54
+ vectors = local_embedder.embed(info["model"], [_function_text(model, a) for a in todo]
55
+ + [_feature_text(model, e) for e in entries])
56
+ functions, features = vectors[:len(todo)], vectors[len(todo):]
57
+ rows = []
58
+ for anchor, vector in zip(todo, functions):
59
+ ranked = sorted(((_cosine(vector, feature), entry) for entry, feature in zip(entries, features)), reverse=True)
60
+ rows += [{"feature_id": entry, "fn_id": anchor, "layer": model.layer_of.get(model.function_of[anchor]["path"]),
61
+ "hops": None, "origin": "embedder", "score": round(score, 4), "status": "candidate"}
62
+ for score, entry in ranked[:MAX_CANDIDATES]]
63
+ return rows
package/doc_check.py CHANGED
@@ -54,9 +54,12 @@ def _python_functions(source):
54
54
  for child in ast.iter_child_nodes(node):
55
55
  if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
56
56
  doc = ast.get_docstring(child)
57
+ first = child.body[0] if child.body else None
58
+ doc_lines = (first.lineno, first.end_lineno) if doc is not None else None
57
59
  returns = f" -> {ast.unparse(child.returns)}" if child.returns else ""
58
60
  found.append({"name": f"{owner}{child.name}", "start": child.lineno, "end": child.end_lineno,
59
61
  "documented": doc is not None, "doc": _first_line(doc or ""),
62
+ "docstring": doc or "", "doc_lines": doc_lines,
60
63
  "decorators": [ast.unparse(d) for d in child.decorator_list], "bases": list(bases),
61
64
  "signature": f"{child.name}({ast.unparse(child.args)}){returns}",
62
65
  "required": public and not nested and _public_name(child.name)
@@ -174,7 +177,8 @@ def _braced_functions(path, source, symbols):
174
177
  annotations, first_line = _annotations(lines, start, end)
175
178
  required = _required(path, symbol, by_id.get(symbol.get("parent")), first_line, annotations)
176
179
  found.append({"name": symbol["name"], "start": start, "end": end, "documented": block is not None,
177
- "doc": _first_line(block or ""), "signature": _SPACE.sub(" ", header.split("{")[0]).strip(),
180
+ "doc": _first_line(block or ""), "docstring": block or "",
181
+ "signature": _SPACE.sub(" ", header.split("{")[0]).strip(),
178
182
  "required": required})
179
183
  return found
180
184
 
@@ -206,7 +210,8 @@ def _go_functions(path, source):
206
210
  def add(decl, name, short, required, doc, articles=False):
207
211
  documented = bool(doc) and _go_form(doc, short, articles)
208
212
  found.append({"name": name, "start": decl["start"], "end": decl["end"], "documented": documented,
209
- "doc": _first_line(doc or ""), "signature": decl["signature"], "required": required,
213
+ "doc": _first_line(doc or ""), "docstring": doc or "", "signature": decl["signature"],
214
+ "required": required,
210
215
  "kind": decl["kind"], "convention": "go", "malformed": bool(doc) and not documented})
211
216
 
212
217
  for decl in decls:
@@ -227,7 +232,8 @@ def _go_functions(path, source):
227
232
  if decl["block"] not in reported_blocks:
228
233
  reported_blocks.add(decl["block"])
229
234
  found.append({"name": exported, "start": decl["start"], "end": decl["end"], "documented": True,
230
- "doc": _first_line(decl["group_doc"]), "signature": "", "required": importable,
235
+ "doc": _first_line(decl["group_doc"]), "docstring": decl["group_doc"], "signature": "",
236
+ "required": importable,
231
237
  "kind": decl["kind"], "convention": "go", "malformed": False})
232
238
  continue
233
239
  if decl["grouped"] and not decl["doc"]:
package/doc_extras.py CHANGED
@@ -439,7 +439,7 @@ def _flows(model, entries):
439
439
  function = model.function_of[anchor]
440
440
  diagram, left, drawn = _sequence(model, anchor)
441
441
  lines += [f'<a id="{doc_scope._escape(anchor)}"></a>', f"## {doc_scope.entry_title(model, anchor, entries)}", "",
442
- f'<!-- smart-tool:symbol id="{doc_scope._escape(anchor)}" signature="{doc_scope.signature_hash(function["signature"])}" -->',
442
+ doc_scope.symbol_marker(anchor, model.stamps(anchor)),
443
443
  "", f"Entry point in `{function['path']}`: {doc_scope.entry_link(model, anchor, entries, '../functional.md')}.", ""]
444
444
  lines += diagram if drawn else ["No call from this entry point reaches another code function."]
445
445
  if left > 0:
@@ -459,8 +459,7 @@ def _features_tests(model, covering, entries):
459
459
  for anchor, tests in covering:
460
460
  function = model.function_of[anchor]
461
461
  shown = ", ".join(f"`{model.function_of[t]['name']}`" for t in tests[:3]) + (f" (+{len(tests) - 3} more)" if len(tests) > 3 else "")
462
- marker = (f'<a id="{doc_scope._escape(anchor)}"></a><!-- smart-tool:symbol id="{doc_scope._escape(anchor)}" '
463
- f'signature="{doc_scope.signature_hash(function["signature"])}" -->')
462
+ marker = f'<a id="{doc_scope._escape(anchor)}"></a>' + doc_scope.symbol_marker(anchor, model.stamps(anchor))
464
463
  lines.append(f"| {marker}{doc_scope.entry_link(model, anchor, entries, '../functional.md')} | {len(tests)} | "
465
464
  f"{shown or 'no test reaches it'} |")
466
465
  bare = [a for a, tests in covering if not tests]
package/doc_insights.py CHANGED
@@ -74,6 +74,20 @@ def history(root):
74
74
  return result
75
75
 
76
76
 
77
+ def cached_history(root):
78
+ """The commits `history` keeps in memory, only while the Git view it was read for still holds.
79
+
80
+ Parameters: root (str) project folder.
81
+ Returns: the commits list, or None when none was read yet, the view moved or the folder is not a Git repository. It
82
+ checks the view by a stat of its files and never starts a process, so it never runs `git log`.
83
+ """
84
+ view = index_views.cached_view(root)
85
+ saved = _histories.get(project_identity.canonical_root(root))
86
+ if view and view.get("git") and saved and saved[0] == view["stamp"] and not saved[1][1]:
87
+ return saved[1][0]
88
+ return None
89
+
90
+
77
91
  def _read_history(root):
78
92
  try:
79
93
  raw = affected_tests._git(root, "-c", "core.quotepath=false", "log", "--no-merges", f"-n{MAX_COMMITS}", "--relative",
@@ -108,13 +122,23 @@ def _lines(model, path):
108
122
  def hotspots(model, commits):
109
123
  """Rows (path, commits, lines, last change) of the code files ranked by commits times lines, highest first."""
110
124
  code = {p for p in model.files() if model.kind_of(p) == "code"}
125
+ return ranked_hotspots(commits, code.__contains__, lambda path: _lines(model, path))
126
+
127
+
128
+ def ranked_hotspots(commits, is_code, lines_of):
129
+ """The ranking `hotspots` returns, for callers that know the code files and their sizes without a document model.
130
+
131
+ Parameters: commits (list) as `history` returns; is_code (callable) tells whether a path is a code file; lines_of
132
+ (callable) the line count of a code file that appears in the history.
133
+ Returns: rows (path, commits, lines, last change), highest commits times lines first; files with no lines are left out.
134
+ """
111
135
  churn, last = Counter(), {}
112
136
  for commit in commits:
113
137
  for path in commit["files"]:
114
- if path in code:
138
+ if is_code(path):
115
139
  churn[path] += 1
116
140
  last.setdefault(path, commit["time"])
117
- rows = [(path, count, _lines(model, path), last[path]) for path, count in churn.items()]
141
+ rows = [(path, count, lines_of(path), last[path]) for path, count in churn.items()]
118
142
  return sorted((r for r in rows if r[2]), key=lambda r: (-r[1] * r[2], r[0]))
119
143
 
120
144
 
package/doc_prose.py CHANGED
@@ -208,20 +208,68 @@ def pending(root, model):
208
208
  for rel in doc_scope._generated_files(root):
209
209
  with open(os.path.join(root, doc_scope.DOCS_DIR, rel), encoding="utf-8") as stream:
210
210
  text = stream.read()
211
- for key, body in doc_scope.PROSE.findall(text):
212
- if body.strip():
213
- continue
214
- kind = kind_of(rel, key)
215
- context = (_module_context(model, key) if kind == "module" else _overview_context(model, entries)
216
- if kind == "overview" else _entry_context(model, key, entries, kind == "simple_entry"))
217
- stamp = None
218
- if kind in ("entry", "simple_entry"):
219
- function = model.function_of[key]
220
- stamp = doc_scope.code_stamp(model.read(function["path"]), function)
221
- found.append({"file": rel, "block": key, "kind": kind, "context": context, "stamp": stamp})
211
+ found += [_task(model, entries, rel, key) for key, body in doc_scope.PROSE.findall(text) if not body.strip()]
222
212
  return found
223
213
 
224
214
 
215
+ def _task(model, entries, rel, key):
216
+ """One prose task: the block's kind, the code context of its instruction and, for an entry point, the flow stamp
217
+ the paragraph will be written for."""
218
+ kind = kind_of(rel, key)
219
+ context = (_module_context(model, key) if kind == "module" else _overview_context(model, entries)
220
+ if kind == "overview" else _entry_context(model, key, entries, kind == "simple_entry"))
221
+ stamp = doc_scope.flow_stamp(model, key) if kind in ("entry", "simple_entry") else None
222
+ return {"file": rel, "block": key, "kind": kind, "context": context, "stamp": stamp}
223
+
224
+
225
+ def rewrite_stale(root, model, features, budget_s=BUDGET_S, call=None):
226
+ """Rewrite with the docs_model the written functional.md and simple.md paragraphs of `features` whose stamp is not
227
+ the feature's flow stamp now. A block is replaced only by a paragraph that passes the same checks as a new one; a
228
+ rejected or failed one keeps its old text and stays stale, reported with the reason.
229
+
230
+ Parameters: root (str); model (doc_scope._Model) of the current code; features (iterable of entry anchors);
231
+ budget_s (float); call (callable like model_client.structured) for tests.
232
+ Returns: dict with 'written' and 'failed' ({file, block, problem}) lists.
233
+ Raises ValueError when no docs_model is configured.
234
+ """
235
+ model_id = config.load_config().get(ROLE)
236
+ if not model_id:
237
+ raise ValueError(f"No {ROLE} configured: choose the writing model (Docs model) in the Smart Tool setup screen; "
238
+ "the stale paragraphs were left as they are.")
239
+ started, entries = time.monotonic(), model.entry_points()
240
+ current = {anchor: doc_scope.flow_stamp(model, anchor) for anchor in features if anchor in model.function_of}
241
+ pages = doc_scope._read_pages(root, ("functional.md", "simple.md"))
242
+ tasks = [_task(model, entries, item["file"], item["id"]) for item in doc_scope.stale_prose(pages, current)]
243
+ corpus = doc_validate.Corpus(model)
244
+ with usage_meter.job(root, index_views.describe(root), "docs"), ThreadPoolExecutor(max_workers=WORKERS) as pool:
245
+ usage_meter.phase("docs")
246
+ contexts = [contextvars.copy_context() for _task_item in tasks]
247
+ results = list(pool.map(lambda pair: pair[0].run(write_one, model_id, pair[1], corpus, started + budget_s, call),
248
+ zip(contexts, tasks)))
249
+ written = [(task, result["text"]) for task, result in zip(tasks, results) if result["text"]]
250
+ _replace(root, written)
251
+ return {"written": [f"{doc_scope.DOCS_DIR}/{t['file']} {t['block']}" for t, _text in written],
252
+ "failed": [{"file": f"{doc_scope.DOCS_DIR}/{t['file']}", "block": t["block"], "problem": r["problem"]}
253
+ for t, r in zip(tasks, results) if not r["text"]]}
254
+
255
+
256
+ def _replace(root, written):
257
+ """Put each accepted paragraph in place of its stale block, stamped with the flow it was written for."""
258
+ by_file = {}
259
+ for task, text in written:
260
+ by_file.setdefault(task["file"], []).append((task["block"], text, task["stamp"]))
261
+ for rel, blocks in by_file.items():
262
+ path = os.path.join(root, doc_scope.DOCS_DIR, rel)
263
+ with open(path, encoding="utf-8") as stream:
264
+ page = stream.read()
265
+ for key, text, stamp in blocks:
266
+ block = re.compile(re.escape(f"<!-- smart-tool:prose {key}") + r'(?: code="[0-9a-f]{12}")? -->\n.*?'
267
+ + re.escape("<!-- /smart-tool:prose -->"), re.S)
268
+ page = block.sub(lambda _m: f"{doc_scope.prose_open(key, stamp)}\n{text}\n<!-- /smart-tool:prose -->",
269
+ page, count=1)
270
+ atomic_io.write_text(path, page)
271
+
272
+
225
273
  def problems(kind, text, corpus):
226
274
  """Reasons a paragraph cannot go into a block of `kind`, in fixed words (the model's text is never quoted).
227
275
 
@@ -0,0 +1,241 @@
1
+ """Per-function documentation registry: a crash-safe queue of the functions whose technical entry no longer matches
2
+ the code, with the dates of each change, kept in its own SQLite file (journal_mode=WAL, synchronous=FULL).
3
+
4
+ The stamps written in the pages (doc_scope.symbol_marker) stay the truth: the registry is a queue and a set of dates,
5
+ rebuilt from the pages and the disk when it is lost. A pending row and the change observed for it are written in the
6
+ same transaction (outbox), and processing is idempotent: a function whose page stamps already match the code is
7
+ settled without rewriting anything, so a crash between a page rename and the registry commit is repaired on resume.
8
+
9
+ Statuses: current, pending_code, pending_link, pending_prose, in_progress, orphan.
10
+ """
11
+ import hashlib
12
+ import os
13
+ import sqlite3
14
+ import time
15
+
16
+ import paths
17
+ import project_identity
18
+
19
+ FILE = "doc_registry.db"
20
+ PENDING = ("pending_code", "pending_link", "pending_prose")
21
+ MAX_ATTEMPTS = 2
22
+ LEASE_S = 120
23
+
24
+
25
+ def database_path():
26
+ """The registry file in the daemon's data folder."""
27
+ return os.path.join(paths.DATA_DIR, FILE)
28
+
29
+
30
+ def connect():
31
+ """Open the registry with WAL and synchronous=FULL, the setting SQLite documents as durable across power loss.
32
+
33
+ Returns: sqlite3.Connection. Raises RuntimeError when SQLite refuses WAL, since the durability promise would not hold.
34
+ """
35
+ os.makedirs(paths.DATA_DIR, exist_ok=True)
36
+ conn = sqlite3.connect(database_path(), timeout=10)
37
+ mode = conn.execute("PRAGMA journal_mode=WAL").fetchone()[0]
38
+ if str(mode).lower() != "wal":
39
+ conn.close()
40
+ raise RuntimeError(f"The documentation registry {database_path()} refused journal_mode=WAL (got {mode}).")
41
+ conn.execute("PRAGMA synchronous=FULL")
42
+ conn.execute("CREATE TABLE IF NOT EXISTS projects (root TEXT PRIMARY KEY, suffix TEXT NOT NULL)")
43
+ return conn
44
+
45
+
46
+ def _suffix(root):
47
+ return hashlib.sha1(project_identity.canonical_root(root).encode("utf-8")).hexdigest()[:12]
48
+
49
+
50
+ def tables(conn, root):
51
+ """Create the project's tables when missing and return their names: (functions, files, links)."""
52
+ suffix = _suffix(root)
53
+ names = (f"function_doc_{suffix}", f"file_state_{suffix}", f"feature_link_{suffix}")
54
+ with conn:
55
+ conn.execute("INSERT OR IGNORE INTO projects (root, suffix) VALUES (?, ?)", (root, suffix))
56
+ conn.execute(f"""CREATE TABLE IF NOT EXISTS {names[0]} (
57
+ fn_id TEXT PRIMARY KEY, path TEXT NOT NULL, public INTEGER NOT NULL, has_docstring INTEGER NOT NULL,
58
+ body_hash TEXT, docstring_hash TEXT, signature_hash TEXT,
59
+ doc_body_hash TEXT, doc_docstring_hash TEXT, doc_signature_hash TEXT,
60
+ method_updated_at REAL, doc_updated_at REAL, status TEXT NOT NULL,
61
+ attempts INTEGER NOT NULL DEFAULT 0, last_error TEXT, lease_until REAL)""")
62
+ conn.execute(f"CREATE TABLE IF NOT EXISTS {names[1]} (path TEXT PRIMARY KEY, mtime_ns INTEGER, size INTEGER)")
63
+ conn.execute(f"""CREATE TABLE IF NOT EXISTS {names[2]} (
64
+ feature_id TEXT NOT NULL, fn_id TEXT NOT NULL, layer TEXT, hops INTEGER, origin TEXT NOT NULL,
65
+ score REAL, status TEXT NOT NULL, computed_at REAL NOT NULL, PRIMARY KEY (feature_id, fn_id))""")
66
+ return names
67
+
68
+
69
+ def projects(conn):
70
+ """Roots of the projects the registry tracks."""
71
+ return [row[0] for row in conn.execute("SELECT root FROM projects ORDER BY root")]
72
+
73
+
74
+ def recover(conn, root, now=None):
75
+ """Rows left in_progress by a stopped run whose lease expired go back to pending_code.
76
+
77
+ Returns: int rows recovered.
78
+ """
79
+ functions = tables(conn, root)[0]
80
+ with conn:
81
+ cursor = conn.execute(f"UPDATE {functions} SET status='pending_code', lease_until=NULL "
82
+ "WHERE status='in_progress' AND (lease_until IS NULL OR lease_until < ?)",
83
+ (now or time.time(),))
84
+ return cursor.rowcount
85
+
86
+
87
+ def changed_files(conn, root, candidates):
88
+ """Candidate project files whose mtime_ns or size differ from the last state recorded (or that were never seen).
89
+
90
+ Parameters: candidates (iterable of project-relative paths). Returns: dict path -> (mtime_ns, size), None for a
91
+ file gone from disk.
92
+ """
93
+ files = tables(conn, root)[1]
94
+ known = {path: (mtime, size) for path, mtime, size in conn.execute(f"SELECT path, mtime_ns, size FROM {files}")}
95
+ found = {}
96
+ for path in candidates:
97
+ try:
98
+ info = os.stat(os.path.join(root, path))
99
+ state = (info.st_mtime_ns, info.st_size)
100
+ except FileNotFoundError:
101
+ state = None
102
+ if known.get(path, ()) != state:
103
+ found[path] = state
104
+ return found
105
+
106
+
107
+ def observe(conn, root, functions_now, markers, file_states, gone=(), now=None):
108
+ """Record what the code is now against what the pages say, and the file states read, in one transaction: a row whose
109
+ code stamps differ from its page stamps becomes pending_code (its pending outbox entry), a matching one current.
110
+
111
+ Parameters: functions_now (dict fn_id -> {path, public, has_docstring, signature, body, docstring}); markers (dict
112
+ fn_id -> {signature, body, docstring} from the technical pages); file_states (dict path -> (mtime_ns, size) or
113
+ None); gone (iterable of fn_id whose function no longer exists: orphan).
114
+ Returns: list of fn_id now pending.
115
+ """
116
+ functions, files, _links = tables(conn, root)
117
+ now = now or time.time()
118
+ pending = []
119
+ with conn:
120
+ for fn_id, code in functions_now.items():
121
+ page = markers.get(fn_id) or {}
122
+ row = conn.execute(f"SELECT body_hash, status, attempts FROM {functions} WHERE fn_id=?", (fn_id,)).fetchone()
123
+ matches = not code["public"] or all(page.get(part) == code[part] for part in ("signature", "body", "docstring"))
124
+ changed = row is None or row[0] != code["body"]
125
+ status = "current" if matches else ("in_progress" if row and row[1] == "in_progress" else "pending_code")
126
+ attempts = 0 if matches or changed else (row[2] if row else 0)
127
+ conn.execute(f"""INSERT INTO {functions} (fn_id, path, public, has_docstring, body_hash, docstring_hash,
128
+ signature_hash, doc_body_hash, doc_docstring_hash, doc_signature_hash, method_updated_at, status,
129
+ attempts) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)
130
+ ON CONFLICT(fn_id) DO UPDATE SET path=excluded.path, public=excluded.public,
131
+ has_docstring=excluded.has_docstring, body_hash=excluded.body_hash,
132
+ docstring_hash=excluded.docstring_hash, signature_hash=excluded.signature_hash,
133
+ doc_body_hash=excluded.doc_body_hash, doc_docstring_hash=excluded.doc_docstring_hash,
134
+ doc_signature_hash=excluded.doc_signature_hash, status=excluded.status, attempts=excluded.attempts,
135
+ method_updated_at=CASE WHEN {functions}.body_hash IS excluded.body_hash
136
+ THEN {functions}.method_updated_at ELSE excluded.method_updated_at END""",
137
+ (fn_id, code["path"], int(code["public"]), int(code["has_docstring"]), code["body"],
138
+ code["docstring"], code["signature"], page.get("body"), page.get("docstring"),
139
+ page.get("signature"), now if changed else None, status, attempts))
140
+ if status != "current":
141
+ pending.append(fn_id)
142
+ for fn_id in gone:
143
+ conn.execute(f"UPDATE {functions} SET status='orphan' WHERE fn_id=?", (fn_id,))
144
+ for path, state in file_states.items():
145
+ if state is None:
146
+ conn.execute(f"DELETE FROM {files} WHERE path=?", (path,))
147
+ else:
148
+ conn.execute(f"INSERT OR REPLACE INTO {files} (path, mtime_ns, size) VALUES (?,?,?)", (path, *state))
149
+ return pending
150
+
151
+
152
+ def claim(conn, root, limit, cost=lambda fn_id: 0, budget=None, now=None):
153
+ """Take the oldest pending functions (attempts below MAX_ATTEMPTS) for one run: in_progress with a lease and one more
154
+ attempt, at most `limit` of them and, when `budget` is given, while the summed `cost` (chain nodes) fits.
155
+
156
+ Returns: list of fn_id claimed.
157
+ """
158
+ functions = tables(conn, root)[0]
159
+ now = now or time.time()
160
+ marks = ",".join("?" * len(PENDING))
161
+ rows = conn.execute(f"SELECT fn_id FROM {functions} WHERE status IN ({marks}) AND attempts < ? "
162
+ "ORDER BY method_updated_at IS NULL, method_updated_at, fn_id", (*PENDING, MAX_ATTEMPTS)).fetchall()
163
+ chosen, spent = [], 0
164
+ for (fn_id,) in rows:
165
+ if len(chosen) == limit:
166
+ break
167
+ weight = cost(fn_id)
168
+ if budget is not None and chosen and spent + weight > budget:
169
+ break
170
+ chosen.append(fn_id)
171
+ spent += weight
172
+ with conn:
173
+ conn.executemany(f"UPDATE {functions} SET status='in_progress', attempts=attempts+1, lease_until=? WHERE fn_id=?",
174
+ [(now + LEASE_S, fn_id) for fn_id in chosen])
175
+ return chosen
176
+
177
+
178
+ def settle(conn, root, results, now=None):
179
+ """Close a run: each claimed function whose page now carries its code stamps becomes current with the stamps and the
180
+ date of the rewrite; the others go back to pending_code with the reason.
181
+
182
+ Parameters: results (dict fn_id -> (stamps dict or None, error str or None)).
183
+ """
184
+ functions = tables(conn, root)[0]
185
+ now = now or time.time()
186
+ with conn:
187
+ for fn_id, (stamps, error) in results.items():
188
+ if stamps:
189
+ conn.execute(f"UPDATE {functions} SET status='current', doc_signature_hash=?, doc_body_hash=?, "
190
+ "doc_docstring_hash=?, doc_updated_at=?, lease_until=NULL, last_error=NULL WHERE fn_id=?",
191
+ (stamps["signature"], stamps["body"], stamps["docstring"], now, fn_id))
192
+ else:
193
+ conn.execute(f"UPDATE {functions} SET status='pending_code', lease_until=NULL, last_error=? WHERE fn_id=?",
194
+ (error, fn_id))
195
+
196
+
197
+ def record_links(conn, root, links, now=None):
198
+ """Replace the project's feature links with `links` (dicts feature_id, fn_id, layer, hops, origin, score, status)."""
199
+ table = tables(conn, root)[2]
200
+ now = now or time.time()
201
+ with conn:
202
+ conn.execute(f"DELETE FROM {table}")
203
+ conn.executemany(f"INSERT INTO {table} (feature_id, fn_id, layer, hops, origin, score, status, computed_at) "
204
+ "VALUES (?,?,?,?,?,?,?,?)",
205
+ [(l["feature_id"], l["fn_id"], l["layer"], l["hops"], l["origin"], l["score"], l["status"], now)
206
+ for l in links])
207
+
208
+
209
+ def rows(conn, root, status=None):
210
+ """Registry rows of the project as dicts, optionally of one status."""
211
+ functions = tables(conn, root)[0]
212
+ cursor = conn.execute(f"SELECT * FROM {functions}" + (" WHERE status=?" if status else "") + " ORDER BY fn_id",
213
+ (status,) if status else ())
214
+ return _as_dicts(cursor)
215
+
216
+
217
+ def _as_dicts(cursor):
218
+ """The rows of an executed query as dicts keyed by column name."""
219
+ names = [c[0] for c in cursor.description]
220
+ return [dict(zip(names, row)) for row in cursor]
221
+
222
+
223
+ def candidates(root):
224
+ """Embedder candidate links (origin embedder, status candidate) of a project, read without creating anything: []
225
+ when the registry file does not exist or does not track the project."""
226
+ if not os.path.isfile(database_path()):
227
+ return []
228
+ conn = connect()
229
+ try:
230
+ if not conn.execute("SELECT 1 FROM projects WHERE root=?", (root,)).fetchone():
231
+ return []
232
+ return [row for row in links(conn, root) if row["origin"] == "embedder" and row["status"] == "candidate"]
233
+ finally:
234
+ conn.close()
235
+
236
+
237
+ def links(conn, root):
238
+ """Feature links of the project as dicts."""
239
+ table = tables(conn, root)[2]
240
+ cursor = conn.execute(f"SELECT * FROM {table} ORDER BY feature_id, fn_id")
241
+ return _as_dicts(cursor)