@allansantos-dev/smart-tool 0.0.0-stage → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/LICENSE +202 -0
- package/NOTICE +32 -0
- package/README.md +116 -2
- package/atomic_io.py +94 -0
- package/autostart.py +199 -0
- package/bin/smart-tool.mjs +86 -0
- package/browser_service_client.py +232 -0
- package/capabilities.py +63 -0
- package/client_hooks.py +192 -0
- package/code_graph.py +403 -0
- package/code_graph_java.mjs +49 -0
- package/code_graph_js.cjs +135 -0
- package/config.py +241 -0
- package/daemon_launcher.py +165 -0
- package/directory_watch.py +131 -0
- package/document_text.py +69 -0
- package/duplicates.py +313 -0
- package/embedding_cache.py +99 -0
- package/endpoint_sync.py +179 -0
- package/folder_picker.py +73 -0
- package/freshness.py +278 -0
- package/gateway.py +220 -0
- package/hook_decision.py +96 -0
- package/hooks/pretool_router.py +45 -0
- package/index_inventory.py +75 -0
- package/index_profile.py +167 -0
- package/index_scope.py +458 -0
- package/index_views.py +121 -0
- package/indexer.py +878 -0
- package/install.py +353 -0
- package/install_runtime.py +283 -0
- package/integration_metrics.py +82 -0
- package/local_embedder.py +172 -0
- package/model_adapters/__init__.py +0 -0
- package/model_adapters/openai_compatible.py +57 -0
- package/model_catalog.py +74 -0
- package/model_client.py +38 -0
- package/model_defaults.py +70 -0
- package/package.json +33 -4
- package/paths.py +6 -0
- package/project_identity.py +42 -0
- package/project_insights_ui.py +38 -0
- package/project_maps_ui.py +96 -0
- package/project_monitor.py +204 -0
- package/project_scheduler.py +61 -0
- package/project_storage.py +165 -0
- package/project_store.py +220 -0
- package/projects_ui.py +146 -0
- package/requirements.txt +8 -0
- package/research_cache.py +171 -0
- package/research_metrics.py +110 -0
- package/router.py +582 -0
- package/secret_store.py +121 -0
- package/setup_ui.py +986 -0
- package/site_memory.py +53 -0
- package/smart_tool_daemon.py +3604 -0
- package/tray.py +245 -0
- package/update_check.py +22 -0
- package/usage_meter.py +124 -0
- package/version.py +18 -0
- package/view_compare.py +68 -0
- package/web_adapters/browser/browser_service.py +302 -0
- package/web_adapters/browser/crawl4ai_search.py +236 -0
- package/web_adapters/browser/fetch_camoufox.py +28 -0
- package/web_adapters/browser/requirements.txt +2 -0
- package/web_adapters/node/adapter-common.mjs +40 -0
- package/web_adapters/node/ddg-search.mjs +89 -0
- package/web_adapters/node/open-websearch-adapter.mjs +77 -0
- package/web_adapters/node/package-lock.json +2720 -0
- package/web_adapters/node/package.json +12 -0
- package/web_adapters/node/wikipedia-search.mjs +44 -0
- package/web_document_graph.py +71 -0
- package/web_fetch.py +242 -0
- package/web_search_adapters.py +193 -0
- package/web_search_ddgs.py +42 -0
- package/web_search_external.py +288 -0
- package/web_search_health.py +88 -0
package/install.py
ADDED
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
import argparse
|
|
3
|
+
import filecmp
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
import time
|
|
11
|
+
import urllib.request
|
|
12
|
+
import venv
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import paths
|
|
16
|
+
|
|
17
|
+
SOURCE = Path(__file__).resolve().parent
|
|
18
|
+
DEFAULT_TARGET = Path(os.environ.get("LOCALAPPDATA", str(Path.home() / "AppData" / "Local"))) / "Programs" / "SmartTool"
|
|
19
|
+
TASK_NAME = "SmartTool-Tray"
|
|
20
|
+
ENTRY_SCRIPTS = ("tray.py", "smart_tool_daemon.py", "pretool_router.py")
|
|
21
|
+
STARTUP_SCRIPT = Path(os.environ.get("APPDATA", str(Path.home() / "AppData" / "Roaming"))) / "Microsoft" / "Windows" / \
|
|
22
|
+
"Start Menu" / "Programs" / "Startup" / "SmartTool-Tray.vbs"
|
|
23
|
+
MANIFEST = ".install-manifest.json"
|
|
24
|
+
REGISTRY = Path(paths.DATA_DIR) / "daemon.json"
|
|
25
|
+
BLOCKED = {"__pycache__", ".token-guard", ".git", ".pytest_cache", ".data", ".artifacts", ".claude", ".vscode",
|
|
26
|
+
".impeccable", "tests", "sessions", "results", "dist", "node_modules", ".node-runtime", ".venv",
|
|
27
|
+
".scratch", ".memory", "docs", "reports", "site", "design", "release", "npm", ".github", "AGENTS.md", "CLAUDE.md", "DESIGN.md", "PRODUCT.md", "install.py", ".gitignore", ".gitattributes", MANIFEST}
|
|
28
|
+
PACKAGE_ONLY = {"bin", "package.json", ".npmignore"}
|
|
29
|
+
REQUIRED_TOOLS = {"smart_search", "smart_search_result", "web_search", "project_manage"}
|
|
30
|
+
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
31
|
+
def ships(rel):
|
|
32
|
+
parts = Path(rel).parts
|
|
33
|
+
*folders, name = parts
|
|
34
|
+
return (parts[0] not in PACKAGE_ONLY
|
|
35
|
+
and not any(part in BLOCKED or part.startswith("_cdp_") for part in folders)
|
|
36
|
+
and name not in BLOCKED and not name.endswith((".log", ".pyc", ".png")))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def source_files():
|
|
40
|
+
files = []
|
|
41
|
+
for current, dirs, names in os.walk(SOURCE):
|
|
42
|
+
dirs[:] = sorted(d for d in dirs if d not in BLOCKED and not d.startswith("_cdp_"))
|
|
43
|
+
files += [rel for rel in ((Path(current) / name).relative_to(SOURCE).as_posix() for name in sorted(names))
|
|
44
|
+
if ships(rel)]
|
|
45
|
+
return files
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def python_paths(target):
|
|
49
|
+
scripts = target / ".venv" / "Scripts"
|
|
50
|
+
return scripts / "python.exe", scripts / "pythonw.exe"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def powershell(script, env=None, timeout=30):
|
|
54
|
+
return subprocess.run(["powershell.exe", "-NoProfile", "-Command", script], env={**os.environ, **(env or {})},
|
|
55
|
+
capture_output=True, text=True, timeout=timeout)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _processes():
|
|
59
|
+
"""(pid, command line) of every python/pythonw process."""
|
|
60
|
+
listed = powershell("Get-CimInstance Win32_Process -Filter \"Name = 'pythonw.exe' OR Name = 'python.exe'\" | "
|
|
61
|
+
"ForEach-Object { \"$($_.ProcessId)`t$($_.CommandLine)\" }").stdout
|
|
62
|
+
rows = (row.partition("\t") for row in listed.splitlines())
|
|
63
|
+
return [(int(pid), line) for pid, _tab, line in rows if pid.strip().isdigit()]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _script_folder(script):
|
|
67
|
+
path = Path(script)
|
|
68
|
+
return path.parent.parent if path.name == "pretool_router.py" else path.parent
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _same_folder(path, folder):
|
|
72
|
+
"""Compares through resolve(), which turns 8.3 short and mixed spellings into the long name."""
|
|
73
|
+
try:
|
|
74
|
+
return path.resolve() == folder.resolve()
|
|
75
|
+
except OSError:
|
|
76
|
+
return False
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def stop_processes(target):
|
|
80
|
+
def running():
|
|
81
|
+
return [pid for pid, line in _processes() for script in _script_paths(line)
|
|
82
|
+
if Path(script).name != "pretool_router.py" and _same_folder(_script_folder(script), target)]
|
|
83
|
+
|
|
84
|
+
for pid in running():
|
|
85
|
+
powershell(f"Stop-Process -Id {pid} -Force -ErrorAction SilentlyContinue")
|
|
86
|
+
deadline = time.monotonic() + 10
|
|
87
|
+
while True:
|
|
88
|
+
left = running()
|
|
89
|
+
if not left:
|
|
90
|
+
return
|
|
91
|
+
if time.monotonic() > deadline:
|
|
92
|
+
raise RuntimeError(f"Could not close the running Smart Tool ({len(left)} process(es) still active).")
|
|
93
|
+
time.sleep(0.5)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def wait_port_free():
|
|
97
|
+
deadline = time.monotonic() + 15
|
|
98
|
+
while time.monotonic() < deadline and _daemon_up():
|
|
99
|
+
time.sleep(0.5)
|
|
100
|
+
if _daemon_up():
|
|
101
|
+
raise RuntimeError("Another Smart Tool daemon is still active on the registered port; close it before updating.")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _daemon_up():
|
|
105
|
+
try:
|
|
106
|
+
url = json.loads(REGISTRY.read_text(encoding="utf-8"))["url"]
|
|
107
|
+
with _OPENER.open(url + "/health", timeout=1) as response:
|
|
108
|
+
return response.status == 200
|
|
109
|
+
except (OSError, ValueError, KeyError):
|
|
110
|
+
return False
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def hook_files():
|
|
114
|
+
claude = Path(os.environ.get("CLAUDE_CONFIG_DIR") or Path.home() / ".claude")
|
|
115
|
+
return claude / "settings.json", Path.home() / ".codex" / "hooks.json"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _script_paths(text):
|
|
119
|
+
"""Absolute paths ending in an entry script, as written in text, tried from every drive letter (paths may hold
|
|
120
|
+
spaces)."""
|
|
121
|
+
script = re.compile(r'[^"\r\n]*?(?:' + "|".join(map(re.escape, ENTRY_SCRIPTS)) + ")")
|
|
122
|
+
for drive in re.finditer(r"[A-Za-z]:[\\/]", text):
|
|
123
|
+
found = script.match(text, drive.start())
|
|
124
|
+
if found:
|
|
125
|
+
yield found.group(0)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _install_folder(script):
|
|
129
|
+
folder = _script_folder(script)
|
|
130
|
+
try:
|
|
131
|
+
return folder.resolve() if (folder / MANIFEST).is_file() and (folder / "smart_tool_daemon.py").is_file() else None
|
|
132
|
+
except OSError:
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _task_definition():
|
|
137
|
+
result = subprocess.run(["schtasks", "/Query", "/TN", TASK_NAME, "/XML"], capture_output=True, text=True,
|
|
138
|
+
timeout=20)
|
|
139
|
+
return result.stdout if result.returncode == 0 else ""
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def other_installs(target):
|
|
143
|
+
"""Installer-made Smart Tool folders other than target, found through what still runs or points at them: tray and
|
|
144
|
+
daemon processes, the logon task or Startup script, and agent hook commands."""
|
|
145
|
+
texts = [line for _pid, line in _processes()] + [_task_definition()]
|
|
146
|
+
texts += [path.read_text(encoding="utf-8", errors="replace") for path in (STARTUP_SCRIPT, *hook_files())
|
|
147
|
+
if path.is_file()]
|
|
148
|
+
folders = {_install_folder(script) for text in texts for script in _script_paths(text)}
|
|
149
|
+
return sorted(folder for folder in folders
|
|
150
|
+
if folder and not _same_folder(folder, target) and not _same_folder(folder, SOURCE))
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _repointed(text, folder, target):
|
|
154
|
+
"""text with each spelling of folder found in front of an entry script (long, 8.3 or mixed) replaced by target."""
|
|
155
|
+
for script in set(_script_paths(text)):
|
|
156
|
+
cut = len(Path(script).name) + 1 + (len("hooks") + 1 if Path(script).name == "pretool_router.py" else 0)
|
|
157
|
+
spelled = script[:-cut]
|
|
158
|
+
if _same_folder(Path(spelled), folder):
|
|
159
|
+
text = text.replace(spelled, target.as_posix() if "/" in spelled else str(target))
|
|
160
|
+
return text
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def remove_other_installs(target):
|
|
164
|
+
"""An older installation in another folder keeps the port and the hooks: stop it, point the hooks at target and
|
|
165
|
+
delete its program folder. Its state folders are left alone."""
|
|
166
|
+
removed = []
|
|
167
|
+
stamp = time.strftime("%Y%m%d-%H%M%S")
|
|
168
|
+
for folder in other_installs(target):
|
|
169
|
+
print(f"Older Smart Tool installation found at {folder}; removing it.")
|
|
170
|
+
stop_processes(folder)
|
|
171
|
+
for path in hook_files():
|
|
172
|
+
original = path.read_text(encoding="utf-8") if path.is_file() else ""
|
|
173
|
+
text = _repointed(original, folder, target)
|
|
174
|
+
if text != original:
|
|
175
|
+
shutil.copy2(path, path.with_name(f"{path.name}.smart-tool-backup-{stamp}"))
|
|
176
|
+
path.write_text(text, encoding="utf-8")
|
|
177
|
+
print(f"Hooks in {path} now point to {target}.")
|
|
178
|
+
startup = STARTUP_SCRIPT.read_text(encoding="utf-8", errors="replace") if STARTUP_SCRIPT.is_file() else ""
|
|
179
|
+
if any(_same_folder(_script_folder(script), folder) for script in _script_paths(startup)):
|
|
180
|
+
STARTUP_SCRIPT.unlink()
|
|
181
|
+
shutil.rmtree(folder)
|
|
182
|
+
removed.append(folder)
|
|
183
|
+
return removed
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def copy_files(target, files, backup, changed, removed):
|
|
187
|
+
previous = set(json.loads((target / MANIFEST).read_text(encoding="utf-8"))["files"]) if (target / MANIFEST).is_file() else set()
|
|
188
|
+
if (target / MANIFEST).is_file():
|
|
189
|
+
shutil.copy2(target / MANIFEST, backup / MANIFEST)
|
|
190
|
+
for rel in files:
|
|
191
|
+
source, dest = SOURCE / rel, target / rel
|
|
192
|
+
if dest.is_file() and filecmp.cmp(source, dest, shallow=False):
|
|
193
|
+
continue
|
|
194
|
+
if dest.is_file():
|
|
195
|
+
(backup / rel).parent.mkdir(parents=True, exist_ok=True)
|
|
196
|
+
shutil.copy2(dest, backup / rel)
|
|
197
|
+
changed.append(rel)
|
|
198
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
199
|
+
shutil.copy2(source, dest)
|
|
200
|
+
for rel in sorted(previous - set(files)):
|
|
201
|
+
if (target / rel).is_file():
|
|
202
|
+
(backup / rel).parent.mkdir(parents=True, exist_ok=True)
|
|
203
|
+
shutil.move(target / rel, backup / rel)
|
|
204
|
+
removed.append(rel)
|
|
205
|
+
folder = (target / rel).parent
|
|
206
|
+
while folder != target and not any(folder.iterdir()):
|
|
207
|
+
folder.rmdir()
|
|
208
|
+
folder = folder.parent
|
|
209
|
+
(target / MANIFEST).write_text(json.dumps({"installed_at": time.time(), "source": str(SOURCE), "files": files},
|
|
210
|
+
indent=1), encoding="utf-8")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def restore(target, backup, changed, removed):
|
|
214
|
+
for rel in changed:
|
|
215
|
+
if (backup / rel).is_file():
|
|
216
|
+
shutil.copy2(backup / rel, target / rel)
|
|
217
|
+
else:
|
|
218
|
+
(target / rel).unlink(missing_ok=True)
|
|
219
|
+
for rel in removed:
|
|
220
|
+
if (backup / rel).is_file():
|
|
221
|
+
(target / rel).parent.mkdir(parents=True, exist_ok=True)
|
|
222
|
+
shutil.move(backup / rel, target / rel)
|
|
223
|
+
if (backup / MANIFEST).is_file():
|
|
224
|
+
shutil.copy2(backup / MANIFEST, target / MANIFEST)
|
|
225
|
+
else:
|
|
226
|
+
(target / MANIFEST).unlink(missing_ok=True)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def freeze(python):
|
|
230
|
+
if not python.exists():
|
|
231
|
+
return None
|
|
232
|
+
return subprocess.run([str(python), "-m", "pip", "freeze", "--disable-pip-version-check"],
|
|
233
|
+
check=True, capture_output=True, text=True).stdout
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def restore_packages(python, frozen, backup):
|
|
237
|
+
if frozen is None:
|
|
238
|
+
return
|
|
239
|
+
(backup / "requirements.frozen.txt").write_text(frozen, encoding="utf-8")
|
|
240
|
+
subprocess.run([str(python), "-m", "pip", "install", "--disable-pip-version-check", "-q", "-r",
|
|
241
|
+
str(backup / "requirements.frozen.txt")], check=True)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def install_dependencies(target, python):
|
|
245
|
+
if not python.exists():
|
|
246
|
+
venv.create(python.parent.parent, with_pip=True)
|
|
247
|
+
subprocess.run([str(python), "-m", "pip", "install", "--disable-pip-version-check", "-q", "-r",
|
|
248
|
+
str(target / "requirements.txt")], check=True)
|
|
249
|
+
subprocess.run([str(python), str(target / "install_runtime.py")], check=True)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def start(target, python, pythonw):
|
|
253
|
+
subprocess.run([str(python), "-c", "import autostart, json; print(json.dumps(autostart.install()))"],
|
|
254
|
+
cwd=target, check=True, capture_output=True, text=True, timeout=60)
|
|
255
|
+
subprocess.Popen([str(pythonw if pythonw.exists() else python), str(target / "tray.py")], cwd=target,
|
|
256
|
+
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
257
|
+
creationflags=getattr(subprocess, "DETACHED_PROCESS", 0) | getattr(subprocess, "CREATE_NO_WINDOW", 0))
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def verify(target, python):
|
|
261
|
+
script = ("import json,urllib.request,daemon_launcher; url=daemon_launcher.ensure_daemon_running(wait_seconds=30); "
|
|
262
|
+
"body=json.dumps({'jsonrpc':'2.0','id':1,'method':'tools/list','params':{}}).encode(); "
|
|
263
|
+
"req=urllib.request.Request(url+'/mcp',data=body,method='POST',headers={'Content-Type':'application/json'}); "
|
|
264
|
+
"opener=urllib.request.build_opener(urllib.request.ProxyHandler({})); "
|
|
265
|
+
"print(json.dumps(sorted(r['name'] for r in json.load(opener.open(req,timeout=10))['result']['tools'])))")
|
|
266
|
+
result = subprocess.run([str(python), "-c", script], cwd=target, capture_output=True, text=True, timeout=60)
|
|
267
|
+
if result.returncode:
|
|
268
|
+
raise RuntimeError("Daemon did not respond: " + ((result.stderr or result.stdout).strip().splitlines() or ["no output"])[-1])
|
|
269
|
+
missing = REQUIRED_TOOLS - set(json.loads(result.stdout.strip().splitlines()[-1]))
|
|
270
|
+
if missing:
|
|
271
|
+
raise RuntimeError("Missing MCP tools: " + ", ".join(sorted(missing)))
|
|
272
|
+
pid = json.loads(REGISTRY.read_text(encoding="utf-8"))["pid"]
|
|
273
|
+
command = powershell(f"(Get-CimInstance Win32_Process -Filter 'ProcessId = {int(pid)}').CommandLine").stdout
|
|
274
|
+
if str(target / "smart_tool_daemon.py").lower() not in command.lower():
|
|
275
|
+
raise RuntimeError(f"The active daemon (pid {pid}) does not belong to this installation: {command.strip()[:200]}")
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
MIN_PYTHON = (3, 11)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def main():
|
|
282
|
+
if sys.version_info < MIN_PYTHON:
|
|
283
|
+
raise SystemExit(f"Smart Tool needs Python {'.'.join(map(str, MIN_PYTHON))} or newer; this is "
|
|
284
|
+
f"{sys.version.split()[0]} ({sys.executable}).")
|
|
285
|
+
parser = argparse.ArgumentParser(description="Install or update Smart Tool over the existing installation.")
|
|
286
|
+
parser.add_argument("--target", type=Path, default=DEFAULT_TARGET)
|
|
287
|
+
parser.add_argument("--skip-deps", action="store_true", help="do not run pip or install_runtime.py")
|
|
288
|
+
parser.add_argument("--no-start", action="store_true", help="do not restart or check the daemon")
|
|
289
|
+
parser.add_argument("--skip-tests", action="store_true",
|
|
290
|
+
help="install without running tests/run.py (npm package, whose tests ran in CI; emergency only otherwise)")
|
|
291
|
+
args = parser.parse_args()
|
|
292
|
+
target = args.target.resolve()
|
|
293
|
+
if target == SOURCE:
|
|
294
|
+
raise SystemExit("The target cannot be the source folder itself.")
|
|
295
|
+
if args.skip_tests:
|
|
296
|
+
print("Tests skipped by --skip-tests.")
|
|
297
|
+
elif not (SOURCE / "tests" / "run.py").is_file():
|
|
298
|
+
raise SystemExit(f"No test suite in {SOURCE}; a packaged install runs with --skip-tests (npx @allansantos-dev/smart-tool install).")
|
|
299
|
+
else:
|
|
300
|
+
interpreter = python_paths(target)[0] if python_paths(target)[0].is_file() else Path(sys.executable)
|
|
301
|
+
tests = subprocess.run([str(interpreter), str(SOURCE / "tests" / "run.py")], capture_output=True, text=True,
|
|
302
|
+
encoding="utf-8", errors="replace")
|
|
303
|
+
print(tests.stdout.rstrip())
|
|
304
|
+
if tests.stderr.strip():
|
|
305
|
+
print(tests.stderr.rstrip()[-3000:], file=sys.stderr)
|
|
306
|
+
if tests.returncode:
|
|
307
|
+
raise SystemExit("Tests failed; installation aborted without touching the installed version.")
|
|
308
|
+
python, pythonw = python_paths(target)
|
|
309
|
+
files = source_files()
|
|
310
|
+
existed = (target / "smart_tool_daemon.py").is_file()
|
|
311
|
+
print(("Updating" if existed else "Installing") + f" Smart Tool at {target} ({len(files)} files from {SOURCE})")
|
|
312
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
313
|
+
remove_other_installs(target)
|
|
314
|
+
stop_processes(target)
|
|
315
|
+
if not args.no_start:
|
|
316
|
+
wait_port_free()
|
|
317
|
+
backup = target.parent / f".smart-tool-backup-{time.strftime('%Y%m%d-%H%M%S')}"
|
|
318
|
+
backup.mkdir()
|
|
319
|
+
changed, removed, frozen, deps_touched = [], [], None, False
|
|
320
|
+
try:
|
|
321
|
+
copy_files(target, files, backup, changed, removed)
|
|
322
|
+
print(f"Files changed: {len(changed)}; removed as obsolete: {len(removed)}")
|
|
323
|
+
if not args.skip_deps:
|
|
324
|
+
frozen = freeze(python)
|
|
325
|
+
deps_touched = True
|
|
326
|
+
install_dependencies(target, python)
|
|
327
|
+
if not args.no_start:
|
|
328
|
+
start(target, python, pythonw)
|
|
329
|
+
verify(target, python)
|
|
330
|
+
except BaseException as exc:
|
|
331
|
+
print(f"Failed: {exc}. Restoring the previous version.", file=sys.stderr)
|
|
332
|
+
problems = []
|
|
333
|
+
steps = [lambda: stop_processes(target), lambda: restore(target, backup, changed, removed)]
|
|
334
|
+
if deps_touched:
|
|
335
|
+
steps.append(lambda: restore_packages(python, frozen, backup))
|
|
336
|
+
if existed and not args.no_start:
|
|
337
|
+
steps.append(lambda: start(target, python, pythonw))
|
|
338
|
+
for step in steps:
|
|
339
|
+
try:
|
|
340
|
+
step()
|
|
341
|
+
except Exception as error:
|
|
342
|
+
problems.append(str(error))
|
|
343
|
+
if problems:
|
|
344
|
+
print(f"Incomplete rollback; backup kept at {backup}: " + " | ".join(problems), file=sys.stderr)
|
|
345
|
+
raise SystemExit(1)
|
|
346
|
+
shutil.rmtree(backup, ignore_errors=True)
|
|
347
|
+
raise SystemExit(1)
|
|
348
|
+
shutil.rmtree(backup)
|
|
349
|
+
print("Smart Tool ready." if not args.no_start else "Files installed; daemon not restarted (--no-start).")
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
if __name__ == "__main__":
|
|
353
|
+
main()
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Instala as dependências opcionais de busca dentro da pasta da Smart Tool.
|
|
3
|
+
|
|
4
|
+
Uso: python install_runtime.py [--check]
|
|
5
|
+
Não altera ~/.claude nem instala pacotes npm globalmente.
|
|
6
|
+
"""
|
|
7
|
+
import argparse
|
|
8
|
+
import collections
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import shutil
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
import tempfile
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
import urllib.request
|
|
20
|
+
import venv
|
|
21
|
+
import zipfile
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
ROOT = Path(__file__).resolve().parent
|
|
26
|
+
NODE_DIR = ROOT / "web_adapters" / "node"
|
|
27
|
+
BROWSER_DIR = ROOT / "web_adapters" / "browser"
|
|
28
|
+
BROWSER_PYTHON = BROWSER_DIR / ".venv" / "Scripts" / "python.exe"
|
|
29
|
+
OPEN_WEBSEARCH_CLI = NODE_DIR / "node_modules" / "open-websearch" / "build" / "index.js"
|
|
30
|
+
TYPESCRIPT_PARSER = NODE_DIR / "node_modules" / "typescript" / "lib" / "typescript.js"
|
|
31
|
+
JAVA_PARSER = NODE_DIR / 'node_modules' / 'java-parser' / 'src' / 'index.js'
|
|
32
|
+
ANGULAR_PARSER = NODE_DIR / 'node_modules' / '@angular' / 'compiler' / 'fesm2022' / 'compiler.mjs'
|
|
33
|
+
NODE_RUNTIME = NODE_DIR / ".node-runtime"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _node_supported(node):
|
|
37
|
+
"""Aceita apenas versoes suportadas pelo @angular/compiler 20.3.0."""
|
|
38
|
+
try:
|
|
39
|
+
result = subprocess.run([str(node), "--version"], capture_output=True, text=True,
|
|
40
|
+
timeout=10, check=False)
|
|
41
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
42
|
+
return False
|
|
43
|
+
match = re.fullmatch(r"v(\d+)\.(\d+)\.\d+", result.stdout.strip())
|
|
44
|
+
if result.returncode or not match:
|
|
45
|
+
return False
|
|
46
|
+
major, minor = map(int, match.groups())
|
|
47
|
+
return (major == 20 and minor >= 19) or (major == 22 and minor >= 12) or major >= 24
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _node_commands():
|
|
51
|
+
local_node, local_npm = NODE_RUNTIME / "node.exe", NODE_RUNTIME / "npm.cmd"
|
|
52
|
+
if NODE_RUNTIME.exists():
|
|
53
|
+
if local_node.is_file() and local_npm.is_file() and _node_supported(local_node):
|
|
54
|
+
return str(local_node), str(local_npm)
|
|
55
|
+
return None, None
|
|
56
|
+
node = shutil.which("node")
|
|
57
|
+
npm = shutil.which("npm.cmd") or shutil.which("npm")
|
|
58
|
+
return (node, npm) if node and npm and _node_supported(node) else (None, None)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _install_node():
|
|
62
|
+
"""Baixa o Node LTS oficial e confere SHA-256 antes de extrair."""
|
|
63
|
+
print("Downloading Node LTS for the Smart Tool web runtime...", flush=True)
|
|
64
|
+
with urllib.request.urlopen("https://nodejs.org/dist/index.json", timeout=30) as response:
|
|
65
|
+
releases = json.load(response)
|
|
66
|
+
release = next((row for row in releases if row.get("lts") and
|
|
67
|
+
"win-x64-zip" in row.get("files", [])), None)
|
|
68
|
+
if not release:
|
|
69
|
+
raise RuntimeError("Node LTS for Windows x64 not found in the official index.")
|
|
70
|
+
version = release["version"]
|
|
71
|
+
archive = f"node-{version}-win-x64.zip"
|
|
72
|
+
base = f"https://nodejs.org/dist/{version}/"
|
|
73
|
+
with urllib.request.urlopen(base + "SHASUMS256.txt", timeout=30) as response:
|
|
74
|
+
checksums = response.read().decode("ascii")
|
|
75
|
+
expected = next((line.split()[0] for line in checksums.splitlines()
|
|
76
|
+
if line.split()[-1:] == [archive]), None)
|
|
77
|
+
if not expected:
|
|
78
|
+
raise RuntimeError("Official Node checksum not found.")
|
|
79
|
+
with tempfile.TemporaryDirectory(prefix="node-setup-", dir=NODE_DIR) as temp:
|
|
80
|
+
temp_dir = Path(temp)
|
|
81
|
+
zip_path = temp_dir / archive
|
|
82
|
+
digest = hashlib.sha256()
|
|
83
|
+
with urllib.request.urlopen(base + archive, timeout=60) as response, zip_path.open("wb") as out:
|
|
84
|
+
while block := response.read(1024 * 1024):
|
|
85
|
+
digest.update(block)
|
|
86
|
+
out.write(block)
|
|
87
|
+
if digest.hexdigest().lower() != expected.lower():
|
|
88
|
+
raise RuntimeError("Node SHA-256 checksum does not match the official index.")
|
|
89
|
+
unpacked = temp_dir / "runtime"
|
|
90
|
+
unpacked.mkdir()
|
|
91
|
+
with zipfile.ZipFile(zip_path) as zf:
|
|
92
|
+
for member in zf.infolist():
|
|
93
|
+
parts = Path(member.filename.replace("\\", "/")).parts
|
|
94
|
+
if parts == (f"node-{version}-win-x64",) and member.is_dir():
|
|
95
|
+
continue
|
|
96
|
+
if len(parts) < 2 or parts[0] != f"node-{version}-win-x64" or ".." in parts:
|
|
97
|
+
raise RuntimeError("Node archive contains an unexpected path.")
|
|
98
|
+
destination = unpacked.joinpath(*parts[1:])
|
|
99
|
+
if member.is_dir():
|
|
100
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
101
|
+
else:
|
|
102
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
103
|
+
with zf.open(member) as source, destination.open("wb") as target:
|
|
104
|
+
shutil.copyfileobj(source, target)
|
|
105
|
+
if not (unpacked / "node.exe").is_file() or not (unpacked / "npm.cmd").is_file():
|
|
106
|
+
raise RuntimeError("Node package does not contain the expected executables.")
|
|
107
|
+
previous = temp_dir / "previous-node-runtime"
|
|
108
|
+
if NODE_RUNTIME.exists():
|
|
109
|
+
os.replace(NODE_RUNTIME, previous)
|
|
110
|
+
try:
|
|
111
|
+
os.replace(unpacked, NODE_RUNTIME)
|
|
112
|
+
except OSError:
|
|
113
|
+
if previous.exists() and not NODE_RUNTIME.exists():
|
|
114
|
+
os.replace(previous, NODE_RUNTIME)
|
|
115
|
+
raise
|
|
116
|
+
print(f"Node {version} installed inside Smart Tool.")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _run(args, *, cwd=None, label="Web dependency", idle_s=600, stream=False, attempts=1):
|
|
120
|
+
"""Runs a step with UTF-8 output and closed stdin (a prompt fails at once instead of hanging); it is stopped only
|
|
121
|
+
after idle_s without any output, and stream shows the output live."""
|
|
122
|
+
env = os.environ.copy()
|
|
123
|
+
env.update(PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
|
|
124
|
+
if NODE_RUNTIME.is_dir():
|
|
125
|
+
env["PATH"] = str(NODE_RUNTIME) + os.pathsep + env.get("PATH", "")
|
|
126
|
+
flags = (subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW) if os.name == "nt" else 0
|
|
127
|
+
for attempt in range(1, attempts + 1):
|
|
128
|
+
print(f"{label}: starting{f' (attempt {attempt} of {attempts})' if attempt > 1 else ''}...", flush=True)
|
|
129
|
+
try:
|
|
130
|
+
_run_once(args, cwd, env, flags, label, idle_s, stream)
|
|
131
|
+
print(f"{label}: done.", flush=True)
|
|
132
|
+
return
|
|
133
|
+
except RuntimeError as exc:
|
|
134
|
+
if attempt == attempts:
|
|
135
|
+
raise
|
|
136
|
+
print(f"{label}: {exc}; trying again.", flush=True)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _run_once(args, cwd, env, flags, label, idle_s, stream):
|
|
140
|
+
with subprocess.Popen(args, cwd=cwd, env=env, stdin=subprocess.DEVNULL,
|
|
141
|
+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8",
|
|
142
|
+
errors="replace", creationflags=flags) as process:
|
|
143
|
+
tail = collections.deque(maxlen=40)
|
|
144
|
+
last_output = [time.monotonic()]
|
|
145
|
+
shown = {"stage": None, "tens": -1}
|
|
146
|
+
|
|
147
|
+
def pump():
|
|
148
|
+
for line in process.stdout:
|
|
149
|
+
line = line.rstrip()
|
|
150
|
+
if not line:
|
|
151
|
+
continue
|
|
152
|
+
tail.append(line)
|
|
153
|
+
last_output[0] = time.monotonic()
|
|
154
|
+
percent = re.match(r"(.*?)(\d{1,3})%$", line)
|
|
155
|
+
if percent:
|
|
156
|
+
stage, tens = percent.group(1), int(percent.group(2)) // 10
|
|
157
|
+
if stage == shown["stage"] and tens <= shown["tens"]:
|
|
158
|
+
continue
|
|
159
|
+
shown.update(stage=stage, tens=tens)
|
|
160
|
+
if stream:
|
|
161
|
+
print(f"{label}: {line}", flush=True)
|
|
162
|
+
|
|
163
|
+
reader = threading.Thread(target=pump, daemon=True)
|
|
164
|
+
reader.start()
|
|
165
|
+
started = noted = time.monotonic()
|
|
166
|
+
while process.poll() is None:
|
|
167
|
+
now = time.monotonic()
|
|
168
|
+
if now - last_output[0] > idle_s:
|
|
169
|
+
if os.name == "nt":
|
|
170
|
+
subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"],
|
|
171
|
+
capture_output=True, creationflags=subprocess.CREATE_NO_WINDOW)
|
|
172
|
+
else:
|
|
173
|
+
process.kill()
|
|
174
|
+
process.wait(timeout=10)
|
|
175
|
+
raise RuntimeError(f"{label} stopped: no output for {idle_s}s; last output: {tail[-1] if tail else 'none'}")
|
|
176
|
+
if not stream and now - noted >= 30:
|
|
177
|
+
noted = now
|
|
178
|
+
print(f"{label}: waiting for {int(now - started)}s...", flush=True)
|
|
179
|
+
time.sleep(0.5)
|
|
180
|
+
reader.join(timeout=10)
|
|
181
|
+
if process.returncode:
|
|
182
|
+
raise RuntimeError(f"{' '.join(map(str, args[:3]))} failed (exit {process.returncode}): "
|
|
183
|
+
+ "\n".join(tail)[-1500:])
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _check():
|
|
187
|
+
node, _npm = _node_commands()
|
|
188
|
+
status = {
|
|
189
|
+
"node": bool(node),
|
|
190
|
+
"open_websearch": bool(node and OPEN_WEBSEARCH_CLI.is_file()),
|
|
191
|
+
"typescript_parser": bool(node and TYPESCRIPT_PARSER.is_file()),
|
|
192
|
+
"java_parser":bool(node and JAVA_PARSER.is_file()),
|
|
193
|
+
"angular_parser":bool(node and ANGULAR_PARSER.is_file()),
|
|
194
|
+
"browser_python": BROWSER_PYTHON.is_file(),
|
|
195
|
+
"browser_packages": False,
|
|
196
|
+
}
|
|
197
|
+
if status["browser_python"]:
|
|
198
|
+
try:
|
|
199
|
+
result = subprocess.run(
|
|
200
|
+
[str(BROWSER_PYTHON), "-c", "import camoufox, crawl4ai, playwright"],
|
|
201
|
+
capture_output=True, check=False, timeout=30,
|
|
202
|
+
)
|
|
203
|
+
status["browser_packages"] = result.returncode == 0
|
|
204
|
+
except subprocess.TimeoutExpired:
|
|
205
|
+
status["browser_packages"] = False
|
|
206
|
+
return status
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _link_existing_browser(existing, link):
|
|
210
|
+
try:
|
|
211
|
+
os.symlink(existing, link, target_is_directory=True)
|
|
212
|
+
except OSError:
|
|
213
|
+
# Junction não exige Developer Mode nem privilégio de symlink no Windows.
|
|
214
|
+
env = os.environ.copy()
|
|
215
|
+
env["SMART_TOOL_BROWSER_LINK_PATH"] = str(link)
|
|
216
|
+
env["SMART_TOOL_BROWSER_LINK_TARGET"] = str(existing)
|
|
217
|
+
result = subprocess.run(
|
|
218
|
+
["powershell.exe", "-NoProfile", "-NonInteractive", "-Command",
|
|
219
|
+
"New-Item -ItemType Junction -Path $env:SMART_TOOL_BROWSER_LINK_PATH "
|
|
220
|
+
"-Target $env:SMART_TOOL_BROWSER_LINK_TARGET | Out-Null"],
|
|
221
|
+
env=env, capture_output=True, text=True, encoding="utf-8", errors="replace",
|
|
222
|
+
)
|
|
223
|
+
if result.returncode or not link.is_dir():
|
|
224
|
+
raise RuntimeError(f"Could not link the existing runtime: {result.stderr.strip()[-600:]}")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def install(reuse_browser_runtime=None):
|
|
228
|
+
if os.name != "nt":
|
|
229
|
+
raise RuntimeError("This browser installer is built for Windows.")
|
|
230
|
+
node, npm = _node_commands()
|
|
231
|
+
if not node or not npm:
|
|
232
|
+
_install_node()
|
|
233
|
+
node, npm = _node_commands()
|
|
234
|
+
if not all(path.is_file() for path in (OPEN_WEBSEARCH_CLI,TYPESCRIPT_PARSER,JAVA_PARSER,ANGULAR_PARSER)):
|
|
235
|
+
_run([npm, "ci", "--omit=dev", "--no-audit", "--no-fund"], cwd=NODE_DIR,
|
|
236
|
+
label="Web adapters and code parser via npm", idle_s=300)
|
|
237
|
+
if reuse_browser_runtime:
|
|
238
|
+
existing = Path(reuse_browser_runtime).expanduser().resolve()
|
|
239
|
+
if not (existing / "Scripts" / "python.exe").is_file():
|
|
240
|
+
raise RuntimeError(f"Invalid browser environment: {existing}")
|
|
241
|
+
link = BROWSER_DIR / ".venv"
|
|
242
|
+
if link.exists() and link.resolve() != existing:
|
|
243
|
+
raise RuntimeError(f"Another browser environment already exists at {link}")
|
|
244
|
+
if not link.exists():
|
|
245
|
+
_link_existing_browser(existing, link)
|
|
246
|
+
if not BROWSER_PYTHON.is_file():
|
|
247
|
+
venv.create(BROWSER_DIR / ".venv", with_pip=True)
|
|
248
|
+
if not _check()["browser_packages"]:
|
|
249
|
+
_run([str(BROWSER_PYTHON), "-m", "pip", "install", "--disable-pip-version-check",
|
|
250
|
+
"--no-input", "-r", str(BROWSER_DIR / "requirements.txt")],
|
|
251
|
+
label="Camoufox and Crawl4AI via pip", idle_s=600)
|
|
252
|
+
# Camoufox usa Firefox modificado; Crawl4AI usa Chromium via Playwright.
|
|
253
|
+
if not reuse_browser_runtime:
|
|
254
|
+
_run([str(BROWSER_PYTHON), str(BROWSER_DIR / "fetch_camoufox.py")], label="Camoufox browser (1.3 GB)",
|
|
255
|
+
idle_s=180, stream=True, attempts=3)
|
|
256
|
+
_run([str(BROWSER_PYTHON), "-m", "playwright", "install", "chromium"],
|
|
257
|
+
label="Chromium browser", idle_s=180, stream=True, attempts=3)
|
|
258
|
+
status = _check()
|
|
259
|
+
if not all(status.values()):
|
|
260
|
+
raise RuntimeError(f"Incomplete web runtime: {status}")
|
|
261
|
+
print("Smart Tool runtime installed: Node, open-websearch, TypeScript, Java, Angular, Camoufox and Crawl4AI.")
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def main():
|
|
265
|
+
parser = argparse.ArgumentParser(description="Install/check the Smart Tool web runtime")
|
|
266
|
+
parser.add_argument("--check", action="store_true", help="Show the status without installing")
|
|
267
|
+
parser.add_argument("--reuse-browser-runtime", metavar="FOLDER",
|
|
268
|
+
help="Reuse an existing venv without duplicating files (useful when disk is low)")
|
|
269
|
+
args = parser.parse_args()
|
|
270
|
+
if args.check:
|
|
271
|
+
for name, ready in _check().items():
|
|
272
|
+
print(f"{name}: {'ok' if ready else 'missing'}")
|
|
273
|
+
return 0 if all(_check().values()) else 1
|
|
274
|
+
try:
|
|
275
|
+
install(args.reuse_browser_runtime)
|
|
276
|
+
except (OSError, RuntimeError, subprocess.SubprocessError) as exc:
|
|
277
|
+
print(f"Web runtime installation failed: {exc}", file=sys.stderr)
|
|
278
|
+
return 1
|
|
279
|
+
return 0
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
if __name__ == "__main__":
|
|
283
|
+
raise SystemExit(main())
|