@allansantos-dev/smart-tool 0.9.79 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +65 -18
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_health.py +180 -0
- package/code_impact.py +13 -1
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +111 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/document_text.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Extração local de texto DOCX, PPTX, XLSX e PDF, sem Office, execução de macros ou acesso externo."""
|
|
2
2
|
import contextlib
|
|
3
|
+
import csv
|
|
3
4
|
import io
|
|
4
5
|
import json
|
|
5
6
|
import posixpath
|
|
@@ -9,6 +10,8 @@ import sys
|
|
|
9
10
|
import zipfile
|
|
10
11
|
import xml.etree.ElementTree as ET
|
|
11
12
|
import zlib
|
|
13
|
+
from html.parser import HTMLParser
|
|
14
|
+
from urllib.parse import unquote
|
|
12
15
|
|
|
13
16
|
VERSION = 3
|
|
14
17
|
PDF_MAX_BYTES = 20 * 1024 * 1024
|
|
@@ -58,7 +61,9 @@ class _Package:
|
|
|
58
61
|
self.label = label
|
|
59
62
|
self.xml_bytes = 0
|
|
60
63
|
|
|
61
|
-
def
|
|
64
|
+
def read(self, name):
|
|
65
|
+
"""The bytes of one part, charged to the XML budget; raises DocumentError for an encrypted or oddly compressed
|
|
66
|
+
part and for one that would pass the budget."""
|
|
62
67
|
info = self.entries[name]
|
|
63
68
|
if info.flag_bits & 1 or info.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
|
|
64
69
|
raise DocumentError(f'Encrypted {self.label} or unsupported compression.')
|
|
@@ -67,7 +72,13 @@ class _Package:
|
|
|
67
72
|
with self.archive.open(info) as stream:
|
|
68
73
|
raw = stream.read(MAX_XML_BYTES - self.xml_bytes + 1)
|
|
69
74
|
self.xml_bytes += len(raw)
|
|
70
|
-
if self.xml_bytes > MAX_XML_BYTES
|
|
75
|
+
if self.xml_bytes > MAX_XML_BYTES:
|
|
76
|
+
raise DocumentError(f'{self.label} contains excessive XML or disallowed entities.')
|
|
77
|
+
return raw
|
|
78
|
+
|
|
79
|
+
def parse(self, name):
|
|
80
|
+
raw = self.read(name)
|
|
81
|
+
if b'<!DOCTYPE' in raw.upper() or b'<!ENTITY' in raw.upper():
|
|
71
82
|
raise DocumentError(f'{self.label} contains excessive XML or disallowed entities.')
|
|
72
83
|
builder = _DocumentTreeBuilder()
|
|
73
84
|
builder.label = self.label
|
|
@@ -451,6 +462,182 @@ def extract_xlsx(data):
|
|
|
451
462
|
return ('\n'.join(lines) + '\n').encode('utf-8')
|
|
452
463
|
|
|
453
464
|
|
|
465
|
+
_SKIPPED_HTML = frozenset(('script', 'style', 'noscript', 'template', 'svg', 'head'))
|
|
466
|
+
_BLOCK_HTML = frozenset(('p', 'div', 'br', 'section', 'article', 'header', 'footer', 'main', 'aside', 'nav', 'ul', 'ol',
|
|
467
|
+
'blockquote', 'pre', 'hr', 'figure', 'figcaption', 'dl', 'dt', 'dd', 'table', 'body', 'form',
|
|
468
|
+
'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'))
|
|
469
|
+
_CHARSET = re.compile(rb'charset\s*=\s*["\']?([\w.:-]+)', re.I)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _decode_text(data, label, declared=False):
|
|
473
|
+
"""Text of a file's bytes: UTF-8 (a BOM is dropped), UTF-16 with a BOM, or, for markup, the charset it declares in
|
|
474
|
+
its first 2 KiB. A file that is none of those is refused, never decoded into wrong characters."""
|
|
475
|
+
if data.startswith((b'\xff\xfe', b'\xfe\xff')):
|
|
476
|
+
return data.decode('utf-16')
|
|
477
|
+
try:
|
|
478
|
+
return data.decode('utf-8-sig')
|
|
479
|
+
except UnicodeDecodeError:
|
|
480
|
+
found = _CHARSET.search(data[:2048]) if declared else None
|
|
481
|
+
if found:
|
|
482
|
+
try:
|
|
483
|
+
return data.decode(found.group(1).decode('ascii'))
|
|
484
|
+
except (LookupError, UnicodeDecodeError):
|
|
485
|
+
pass
|
|
486
|
+
raise DocumentError(f'{label} is not UTF-8 and declares no encoding the reader can use.')
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
class _HtmlText(HTMLParser):
|
|
490
|
+
"""Markdown blocks of an HTML document: headings as `#`, list items as `- `, table rows as `a | b`, the other text
|
|
491
|
+
as lines. No tree is built and nothing is fetched or executed; script, style and the head except its title are
|
|
492
|
+
skipped. Output goes through a _Lines budget, so a page over MAX_TEXT_BYTES of text is refused."""
|
|
493
|
+
|
|
494
|
+
def __init__(self, label):
|
|
495
|
+
super().__init__(convert_charrefs=True)
|
|
496
|
+
self.blocks = _Lines(label)
|
|
497
|
+
self.pieces, self.prefix, self.skipped = [], '', 0
|
|
498
|
+
self.row, self.cell, self.title, self.in_title = [], 0, [], False
|
|
499
|
+
|
|
500
|
+
def _flush(self):
|
|
501
|
+
text = ' '.join(''.join(self.pieces).split())
|
|
502
|
+
self.pieces = []
|
|
503
|
+
if text:
|
|
504
|
+
self.blocks.add(self.prefix + text)
|
|
505
|
+
self.prefix = ''
|
|
506
|
+
|
|
507
|
+
def handle_starttag(self, tag, attrs):
|
|
508
|
+
if tag == 'body':
|
|
509
|
+
self.skipped = 0
|
|
510
|
+
if tag == 'title':
|
|
511
|
+
self.in_title = not self.title
|
|
512
|
+
elif tag in _SKIPPED_HTML:
|
|
513
|
+
self.skipped += 1
|
|
514
|
+
elif self.skipped:
|
|
515
|
+
return
|
|
516
|
+
elif tag in ('td', 'th'):
|
|
517
|
+
self.cell += 1
|
|
518
|
+
elif tag == 'tr':
|
|
519
|
+
self._flush()
|
|
520
|
+
elif tag in _BLOCK_HTML and not self.cell:
|
|
521
|
+
self._flush()
|
|
522
|
+
if tag[0] == 'h' and len(tag) == 2:
|
|
523
|
+
self.prefix = '#' * int(tag[1]) + ' '
|
|
524
|
+
elif tag == 'li':
|
|
525
|
+
self.prefix = '- '
|
|
526
|
+
|
|
527
|
+
def handle_endtag(self, tag):
|
|
528
|
+
if tag == 'title':
|
|
529
|
+
self.in_title = False
|
|
530
|
+
elif tag in _SKIPPED_HTML:
|
|
531
|
+
self.skipped = max(0, self.skipped - 1)
|
|
532
|
+
elif self.skipped:
|
|
533
|
+
return
|
|
534
|
+
elif tag in ('td', 'th') and self.cell:
|
|
535
|
+
self.cell -= 1
|
|
536
|
+
self.row.append(' '.join(''.join(self.pieces).split()))
|
|
537
|
+
self.pieces = []
|
|
538
|
+
elif tag == 'tr':
|
|
539
|
+
if any(self.row):
|
|
540
|
+
self.blocks.add(' | '.join(self.row))
|
|
541
|
+
self.row = []
|
|
542
|
+
elif tag in _BLOCK_HTML and not self.cell:
|
|
543
|
+
self._flush()
|
|
544
|
+
|
|
545
|
+
def handle_data(self, data):
|
|
546
|
+
if self.in_title:
|
|
547
|
+
self.title.append(data)
|
|
548
|
+
elif not self.skipped:
|
|
549
|
+
self.pieces.append(data)
|
|
550
|
+
|
|
551
|
+
def result(self):
|
|
552
|
+
self.close()
|
|
553
|
+
self._flush()
|
|
554
|
+
title = ' '.join(''.join(self.title).split())
|
|
555
|
+
return ([f'# {title}'] if title else []) + list(self.blocks)
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def _html_blocks(data, label):
|
|
559
|
+
parser = _HtmlText(label)
|
|
560
|
+
parser.feed(_decode_text(data, label, declared=True))
|
|
561
|
+
return parser.result()
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def html_blocks(data):
|
|
565
|
+
"""Markdown blocks of an .html or .htm file in document order, for reading: the title as `#`, headings as `#`
|
|
566
|
+
levels, list items as `- `, table rows as `a | b`, other text as lines. Scripts, styles and the rest of the head
|
|
567
|
+
are not text; a link is its text.
|
|
568
|
+
|
|
569
|
+
Parameters: data (the file bytes). Returns: a list of blocks. Raises: DocumentError with a readable reason for an
|
|
570
|
+
encoding the reader cannot use, a page over MAX_TEXT_BYTES of text, or one without text."""
|
|
571
|
+
blocks = _html_blocks(data, 'HTML')
|
|
572
|
+
if not blocks:
|
|
573
|
+
raise DocumentError('HTML has no text.')
|
|
574
|
+
return blocks
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def csv_rows(data):
|
|
578
|
+
"""The rows of a .csv file, for reading: one tab-joined line per non-empty row (the delimiter is `,`, `;`, tab or
|
|
579
|
+
`|`, found from the first 4 KiB), a line break inside a cell turned into a space.
|
|
580
|
+
|
|
581
|
+
Parameters: data (the file bytes). Returns: a list of lines. Raises: DocumentError with a readable reason for an
|
|
582
|
+
encoding the reader cannot use, malformed quoting, a file over XLSX_MAX_ROWS rows or MAX_TEXT_BYTES of text, or one
|
|
583
|
+
without text."""
|
|
584
|
+
text = _decode_text(data, 'CSV')
|
|
585
|
+
try:
|
|
586
|
+
dialect = csv.Sniffer().sniff(text[:4096], delimiters=',;\t|')
|
|
587
|
+
except csv.Error:
|
|
588
|
+
dialect = csv.excel
|
|
589
|
+
rows = _Lines('CSV')
|
|
590
|
+
try:
|
|
591
|
+
for record in csv.reader(io.StringIO(text, newline=''), dialect):
|
|
592
|
+
values = [' '.join(cell.split()) for cell in record]
|
|
593
|
+
if any(values):
|
|
594
|
+
if len(rows) >= XLSX_MAX_ROWS:
|
|
595
|
+
raise DocumentError(f'CSV over {XLSX_MAX_ROWS} rows.')
|
|
596
|
+
rows.add('\t'.join(values))
|
|
597
|
+
except csv.Error as exc:
|
|
598
|
+
raise DocumentError(f'Could not read this CSV: {exc}.') from None
|
|
599
|
+
if not rows:
|
|
600
|
+
raise DocumentError('CSV has no text.')
|
|
601
|
+
return list(rows)
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def _epub_chapter_paths(package):
|
|
605
|
+
"""Zip names of the XHTML parts of an EPUB in reading order: the package document the container names, its spine
|
|
606
|
+
resolved through the manifest. A part outside the archive or a path leaving it is refused."""
|
|
607
|
+
container = package.parse('META-INF/container.xml')
|
|
608
|
+
rootfile = next((node.get('full-path') for node in container.iter() if _local(node.tag) == 'rootfile'), None)
|
|
609
|
+
opf = posixpath.normpath((rootfile or '').lstrip('/'))
|
|
610
|
+
if not rootfile or opf.startswith('..') or opf not in package.entries:
|
|
611
|
+
raise DocumentError('Invalid EPUB or excessive structure.')
|
|
612
|
+
package_root = package.parse(opf)
|
|
613
|
+
manifest = {node.get('id'): node for node in package_root.iter() if _local(node.tag) == 'item'}
|
|
614
|
+
base = posixpath.dirname(opf)
|
|
615
|
+
paths = []
|
|
616
|
+
for itemref in (node for node in package_root.iter() if _local(node.tag) == 'itemref'):
|
|
617
|
+
item = manifest.get(itemref.get('idref'))
|
|
618
|
+
if item is None or item.get('media-type') not in ('application/xhtml+xml', 'text/html'):
|
|
619
|
+
continue
|
|
620
|
+
part = posixpath.normpath(posixpath.join(base, unquote(item.get('href', '')).split('#')[0]))
|
|
621
|
+
if part.startswith('..') or part not in package.entries:
|
|
622
|
+
raise DocumentError('Invalid EPUB or excessive structure.')
|
|
623
|
+
paths.append(part)
|
|
624
|
+
return paths
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def epub_chapters(data):
|
|
628
|
+
"""The chapters of an .epub in reading order, for reading. Each XHTML part of the spine is read as HTML (see
|
|
629
|
+
html_blocks); the package is opened with the same guards as the Office formats.
|
|
630
|
+
|
|
631
|
+
Parameters: data (the file bytes). Returns: a list of block lists, one per chapter (a chapter without text is an
|
|
632
|
+
empty list). Raises: DocumentError with a readable reason for a corrupt, encrypted, oversized or hostile package, an
|
|
633
|
+
encoding the reader cannot use, or a book without any text."""
|
|
634
|
+
with _open_package(data, 'EPUB', 'META-INF/container.xml') as package:
|
|
635
|
+
chapters = [_html_blocks(package.read(path), 'EPUB') for path in _epub_chapter_paths(package)]
|
|
636
|
+
if not any(chapters):
|
|
637
|
+
raise DocumentError('EPUB has no text.')
|
|
638
|
+
return chapters
|
|
639
|
+
|
|
640
|
+
|
|
454
641
|
PDF_SCRIPT = """
|
|
455
642
|
import io, json, sys
|
|
456
643
|
import pypdf
|
|
@@ -524,14 +711,15 @@ def extract_pdf(data):
|
|
|
524
711
|
return (text + "\n").encode("utf-8", "replace")
|
|
525
712
|
|
|
526
713
|
|
|
527
|
-
_EXTENSIONS = ('.docx', '.pptx', '.xlsx', '.pdf')
|
|
714
|
+
_EXTENSIONS = ('.docx', '.pptx', '.xlsx', '.xlsm', '.pdf')
|
|
528
715
|
|
|
529
716
|
|
|
530
717
|
def is_extracted(path):
|
|
531
718
|
"""Whether the file type goes through a text extractor before indexing.
|
|
532
719
|
|
|
533
|
-
Parameters: path (any file name or path). Returns: True for `.docx`, `.pptx`, `.xlsx` and `.pdf`,
|
|
534
|
-
|
|
720
|
+
Parameters: path (any file name or path). Returns: True for `.docx`, `.pptx`, `.xlsx`, `.xlsm` and `.pdf`,
|
|
721
|
+
case-insensitive; an `.xlsm` is read as an `.xlsx` (its XML parts only, no macro runs); the binary and other macro
|
|
722
|
+
formats (`.xlsb`, `.xls`, `.ppt`, `.pptm`) are not extracted."""
|
|
535
723
|
return str(path).lower().endswith(_EXTENSIONS)
|
|
536
724
|
|
|
537
725
|
|
|
@@ -552,7 +740,7 @@ def extract(path, data):
|
|
|
552
740
|
return extract_pdf(data)
|
|
553
741
|
if name.endswith('.pptx'):
|
|
554
742
|
return extract_pptx(data)
|
|
555
|
-
if name.endswith('.xlsx'):
|
|
743
|
+
if name.endswith(('.xlsx', '.xlsm')):
|
|
556
744
|
return extract_xlsx(data)
|
|
557
745
|
return extract_docx(data)
|
|
558
746
|
|
package/dossier.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"""Dossier mode of smart_search: a narrow keyword query over the notes a project keeps about its own intent
|
|
2
|
-
(docs/research, the backlog, CLAUDE.md and AGENTS.md
|
|
3
|
-
budget. It reads straight from disk, uses no index and no model, and
|
|
4
|
-
stops at the deadline and says so in `degraded`/`reasons`.
|
|
2
|
+
(docs/research, the backlog, CLAUDE.md and AGENTS.md; with `history`, also the messages of its commits), answered with
|
|
3
|
+
ids and one short excerpt per hit, inside a time budget. It reads straight from disk, uses no index and no model, and
|
|
4
|
+
never raises on a slow or unreadable file: it stops at the deadline and says so in `degraded`/`reasons`. The commit
|
|
5
|
+
messages come from `doc_insights.history`, whose first read of a repository runs one `git log` that no deadline can
|
|
6
|
+
cut, so `history` is opt-in and a read that outlasts the budget is reported in `reasons`."""
|
|
5
7
|
import json
|
|
6
8
|
import os
|
|
7
9
|
import re
|
|
@@ -89,11 +91,20 @@ def _score(unit_lines, terms):
|
|
|
89
91
|
return sum((term in low) + (term in title) for term in terms)
|
|
90
92
|
|
|
91
93
|
|
|
92
|
-
def
|
|
94
|
+
def _commit_units(root):
|
|
95
|
+
"""`[(hash, subject, [lines])]` of the commits of `root`, newest first (a subject can carry the whole reason), and
|
|
96
|
+
the reason no history could be read (None when it was)."""
|
|
97
|
+
import doc_insights
|
|
98
|
+
commits, note = doc_insights.history(root)
|
|
99
|
+
return [(commit["hash"], commit["subject"], [commit["subject"], *commit["body"].splitlines()])
|
|
100
|
+
for commit in commits], note
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def query(root, text, timeout_ms=DEFAULT_TIMEOUT_MS, max_hits=DEFAULT_MAX_HITS, history=False):
|
|
93
104
|
"""The dossier answer for `text` over the notes of `root`: `{"mode", "degraded", "reasons", "elapsed_ms",
|
|
94
105
|
"scanned", "hits": [{"id", "source", "title", "snippet"}]}`, best hit first. `id` is `path:line` of the unit's
|
|
95
|
-
first line
|
|
96
|
-
was found until then."""
|
|
106
|
+
first line, or `git:<hash>` for a commit message (`history` true; source `git`). `degraded` is true when the
|
|
107
|
+
deadline cut the scan or a file could not be read; `hits` then holds what was found until then."""
|
|
97
108
|
started = time.monotonic()
|
|
98
109
|
deadline = started + timeout_ms / 1000
|
|
99
110
|
terms = focus_terms(text)
|
|
@@ -119,8 +130,21 @@ def query(root, text, timeout_ms=DEFAULT_TIMEOUT_MS, max_hits=DEFAULT_MAX_HITS):
|
|
|
119
130
|
scored.append((score, rel, kind, number, unit))
|
|
120
131
|
if unreadable:
|
|
121
132
|
reasons.append("not read: " + "; ".join(unreadable[:5]))
|
|
133
|
+
if history and time.monotonic() <= deadline:
|
|
134
|
+
commits, note = _commit_units(root)
|
|
135
|
+
scanned += 1
|
|
136
|
+
if note:
|
|
137
|
+
reasons.append(f"git history: {note}")
|
|
138
|
+
if time.monotonic() > deadline:
|
|
139
|
+
reasons.append(f"git history read took longer than {timeout_ms} ms")
|
|
140
|
+
for short, _subject, unit in commits:
|
|
141
|
+
score = _score(unit, terms)
|
|
142
|
+
if score:
|
|
143
|
+
scored.append((score, f"git:{short}", "git", None, unit))
|
|
144
|
+
elif history:
|
|
145
|
+
reasons.append(f"git history not read: timeout after {timeout_ms} ms")
|
|
122
146
|
scored.sort(key=lambda item: -item[0])
|
|
123
|
-
hits = [{"id": f"{rel}:{number}", "source": kind, "title": _clean(unit[0])[:TITLE_CHARS],
|
|
147
|
+
hits = [{"id": rel if number is None else f"{rel}:{number}", "source": kind, "title": _clean(unit[0])[:TITLE_CHARS],
|
|
124
148
|
"snippet": _snippet(unit, terms)} for _score_value, rel, kind, number, unit in scored[:max_hits]]
|
|
125
149
|
return {"mode": "dossier", "degraded": bool(reasons), "reasons": reasons,
|
|
126
150
|
"elapsed_ms": round((time.monotonic() - started) * 1000), "scanned": scanned, "hits": hits}
|
|
@@ -138,7 +162,10 @@ def answer(arguments):
|
|
|
138
162
|
raise ValueError("smart_search mode=dossier requires 'query_identifiers' (or 'doc_query') with at least one word.")
|
|
139
163
|
timeout_ms = _bounded(arguments, "timeout_ms", DEFAULT_TIMEOUT_MS, 1, MAX_TIMEOUT_MS)
|
|
140
164
|
max_hits = _bounded(arguments, "max_hits", DEFAULT_MAX_HITS, 1, MAX_HITS)
|
|
141
|
-
|
|
165
|
+
history = arguments.get("history", False)
|
|
166
|
+
if not isinstance(history, bool):
|
|
167
|
+
raise ValueError("history must be true or false.")
|
|
168
|
+
return json.dumps(query(root, text, timeout_ms, max_hits, history), ensure_ascii=False, separators=(",", ":"))
|
|
142
169
|
|
|
143
170
|
|
|
144
171
|
def _bounded(arguments, name, default, low, high):
|
package/embedding_cache.py
CHANGED
|
@@ -12,13 +12,35 @@ import time
|
|
|
12
12
|
import project_identity
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
def _open(root, directory):
|
|
16
|
+
os.makedirs(directory, exist_ok=True)
|
|
17
|
+
db = sqlite3.connect(os.path.join(directory, project_identity.project_id(root) + '.vectors.sqlite3'), timeout=10)
|
|
18
|
+
db.execute('PRAGMA secure_delete=ON')
|
|
19
|
+
db.execute('CREATE TABLE IF NOT EXISTS vectors(key TEXT PRIMARY KEY, vector BLOB NOT NULL, used_at REAL NOT NULL, '
|
|
20
|
+
'model TEXT)')
|
|
21
|
+
if 'model' not in {row[1] for row in db.execute('PRAGMA table_info(vectors)')}:
|
|
22
|
+
db.execute('ALTER TABLE vectors ADD COLUMN model TEXT')
|
|
23
|
+
return db
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def drop_other_models(root, directory, model):
|
|
27
|
+
"""Deletes the cached vectors of every model but `model` and gives their space back to the disk. Rows written
|
|
28
|
+
before the cache recorded the model are deleted too, unless a build of `model` reused them since."""
|
|
29
|
+
db = _open(root, directory)
|
|
30
|
+
try:
|
|
31
|
+
dropped = db.execute('DELETE FROM vectors WHERE model IS NOT ?', (model,)).rowcount
|
|
32
|
+
db.commit()
|
|
33
|
+
if dropped:
|
|
34
|
+
db.execute('VACUUM')
|
|
35
|
+
return dropped
|
|
36
|
+
finally:
|
|
37
|
+
db.close()
|
|
38
|
+
|
|
39
|
+
|
|
15
40
|
class VectorCache:
|
|
16
41
|
def __init__(self, root, directory, contract):
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
self.db = sqlite3.connect(path, timeout=10)
|
|
20
|
-
self.db.execute('PRAGMA secure_delete=ON')
|
|
21
|
-
self.db.execute('CREATE TABLE IF NOT EXISTS vectors(key TEXT PRIMARY KEY, vector BLOB NOT NULL, used_at REAL NOT NULL)')
|
|
42
|
+
self.db = _open(root, directory)
|
|
43
|
+
self.model = contract.get('model')
|
|
22
44
|
self.contract = json.dumps(contract, sort_keys=True, ensure_ascii=False)
|
|
23
45
|
self.dimensions = contract.get('dimensions')
|
|
24
46
|
self.hits = self.misses = self.characters = 0
|
|
@@ -32,18 +54,20 @@ class VectorCache:
|
|
|
32
54
|
packed = array.array('f', vector)
|
|
33
55
|
if not all(math.isfinite(value) for value in packed):
|
|
34
56
|
raise ValueError('Embedding does not fit in float32; cache not written.')
|
|
35
|
-
self.db.execute('INSERT OR REPLACE INTO vectors VALUES(
|
|
36
|
-
(self.key(text), packed.tobytes(), time.time()))
|
|
57
|
+
self.db.execute('INSERT OR REPLACE INTO vectors VALUES(?,?,?,?)',
|
|
58
|
+
(self.key(text), packed.tobytes(), time.time(), self.model))
|
|
37
59
|
|
|
38
60
|
def embed(self, texts, embed_batch, validate, use_cache=True, split=None, workers=1):
|
|
39
61
|
"""Cache lookups and writes stay on the calling thread (sqlite); only embed_batch runs in workers."""
|
|
40
62
|
output = [None] * len(texts)
|
|
41
63
|
missing = {}
|
|
42
64
|
for i, text in enumerate(texts):
|
|
43
|
-
row = self.db.execute('SELECT vector FROM vectors WHERE key=?', (self.key(text),)).fetchone() if use_cache else None
|
|
65
|
+
row = self.db.execute('SELECT vector, model FROM vectors WHERE key=?', (self.key(text),)).fetchone() if use_cache else None
|
|
44
66
|
if row and self.dimensions:
|
|
45
67
|
vector = array.array('f'); vector.frombytes(row[0])
|
|
46
68
|
validate(vector, self.dimensions)
|
|
69
|
+
if row[1] is None:
|
|
70
|
+
self.db.execute('UPDATE vectors SET model=? WHERE key=?', (self.model, self.key(text)))
|
|
47
71
|
output[i] = list(vector)
|
|
48
72
|
self.hits += 1
|
|
49
73
|
else:
|
package/freshness.py
CHANGED
|
@@ -74,7 +74,8 @@ def npm_registry_evidence(package):
|
|
|
74
74
|
|
|
75
75
|
NPM_NAME_RE = re.compile(r"(?:@[a-z0-9._-]+/)?[a-z0-9._-]+")
|
|
76
76
|
NAME_RES = {"npm": NPM_NAME_RE, "PyPI": re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*"),
|
|
77
|
-
"Go": re.compile(r"[A-Za-z0-9][A-Za-z0-9._~/-]*")
|
|
77
|
+
"Go": re.compile(r"[A-Za-z0-9][A-Za-z0-9._~/-]*"), "crates.io": re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*"),
|
|
78
|
+
"Maven": re.compile(r"[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+")}
|
|
78
79
|
|
|
79
80
|
|
|
80
81
|
def _osv_query(ecosystem, package, package_version):
|
|
@@ -120,7 +121,7 @@ def npm_advisories(package, package_version):
|
|
|
120
121
|
def advisories_batch(ecosystem, packages, timeout):
|
|
121
122
|
"""Advisory IDs for each (package, version) of `packages` from one OSV `querybatch` call.
|
|
122
123
|
|
|
123
|
-
Parameters: ecosystem (str) "npm", "PyPI" or "
|
|
124
|
+
Parameters: ecosystem (str) "npm", "PyPI", "Go", "crates.io" or "Maven"; packages (list of (name, version)); timeout (float) seconds for
|
|
124
125
|
the request.
|
|
125
126
|
Returns: list in the same order, each the sorted ID list or None when that package's answer is incomplete
|
|
126
127
|
(next_page_token) or malformed, so it is unverified and not clean.
|
package/hook_decision.py
CHANGED
|
@@ -19,6 +19,7 @@ import code_graph
|
|
|
19
19
|
import code_impact
|
|
20
20
|
import comment_check
|
|
21
21
|
import config
|
|
22
|
+
import credential_check
|
|
22
23
|
import dependency_notice
|
|
23
24
|
import doc_check
|
|
24
25
|
import doc_scope
|
|
@@ -26,6 +27,7 @@ import duplicates
|
|
|
26
27
|
import edit_preview
|
|
27
28
|
import index_profile
|
|
28
29
|
import index_scope
|
|
30
|
+
import layer_check
|
|
29
31
|
import endpoint_sync
|
|
30
32
|
import project_admission
|
|
31
33
|
import project_store
|
|
@@ -283,7 +285,22 @@ def _duplicate_lines(findings):
|
|
|
283
285
|
+ " If the copy is intentional (moving the function, a deliberate fork), go ahead.")
|
|
284
286
|
|
|
285
287
|
|
|
286
|
-
def
|
|
288
|
+
def _layer_lines(findings):
|
|
289
|
+
lines = [f"`{item['function']}` ({rel}:{item['line']}) now calls `{item['called']}` ({item['path']}), a function of the "
|
|
290
|
+
f"higher layer {item['to']}, from layer {item['from']}" for rel, item in findings]
|
|
291
|
+
return ("Smart Tool layers: " + "; ".join(lines) + ". Calls go down the declared layers (.smart-tool/layers.json); "
|
|
292
|
+
"if this one is intended, go ahead.")
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _credential_lines(findings):
|
|
296
|
+
lines = [f"{rel}:{item['line']} looks like a {item['kind']}" for rel, item in findings[:MAX_DOC_FINDINGS]]
|
|
297
|
+
if len(findings) > MAX_DOC_FINDINGS:
|
|
298
|
+
lines.append(f"{len(findings) - MAX_DOC_FINDINGS} more line(s) in the same situation.")
|
|
299
|
+
return ("Smart Tool credentials: " + "; ".join(lines) + ". Keep it out of the file: read it from an environment "
|
|
300
|
+
"variable or a secret store. If it is a fake value for a test, go ahead.")
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _edit_review(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode="off", layer_mode="off"):
|
|
287
304
|
"""Checks of an edit before it runs. Docstrings follow doc_mode, in code files of registered projects only (tests,
|
|
288
305
|
docs and files outside a project are left out, as in the docs coverage): missing ones deny the edit in require mode and become a reminder in remind mode; a
|
|
289
306
|
documented function whose signature changed always gets a reminder. Comments the edit adds in the wrong place
|
|
@@ -291,7 +308,7 @@ def _edit_review(payload, doc_mode, duplicate_mode, dependency_mode, comment_mod
|
|
|
291
308
|
functions follow duplicate_mode and only add context, never deny. An edit of a package.json, requirements*.txt or go.mod follows dependency_mode: advisories
|
|
292
309
|
of the added or changed dependencies are added as context, never a denial."""
|
|
293
310
|
with code_graph.pending_scope():
|
|
294
|
-
return _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode)
|
|
311
|
+
return _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode, layer_mode)
|
|
295
312
|
|
|
296
313
|
|
|
297
314
|
def _dependency_part(before, after, ecosystem):
|
|
@@ -307,19 +324,25 @@ def _dependency_part(before, after, ecosystem):
|
|
|
307
324
|
return f"{dependency_notice.PREFIX}dependency check skipped: {exc}"
|
|
308
325
|
|
|
309
326
|
|
|
310
|
-
def
|
|
311
|
-
"""
|
|
327
|
+
def _review_changes(changes, doc_mode, duplicate_mode, dependency_mode, comment_mode, layer_mode="off"):
|
|
328
|
+
"""The checks of an edit over (path, before, after) triples: docstrings, comments, documentation anchors, copies and
|
|
329
|
+
dependency advisories, each following its mode.
|
|
330
|
+
|
|
331
|
+
Parameters: changes (iterable of (absolute path, text before or None, text after)); the four modes (`off` skips a check).
|
|
332
|
+
Returns: tuple (message parts, doc findings, comment findings); the parts are empty when nothing is worth saying.
|
|
333
|
+
"""
|
|
312
334
|
doc_findings, scope_findings, duplicate_findings, failures, dependency_parts = [], [], [], [], []
|
|
313
|
-
comment_findings = []
|
|
314
|
-
for path, before, after in
|
|
335
|
+
comment_findings, layer_findings, credential_findings = [], [], []
|
|
336
|
+
for path, before, after in changes:
|
|
337
|
+
root = _project_of(path)
|
|
338
|
+
rel = os.path.relpath(os.path.realpath(path), os.path.realpath(root)).replace(os.sep, "/") if root else os.path.basename(path)
|
|
339
|
+
credential_findings += [(rel, item) for item in credential_check.review(before, after)]
|
|
315
340
|
ecosystem = dependency_notice.ecosystem_of(path)
|
|
316
341
|
if dependency_mode != "off" and ecosystem:
|
|
317
342
|
dependency_parts += [part for part in (_dependency_part(before, after, ecosystem),) if part]
|
|
318
343
|
continue
|
|
319
344
|
if not doc_check.supported(path):
|
|
320
345
|
continue
|
|
321
|
-
root = _project_of(path)
|
|
322
|
-
rel = os.path.relpath(os.path.realpath(path), os.path.realpath(root)).replace(os.sep, "/") if root else os.path.basename(path)
|
|
323
346
|
edited = doc_check.written(path, before, after)
|
|
324
347
|
profile = index_profile.current((index_scope.load_scope(root) or {}).get("profile")) if root else None
|
|
325
348
|
if doc_mode != "off" and root and index_profile.kind(rel, profile) == "code":
|
|
@@ -336,10 +359,37 @@ def _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode, comm
|
|
|
336
359
|
duplicate_findings += [(rel, item) for item in duplicates.on_write(root, path, before, after, edited)]
|
|
337
360
|
except (ValueError, RuntimeError) as exc:
|
|
338
361
|
failures.append(f"Smart Tool duplicates check skipped: {exc}")
|
|
362
|
+
if layer_mode != "off" and root and edited["touched"]:
|
|
363
|
+
try:
|
|
364
|
+
layer_findings += [(rel, item) for item in layer_check.review(root, rel, before, after, edited)]
|
|
365
|
+
except (ValueError, RuntimeError, OSError) as exc:
|
|
366
|
+
failures.append(f"Smart Tool layers check skipped: {exc}")
|
|
339
367
|
parts = ([_doc_lines(doc_findings)] if doc_findings else []) + \
|
|
340
368
|
([_comment_lines(comment_findings)] if comment_findings else []) + \
|
|
341
369
|
([_scope_lines(scope_findings)] if scope_findings else []) + \
|
|
342
|
-
([_duplicate_lines(duplicate_findings)] if duplicate_findings else []) +
|
|
370
|
+
([_duplicate_lines(duplicate_findings)] if duplicate_findings else []) + \
|
|
371
|
+
([_layer_lines(layer_findings)] if layer_findings else []) + \
|
|
372
|
+
([_credential_lines(credential_findings)] if credential_findings else []) + failures + dependency_parts
|
|
373
|
+
return parts, doc_findings, comment_findings
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def review_diff(changes, cfg):
|
|
377
|
+
"""The notes the edit hook would have added for changes made without it (a commit, a patch applied by hand).
|
|
378
|
+
|
|
379
|
+
Parameters: changes (iterable of (absolute path, text before or None, text after)); cfg (dict) the loaded config.
|
|
380
|
+
Returns: str, the hook's message for the whole set; empty when every check is quiet or off.
|
|
381
|
+
"""
|
|
382
|
+
modes = (config.doc_mode(cfg), config.duplicate_mode(cfg), config.dependency_mode(cfg), config.comment_mode(cfg),
|
|
383
|
+
config.layer_mode(cfg))
|
|
384
|
+
with code_graph.pending_scope():
|
|
385
|
+
return " ".join(_review_changes(changes, *modes)[0])
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode="off", layer_mode="off"):
|
|
389
|
+
"""`_edit_review` inside one `code_graph.pending_scope`, so the pending-changes scan runs once per review."""
|
|
390
|
+
parts, doc_findings, comment_findings = _review_changes(
|
|
391
|
+
edit_preview.preview(payload.get("tool_name"), payload.get("tool_input"), payload.get("cwd")),
|
|
392
|
+
doc_mode, duplicate_mode, dependency_mode, comment_mode, layer_mode)
|
|
343
393
|
if not parts:
|
|
344
394
|
return {}
|
|
345
395
|
message = " ".join(parts)
|
|
@@ -354,7 +404,7 @@ def _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode, comm
|
|
|
354
404
|
|
|
355
405
|
def decide(payload, client, web_route, jobs):
|
|
356
406
|
"""Hook output for one PreToolUse payload: edits follow doc_mode (require | remind | off) and duplicate_mode
|
|
357
|
-
(warn | off) and dependency_mode (warn | off), the other tools follow hook_mode (redirect | advise | off). In redirect mode the Smart Tool call that
|
|
407
|
+
(warn | off) and dependency_mode (warn | off), and every edit review also notes credentials the edit writes, the other tools follow hook_mode (redirect | advise | off). In redirect mode the Smart Tool call that
|
|
358
408
|
replaces a redirected one starts in `jobs` under the call's tool_use_id (see `_redirect`); only a search whose
|
|
359
409
|
query cannot be built, or a client that sends no tool_use_id, is denied with the exact call to make, and never in a
|
|
360
410
|
subagent whose tool list has no Smart Tool MCP tool: it has nothing to switch to."""
|
|
@@ -362,11 +412,12 @@ def decide(payload, client, web_route, jobs):
|
|
|
362
412
|
if isinstance(payload, dict):
|
|
363
413
|
requester.note_origin(payload.get("session_id"), payload.get("cwd"))
|
|
364
414
|
if isinstance(payload, dict) and payload.get("tool_name") in EDIT_TOOLS:
|
|
365
|
-
doc_mode, duplicate_mode, dependency_mode, comment_mode = (
|
|
366
|
-
config.doc_mode(cfg), config.duplicate_mode(cfg), config.dependency_mode(cfg), config.comment_mode(cfg)
|
|
367
|
-
|
|
415
|
+
doc_mode, duplicate_mode, dependency_mode, comment_mode, layer_mode = (
|
|
416
|
+
config.doc_mode(cfg), config.duplicate_mode(cfg), config.dependency_mode(cfg), config.comment_mode(cfg),
|
|
417
|
+
config.layer_mode(cfg))
|
|
418
|
+
if "off" == doc_mode == duplicate_mode == dependency_mode == comment_mode == layer_mode:
|
|
368
419
|
return {}
|
|
369
|
-
return _edit_review(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode)
|
|
420
|
+
return _edit_review(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode, layer_mode)
|
|
370
421
|
if isinstance(payload, dict) and str(payload.get("tool_name")).startswith(f"mcp__{SMART_TOOL_SERVER}__"):
|
|
371
422
|
if payload["tool_name"] == SMART_SEARCH_TOOL:
|
|
372
423
|
router.CLIENT.set(client)
|