@allansantos-dev/smart-tool 0.9.78 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +79 -0
- package/README.md +65 -18
- package/affected_tests.py +4 -0
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_graph.py +39 -17
- package/code_graph_go.py +6 -6
- package/code_graph_java.mjs +10 -9
- package/code_graph_js.cjs +8 -8
- package/code_health.py +180 -0
- package/code_impact.py +16 -2
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/doc_scope.py +1 -1
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/duplicates.py +1 -1
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/layers.py +3 -3
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +113 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/document_read.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
"""read_document: PDF, Word, PowerPoint and
|
|
1
|
+
"""read_document: PDF, Word, PowerPoint, Excel, HTML, CSV and EPUB files read as compact text, one page, slide, sheet,
|
|
2
|
+
part or chapter range at a time.
|
|
2
3
|
|
|
3
4
|
The agent gets the file facts first (type, size, pages, slides or sheets, estimated tokens) and then only the range it
|
|
4
5
|
asks for, so a 200-page PDF is never loaded whole. Extraction is the one of document_text (pypdf in an isolated
|
|
5
|
-
subprocess for PDF; zipfile and ElementTree with the shared guards for docx, pptx and
|
|
6
|
-
split, the markdown shape, the compaction of PDF line wrapping and the range arithmetic."""
|
|
6
|
+
subprocess for PDF; zipfile and ElementTree with the shared guards for docx, pptx, xlsx and epub; the standard HTML
|
|
7
|
+
and CSV parsers for html and csv); this module adds the unit split, the markdown shape, the compaction of PDF line wrapping and the range arithmetic."""
|
|
7
8
|
import collections
|
|
8
9
|
import math
|
|
9
10
|
import os
|
|
@@ -11,10 +12,14 @@ import re
|
|
|
11
12
|
import threading
|
|
12
13
|
import unicodedata
|
|
13
14
|
|
|
15
|
+
import document_converter
|
|
14
16
|
import document_text
|
|
15
17
|
|
|
16
|
-
KINDS = {'.pdf': 'pdf', '.docx': 'docx', '.pptx': 'pptx', '.xlsx': 'xlsx'
|
|
17
|
-
|
|
18
|
+
KINDS = {'.pdf': 'pdf', '.docx': 'docx', '.pptx': 'pptx', '.xlsx': 'xlsx', '.xlsm': 'xlsx', '.html': 'html', '.htm': 'html',
|
|
19
|
+
'.csv': 'csv', '.epub': 'epub'}
|
|
20
|
+
UNIT_NAMES = {'pdf': 'page', 'docx': 'part', 'pptx': 'slide', 'xlsx': 'sheet', 'html': 'part', 'csv': 'sheet',
|
|
21
|
+
'epub': 'chapter', 'converted': 'part'}
|
|
22
|
+
LABELLED_KINDS = ('pdf', 'docx', 'html', 'epub', 'converted')
|
|
18
23
|
DEFAULT_MAX_CHARS = 20_000
|
|
19
24
|
MAX_CHARS_LIMIT = 100_000
|
|
20
25
|
DOCX_PART_CHARS = 6_000
|
|
@@ -26,6 +31,14 @@ _cache = collections.OrderedDict()
|
|
|
26
31
|
_cache_lock = threading.Lock()
|
|
27
32
|
|
|
28
33
|
|
|
34
|
+
class NoTextLayer(document_text.DocumentError):
|
|
35
|
+
"""A PDF whose pages carry no text: `pages` is its page count."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, pages):
|
|
38
|
+
super().__init__(f'PDF of {pages} pages has no text layer (scanned pages are not read; there is no OCR).')
|
|
39
|
+
self.pages = pages
|
|
40
|
+
|
|
41
|
+
|
|
29
42
|
class Unit:
|
|
30
43
|
"""One readable unit of a document: a PDF page, a docx part, a slide or a sheet."""
|
|
31
44
|
|
|
@@ -74,13 +87,12 @@ def compact_pdf_text(text):
|
|
|
74
87
|
|
|
75
88
|
|
|
76
89
|
def _pdf_units(data):
|
|
77
|
-
"""Units and facts of a PDF, one unit per page; raises DocumentError for a password
|
|
78
|
-
|
|
90
|
+
"""Units and facts of a PDF, one unit per page; raises DocumentError for a password or a corrupt file, and NoTextLayer
|
|
91
|
+
for a file with no text layer at all."""
|
|
79
92
|
pages, count = document_text.pdf_pages(data, document_text.PDF_TIMEOUT_S, document_text.PDF_MAX_PAGES,
|
|
80
93
|
document_text.PDF_MAX_CHARS)
|
|
81
94
|
if not any(pages):
|
|
82
|
-
raise
|
|
83
|
-
f'PDF of {count} pages has no text layer (scanned pages are not read; there is no OCR).')
|
|
95
|
+
raise NoTextLayer(count)
|
|
84
96
|
units = [Unit(number, compact_pdf_text(page) or '(no text on this page)') for number, page in enumerate(pages, 1)]
|
|
85
97
|
facts = {'pages': count, 'pages without text': sum(1 for page in pages if not page)}
|
|
86
98
|
if count > len(pages):
|
|
@@ -89,21 +101,47 @@ def _pdf_units(data):
|
|
|
89
101
|
return units, facts
|
|
90
102
|
|
|
91
103
|
|
|
92
|
-
def
|
|
93
|
-
"""
|
|
94
|
-
split (a docx has no fixed pages, they depend on the renderer)."""
|
|
104
|
+
def _parts(blocks):
|
|
105
|
+
"""The markdown blocks grouped in parts of about DOCX_PART_CHARS characters, a block never split."""
|
|
95
106
|
parts, current, size = [], [], 0
|
|
96
|
-
for block in
|
|
107
|
+
for block in blocks:
|
|
97
108
|
if current and size + len(block) > DOCX_PART_CHARS:
|
|
98
109
|
parts.append('\n'.join(current))
|
|
99
110
|
current, size = [], 0
|
|
100
111
|
current.append(block)
|
|
101
112
|
size += len(block) + 1
|
|
102
113
|
parts.append('\n'.join(current))
|
|
114
|
+
return parts
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _docx_units(data):
|
|
118
|
+
"""Units and facts of a docx: the markdown blocks grouped in parts (a docx has no fixed pages, they depend on the
|
|
119
|
+
renderer)."""
|
|
120
|
+
parts = _parts(document_text.docx_blocks(data))
|
|
103
121
|
return [Unit(number, text) for number, text in enumerate(parts, 1)], {
|
|
104
122
|
'parts': len(parts), 'note': f'a docx has no fixed pages; a part is about {DOCX_PART_CHARS:,} characters'}
|
|
105
123
|
|
|
106
124
|
|
|
125
|
+
def _html_units(data):
|
|
126
|
+
"""Units and facts of an html file: the markdown blocks grouped in parts, as a docx (an html file has no pages)."""
|
|
127
|
+
parts = _parts(document_text.html_blocks(data))
|
|
128
|
+
return [Unit(number, text) for number, text in enumerate(parts, 1)], {
|
|
129
|
+
'parts': len(parts), 'note': f'a part is about {DOCX_PART_CHARS:,} characters of the page text'}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _csv_units(data):
|
|
133
|
+
"""Units and facts of a csv file: one unit, the sheet `csv`, one tab-separated line per row."""
|
|
134
|
+
rows = document_text.csv_rows(data)
|
|
135
|
+
return [Unit(1, '\n'.join(['[sheet csv]'] + rows), 'csv')], {'rows': len(rows)}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _epub_units(data):
|
|
139
|
+
"""Units and facts of an epub: one unit per chapter of the reading order."""
|
|
140
|
+
chapters = document_text.epub_chapters(data)
|
|
141
|
+
units = [Unit(number, '\n'.join(blocks) or '(no text)') for number, blocks in enumerate(chapters, 1)]
|
|
142
|
+
return units, {'chapters without text': sum(1 for blocks in chapters if not blocks)}
|
|
143
|
+
|
|
144
|
+
|
|
107
145
|
def _pptx_units(data):
|
|
108
146
|
"""Units and facts of a pptx: one unit per slide under `[slide N]`, the speaker notes under `[notes N]`."""
|
|
109
147
|
units = []
|
|
@@ -121,14 +159,51 @@ def _xlsx_units(data):
|
|
|
121
159
|
return units, {'sheet names': ', '.join(f'{u.number}={u.name}' for u in units)}
|
|
122
160
|
|
|
123
161
|
|
|
124
|
-
|
|
162
|
+
def _converted_pdf_units(path, count):
|
|
163
|
+
"""Units and facts of a PDF with no text layer, read by the external converter as page images."""
|
|
164
|
+
result = document_converter.convert(path, expected_pages=count, force_ocr=True)
|
|
165
|
+
units = [Unit(number, page.strip() or '(no text on this page)') for number, page in enumerate(result.pages, 1)]
|
|
166
|
+
facts = {'pages without text': sum(1 for page in result.pages if not page.strip()),
|
|
167
|
+
'read by': f'{result.engine} (OCR of the page images)'}
|
|
168
|
+
if len(units) != count:
|
|
169
|
+
facts['note'] = f'the converter returned {len(units)} pages for a file of {count}'
|
|
170
|
+
return units, facts
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _converted_units(path):
|
|
174
|
+
"""Units and facts of a file the native reader has no type for, read by the external converter: its text grouped in
|
|
175
|
+
parts as a docx."""
|
|
176
|
+
result = document_converter.convert(path)
|
|
177
|
+
text = '\n\n'.join(page.strip() for page in result.pages if page.strip())
|
|
178
|
+
parts = _parts(text.split('\n'))
|
|
179
|
+
return [Unit(number, part) for number, part in enumerate(parts, 1)], {'parts': len(parts),
|
|
180
|
+
'read by': result.engine}
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _build(path, kind):
|
|
184
|
+
"""Units and facts of the file at `path`: its native builder, or, for a kind the native reader has none for and for a
|
|
185
|
+
PDF without a text layer, the external converter when the setting turns it on."""
|
|
186
|
+
if kind == 'converted':
|
|
187
|
+
return _converted_units(path)
|
|
188
|
+
with open(path, 'rb') as stream:
|
|
189
|
+
data = stream.read()
|
|
190
|
+
try:
|
|
191
|
+
return _BUILDERS[kind](data)
|
|
192
|
+
except NoTextLayer as exc:
|
|
193
|
+
if not document_converter.enabled():
|
|
194
|
+
raise
|
|
195
|
+
return _converted_pdf_units(path, exc.pages)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
_BUILDERS = {'pdf': _pdf_units, 'docx': _docx_units, 'pptx': _pptx_units, 'xlsx': _xlsx_units, 'html': _html_units,
|
|
199
|
+
'csv': _csv_units, 'epub': _epub_units}
|
|
125
200
|
|
|
126
201
|
|
|
127
202
|
def load(path):
|
|
128
203
|
"""The document at `path`, split into units. Parsed documents are kept for a few calls (keyed by path, size and
|
|
129
204
|
modification time), so reading page after page parses the file once.
|
|
130
205
|
|
|
131
|
-
Parameters: path (absolute path of a .pdf, .docx, .pptx or .
|
|
206
|
+
Parameters: path (absolute path of a .pdf, .docx, .pptx, .xlsx, .html, .htm, .csv or .epub file). Returns: a Document. Raises: ValueError for a
|
|
132
207
|
relative path, a missing file, an unsupported type or a file over the size limit of its type; DocumentError with a
|
|
133
208
|
readable reason for a file the extractor refuses (corrupt, encrypted, hostile, scanned, without text)."""
|
|
134
209
|
if not isinstance(path, str) or not path.strip():
|
|
@@ -136,27 +211,27 @@ def load(path):
|
|
|
136
211
|
if not os.path.isabs(path):
|
|
137
212
|
raise ValueError(f'read_document needs an absolute path, got {path!r}')
|
|
138
213
|
extension = os.path.splitext(path)[1].lower()
|
|
139
|
-
|
|
214
|
+
converted = extension not in KINDS and extension in document_converter.EXTENSIONS and document_converter.enabled()
|
|
215
|
+
if extension not in KINDS and not converted:
|
|
140
216
|
raise ValueError(f"read_document reads {', '.join(KINDS)} files; {extension or 'this file'} is not supported "
|
|
141
|
-
'(use the normal Read for plain text; legacy .doc, .xls and .ppt and macro files are not read)')
|
|
217
|
+
'(use the normal Read for plain text; legacy .doc, .xls, .xlsb and .ppt and macro files other than .xlsm are not read)')
|
|
142
218
|
try:
|
|
143
219
|
info = os.stat(path)
|
|
144
220
|
except OSError as exc:
|
|
145
221
|
raise ValueError(f'cannot read {path}: {exc.strerror or type(exc).__name__}') from None
|
|
146
222
|
if not os.path.isfile(path):
|
|
147
223
|
raise ValueError(f'{path} is not a file')
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
224
|
+
limit = document_text.PDF_MAX_BYTES if converted else document_text.size_limit(path)
|
|
225
|
+
if info.st_size > limit:
|
|
226
|
+
raise ValueError(f'{path} is {info.st_size / 1048576:.1f} MiB, over the {limit // 1048576} MiB limit of a '
|
|
227
|
+
f'{extension} file')
|
|
151
228
|
key = (os.path.normcase(path), info.st_size, info.st_mtime_ns)
|
|
152
229
|
with _cache_lock:
|
|
153
230
|
if key in _cache:
|
|
154
231
|
_cache.move_to_end(key)
|
|
155
232
|
return _cache[key]
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
kind = KINDS[extension]
|
|
159
|
-
units, facts = _BUILDERS[kind](data)
|
|
233
|
+
kind = 'converted' if converted else KINDS[extension]
|
|
234
|
+
units, facts = _build(path, kind)
|
|
160
235
|
document = Document(path, kind, info.st_size, units, facts)
|
|
161
236
|
with _cache_lock:
|
|
162
237
|
_cache[key] = document
|
|
@@ -241,7 +316,7 @@ def read(path, range_spec=None, max_chars=DEFAULT_MAX_CHARS, info_only=False):
|
|
|
241
316
|
body, shown, size, cut = [], [], 0, None
|
|
242
317
|
for number in numbers:
|
|
243
318
|
text = document.units[number - 1].text
|
|
244
|
-
label = f'[{unit} {number}]\n' if document.kind in
|
|
319
|
+
label = f'[{unit} {number}]\n' if document.kind in LABELLED_KINDS else ''
|
|
245
320
|
piece = label + text
|
|
246
321
|
if shown and size + len(piece) > max_chars:
|
|
247
322
|
break
|
package/document_text.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Extração local de texto DOCX, PPTX, XLSX e PDF, sem Office, execução de macros ou acesso externo."""
|
|
2
2
|
import contextlib
|
|
3
|
+
import csv
|
|
3
4
|
import io
|
|
4
5
|
import json
|
|
5
6
|
import posixpath
|
|
@@ -9,6 +10,8 @@ import sys
|
|
|
9
10
|
import zipfile
|
|
10
11
|
import xml.etree.ElementTree as ET
|
|
11
12
|
import zlib
|
|
13
|
+
from html.parser import HTMLParser
|
|
14
|
+
from urllib.parse import unquote
|
|
12
15
|
|
|
13
16
|
VERSION = 3
|
|
14
17
|
PDF_MAX_BYTES = 20 * 1024 * 1024
|
|
@@ -58,7 +61,9 @@ class _Package:
|
|
|
58
61
|
self.label = label
|
|
59
62
|
self.xml_bytes = 0
|
|
60
63
|
|
|
61
|
-
def
|
|
64
|
+
def read(self, name):
|
|
65
|
+
"""The bytes of one part, charged to the XML budget; raises DocumentError for an encrypted or oddly compressed
|
|
66
|
+
part and for one that would pass the budget."""
|
|
62
67
|
info = self.entries[name]
|
|
63
68
|
if info.flag_bits & 1 or info.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
|
|
64
69
|
raise DocumentError(f'Encrypted {self.label} or unsupported compression.')
|
|
@@ -67,7 +72,13 @@ class _Package:
|
|
|
67
72
|
with self.archive.open(info) as stream:
|
|
68
73
|
raw = stream.read(MAX_XML_BYTES - self.xml_bytes + 1)
|
|
69
74
|
self.xml_bytes += len(raw)
|
|
70
|
-
if self.xml_bytes > MAX_XML_BYTES
|
|
75
|
+
if self.xml_bytes > MAX_XML_BYTES:
|
|
76
|
+
raise DocumentError(f'{self.label} contains excessive XML or disallowed entities.')
|
|
77
|
+
return raw
|
|
78
|
+
|
|
79
|
+
def parse(self, name):
|
|
80
|
+
raw = self.read(name)
|
|
81
|
+
if b'<!DOCTYPE' in raw.upper() or b'<!ENTITY' in raw.upper():
|
|
71
82
|
raise DocumentError(f'{self.label} contains excessive XML or disallowed entities.')
|
|
72
83
|
builder = _DocumentTreeBuilder()
|
|
73
84
|
builder.label = self.label
|
|
@@ -451,6 +462,182 @@ def extract_xlsx(data):
|
|
|
451
462
|
return ('\n'.join(lines) + '\n').encode('utf-8')
|
|
452
463
|
|
|
453
464
|
|
|
465
|
+
_SKIPPED_HTML = frozenset(('script', 'style', 'noscript', 'template', 'svg', 'head'))
|
|
466
|
+
_BLOCK_HTML = frozenset(('p', 'div', 'br', 'section', 'article', 'header', 'footer', 'main', 'aside', 'nav', 'ul', 'ol',
|
|
467
|
+
'blockquote', 'pre', 'hr', 'figure', 'figcaption', 'dl', 'dt', 'dd', 'table', 'body', 'form',
|
|
468
|
+
'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'))
|
|
469
|
+
_CHARSET = re.compile(rb'charset\s*=\s*["\']?([\w.:-]+)', re.I)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _decode_text(data, label, declared=False):
|
|
473
|
+
"""Text of a file's bytes: UTF-8 (a BOM is dropped), UTF-16 with a BOM, or, for markup, the charset it declares in
|
|
474
|
+
its first 2 KiB. A file that is none of those is refused, never decoded into wrong characters."""
|
|
475
|
+
if data.startswith((b'\xff\xfe', b'\xfe\xff')):
|
|
476
|
+
return data.decode('utf-16')
|
|
477
|
+
try:
|
|
478
|
+
return data.decode('utf-8-sig')
|
|
479
|
+
except UnicodeDecodeError:
|
|
480
|
+
found = _CHARSET.search(data[:2048]) if declared else None
|
|
481
|
+
if found:
|
|
482
|
+
try:
|
|
483
|
+
return data.decode(found.group(1).decode('ascii'))
|
|
484
|
+
except (LookupError, UnicodeDecodeError):
|
|
485
|
+
pass
|
|
486
|
+
raise DocumentError(f'{label} is not UTF-8 and declares no encoding the reader can use.')
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
class _HtmlText(HTMLParser):
|
|
490
|
+
"""Markdown blocks of an HTML document: headings as `#`, list items as `- `, table rows as `a | b`, the other text
|
|
491
|
+
as lines. No tree is built and nothing is fetched or executed; script, style and the head except its title are
|
|
492
|
+
skipped. Output goes through a _Lines budget, so a page over MAX_TEXT_BYTES of text is refused."""
|
|
493
|
+
|
|
494
|
+
def __init__(self, label):
|
|
495
|
+
super().__init__(convert_charrefs=True)
|
|
496
|
+
self.blocks = _Lines(label)
|
|
497
|
+
self.pieces, self.prefix, self.skipped = [], '', 0
|
|
498
|
+
self.row, self.cell, self.title, self.in_title = [], 0, [], False
|
|
499
|
+
|
|
500
|
+
def _flush(self):
|
|
501
|
+
text = ' '.join(''.join(self.pieces).split())
|
|
502
|
+
self.pieces = []
|
|
503
|
+
if text:
|
|
504
|
+
self.blocks.add(self.prefix + text)
|
|
505
|
+
self.prefix = ''
|
|
506
|
+
|
|
507
|
+
def handle_starttag(self, tag, attrs):
|
|
508
|
+
if tag == 'body':
|
|
509
|
+
self.skipped = 0
|
|
510
|
+
if tag == 'title':
|
|
511
|
+
self.in_title = not self.title
|
|
512
|
+
elif tag in _SKIPPED_HTML:
|
|
513
|
+
self.skipped += 1
|
|
514
|
+
elif self.skipped:
|
|
515
|
+
return
|
|
516
|
+
elif tag in ('td', 'th'):
|
|
517
|
+
self.cell += 1
|
|
518
|
+
elif tag == 'tr':
|
|
519
|
+
self._flush()
|
|
520
|
+
elif tag in _BLOCK_HTML and not self.cell:
|
|
521
|
+
self._flush()
|
|
522
|
+
if tag[0] == 'h' and len(tag) == 2:
|
|
523
|
+
self.prefix = '#' * int(tag[1]) + ' '
|
|
524
|
+
elif tag == 'li':
|
|
525
|
+
self.prefix = '- '
|
|
526
|
+
|
|
527
|
+
def handle_endtag(self, tag):
|
|
528
|
+
if tag == 'title':
|
|
529
|
+
self.in_title = False
|
|
530
|
+
elif tag in _SKIPPED_HTML:
|
|
531
|
+
self.skipped = max(0, self.skipped - 1)
|
|
532
|
+
elif self.skipped:
|
|
533
|
+
return
|
|
534
|
+
elif tag in ('td', 'th') and self.cell:
|
|
535
|
+
self.cell -= 1
|
|
536
|
+
self.row.append(' '.join(''.join(self.pieces).split()))
|
|
537
|
+
self.pieces = []
|
|
538
|
+
elif tag == 'tr':
|
|
539
|
+
if any(self.row):
|
|
540
|
+
self.blocks.add(' | '.join(self.row))
|
|
541
|
+
self.row = []
|
|
542
|
+
elif tag in _BLOCK_HTML and not self.cell:
|
|
543
|
+
self._flush()
|
|
544
|
+
|
|
545
|
+
def handle_data(self, data):
|
|
546
|
+
if self.in_title:
|
|
547
|
+
self.title.append(data)
|
|
548
|
+
elif not self.skipped:
|
|
549
|
+
self.pieces.append(data)
|
|
550
|
+
|
|
551
|
+
def result(self):
|
|
552
|
+
self.close()
|
|
553
|
+
self._flush()
|
|
554
|
+
title = ' '.join(''.join(self.title).split())
|
|
555
|
+
return ([f'# {title}'] if title else []) + list(self.blocks)
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def _html_blocks(data, label):
|
|
559
|
+
parser = _HtmlText(label)
|
|
560
|
+
parser.feed(_decode_text(data, label, declared=True))
|
|
561
|
+
return parser.result()
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def html_blocks(data):
|
|
565
|
+
"""Markdown blocks of an .html or .htm file in document order, for reading: the title as `#`, headings as `#`
|
|
566
|
+
levels, list items as `- `, table rows as `a | b`, other text as lines. Scripts, styles and the rest of the head
|
|
567
|
+
are not text; a link is its text.
|
|
568
|
+
|
|
569
|
+
Parameters: data (the file bytes). Returns: a list of blocks. Raises: DocumentError with a readable reason for an
|
|
570
|
+
encoding the reader cannot use, a page over MAX_TEXT_BYTES of text, or one without text."""
|
|
571
|
+
blocks = _html_blocks(data, 'HTML')
|
|
572
|
+
if not blocks:
|
|
573
|
+
raise DocumentError('HTML has no text.')
|
|
574
|
+
return blocks
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def csv_rows(data):
|
|
578
|
+
"""The rows of a .csv file, for reading: one tab-joined line per non-empty row (the delimiter is `,`, `;`, tab or
|
|
579
|
+
`|`, found from the first 4 KiB), a line break inside a cell turned into a space.
|
|
580
|
+
|
|
581
|
+
Parameters: data (the file bytes). Returns: a list of lines. Raises: DocumentError with a readable reason for an
|
|
582
|
+
encoding the reader cannot use, malformed quoting, a file over XLSX_MAX_ROWS rows or MAX_TEXT_BYTES of text, or one
|
|
583
|
+
without text."""
|
|
584
|
+
text = _decode_text(data, 'CSV')
|
|
585
|
+
try:
|
|
586
|
+
dialect = csv.Sniffer().sniff(text[:4096], delimiters=',;\t|')
|
|
587
|
+
except csv.Error:
|
|
588
|
+
dialect = csv.excel
|
|
589
|
+
rows = _Lines('CSV')
|
|
590
|
+
try:
|
|
591
|
+
for record in csv.reader(io.StringIO(text, newline=''), dialect):
|
|
592
|
+
values = [' '.join(cell.split()) for cell in record]
|
|
593
|
+
if any(values):
|
|
594
|
+
if len(rows) >= XLSX_MAX_ROWS:
|
|
595
|
+
raise DocumentError(f'CSV over {XLSX_MAX_ROWS} rows.')
|
|
596
|
+
rows.add('\t'.join(values))
|
|
597
|
+
except csv.Error as exc:
|
|
598
|
+
raise DocumentError(f'Could not read this CSV: {exc}.') from None
|
|
599
|
+
if not rows:
|
|
600
|
+
raise DocumentError('CSV has no text.')
|
|
601
|
+
return list(rows)
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def _epub_chapter_paths(package):
|
|
605
|
+
"""Zip names of the XHTML parts of an EPUB in reading order: the package document the container names, its spine
|
|
606
|
+
resolved through the manifest. A part outside the archive or a path leaving it is refused."""
|
|
607
|
+
container = package.parse('META-INF/container.xml')
|
|
608
|
+
rootfile = next((node.get('full-path') for node in container.iter() if _local(node.tag) == 'rootfile'), None)
|
|
609
|
+
opf = posixpath.normpath((rootfile or '').lstrip('/'))
|
|
610
|
+
if not rootfile or opf.startswith('..') or opf not in package.entries:
|
|
611
|
+
raise DocumentError('Invalid EPUB or excessive structure.')
|
|
612
|
+
package_root = package.parse(opf)
|
|
613
|
+
manifest = {node.get('id'): node for node in package_root.iter() if _local(node.tag) == 'item'}
|
|
614
|
+
base = posixpath.dirname(opf)
|
|
615
|
+
paths = []
|
|
616
|
+
for itemref in (node for node in package_root.iter() if _local(node.tag) == 'itemref'):
|
|
617
|
+
item = manifest.get(itemref.get('idref'))
|
|
618
|
+
if item is None or item.get('media-type') not in ('application/xhtml+xml', 'text/html'):
|
|
619
|
+
continue
|
|
620
|
+
part = posixpath.normpath(posixpath.join(base, unquote(item.get('href', '')).split('#')[0]))
|
|
621
|
+
if part.startswith('..') or part not in package.entries:
|
|
622
|
+
raise DocumentError('Invalid EPUB or excessive structure.')
|
|
623
|
+
paths.append(part)
|
|
624
|
+
return paths
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def epub_chapters(data):
|
|
628
|
+
"""The chapters of an .epub in reading order, for reading. Each XHTML part of the spine is read as HTML (see
|
|
629
|
+
html_blocks); the package is opened with the same guards as the Office formats.
|
|
630
|
+
|
|
631
|
+
Parameters: data (the file bytes). Returns: a list of block lists, one per chapter (a chapter without text is an
|
|
632
|
+
empty list). Raises: DocumentError with a readable reason for a corrupt, encrypted, oversized or hostile package, an
|
|
633
|
+
encoding the reader cannot use, or a book without any text."""
|
|
634
|
+
with _open_package(data, 'EPUB', 'META-INF/container.xml') as package:
|
|
635
|
+
chapters = [_html_blocks(package.read(path), 'EPUB') for path in _epub_chapter_paths(package)]
|
|
636
|
+
if not any(chapters):
|
|
637
|
+
raise DocumentError('EPUB has no text.')
|
|
638
|
+
return chapters
|
|
639
|
+
|
|
640
|
+
|
|
454
641
|
PDF_SCRIPT = """
|
|
455
642
|
import io, json, sys
|
|
456
643
|
import pypdf
|
|
@@ -524,14 +711,15 @@ def extract_pdf(data):
|
|
|
524
711
|
return (text + "\n").encode("utf-8", "replace")
|
|
525
712
|
|
|
526
713
|
|
|
527
|
-
_EXTENSIONS = ('.docx', '.pptx', '.xlsx', '.pdf')
|
|
714
|
+
_EXTENSIONS = ('.docx', '.pptx', '.xlsx', '.xlsm', '.pdf')
|
|
528
715
|
|
|
529
716
|
|
|
530
717
|
def is_extracted(path):
|
|
531
718
|
"""Whether the file type goes through a text extractor before indexing.
|
|
532
719
|
|
|
533
|
-
Parameters: path (any file name or path). Returns: True for `.docx`, `.pptx`, `.xlsx` and `.pdf`,
|
|
534
|
-
|
|
720
|
+
Parameters: path (any file name or path). Returns: True for `.docx`, `.pptx`, `.xlsx`, `.xlsm` and `.pdf`,
|
|
721
|
+
case-insensitive; an `.xlsm` is read as an `.xlsx` (its XML parts only, no macro runs); the binary and other macro
|
|
722
|
+
formats (`.xlsb`, `.xls`, `.ppt`, `.pptm`) are not extracted."""
|
|
535
723
|
return str(path).lower().endswith(_EXTENSIONS)
|
|
536
724
|
|
|
537
725
|
|
|
@@ -552,7 +740,7 @@ def extract(path, data):
|
|
|
552
740
|
return extract_pdf(data)
|
|
553
741
|
if name.endswith('.pptx'):
|
|
554
742
|
return extract_pptx(data)
|
|
555
|
-
if name.endswith('.xlsx'):
|
|
743
|
+
if name.endswith(('.xlsx', '.xlsm')):
|
|
556
744
|
return extract_xlsx(data)
|
|
557
745
|
return extract_docx(data)
|
|
558
746
|
|
package/dossier.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"""Dossier mode of smart_search: a narrow keyword query over the notes a project keeps about its own intent
|
|
2
|
-
(docs/research, the backlog, CLAUDE.md and AGENTS.md
|
|
3
|
-
budget. It reads straight from disk, uses no index and no model, and
|
|
4
|
-
stops at the deadline and says so in `degraded`/`reasons`.
|
|
2
|
+
(docs/research, the backlog, CLAUDE.md and AGENTS.md; with `history`, also the messages of its commits), answered with
|
|
3
|
+
ids and one short excerpt per hit, inside a time budget. It reads straight from disk, uses no index and no model, and
|
|
4
|
+
never raises on a slow or unreadable file: it stops at the deadline and says so in `degraded`/`reasons`. The commit
|
|
5
|
+
messages come from `doc_insights.history`, whose first read of a repository runs one `git log` that no deadline can
|
|
6
|
+
cut, so `history` is opt-in and a read that outlasts the budget is reported in `reasons`."""
|
|
5
7
|
import json
|
|
6
8
|
import os
|
|
7
9
|
import re
|
|
@@ -89,11 +91,20 @@ def _score(unit_lines, terms):
|
|
|
89
91
|
return sum((term in low) + (term in title) for term in terms)
|
|
90
92
|
|
|
91
93
|
|
|
92
|
-
def
|
|
94
|
+
def _commit_units(root):
|
|
95
|
+
"""`[(hash, subject, [lines])]` of the commits of `root`, newest first (a subject can carry the whole reason), and
|
|
96
|
+
the reason no history could be read (None when it was)."""
|
|
97
|
+
import doc_insights
|
|
98
|
+
commits, note = doc_insights.history(root)
|
|
99
|
+
return [(commit["hash"], commit["subject"], [commit["subject"], *commit["body"].splitlines()])
|
|
100
|
+
for commit in commits], note
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def query(root, text, timeout_ms=DEFAULT_TIMEOUT_MS, max_hits=DEFAULT_MAX_HITS, history=False):
|
|
93
104
|
"""The dossier answer for `text` over the notes of `root`: `{"mode", "degraded", "reasons", "elapsed_ms",
|
|
94
105
|
"scanned", "hits": [{"id", "source", "title", "snippet"}]}`, best hit first. `id` is `path:line` of the unit's
|
|
95
|
-
first line
|
|
96
|
-
was found until then."""
|
|
106
|
+
first line, or `git:<hash>` for a commit message (`history` true; source `git`). `degraded` is true when the
|
|
107
|
+
deadline cut the scan or a file could not be read; `hits` then holds what was found until then."""
|
|
97
108
|
started = time.monotonic()
|
|
98
109
|
deadline = started + timeout_ms / 1000
|
|
99
110
|
terms = focus_terms(text)
|
|
@@ -119,8 +130,21 @@ def query(root, text, timeout_ms=DEFAULT_TIMEOUT_MS, max_hits=DEFAULT_MAX_HITS):
|
|
|
119
130
|
scored.append((score, rel, kind, number, unit))
|
|
120
131
|
if unreadable:
|
|
121
132
|
reasons.append("not read: " + "; ".join(unreadable[:5]))
|
|
133
|
+
if history and time.monotonic() <= deadline:
|
|
134
|
+
commits, note = _commit_units(root)
|
|
135
|
+
scanned += 1
|
|
136
|
+
if note:
|
|
137
|
+
reasons.append(f"git history: {note}")
|
|
138
|
+
if time.monotonic() > deadline:
|
|
139
|
+
reasons.append(f"git history read took longer than {timeout_ms} ms")
|
|
140
|
+
for short, _subject, unit in commits:
|
|
141
|
+
score = _score(unit, terms)
|
|
142
|
+
if score:
|
|
143
|
+
scored.append((score, f"git:{short}", "git", None, unit))
|
|
144
|
+
elif history:
|
|
145
|
+
reasons.append(f"git history not read: timeout after {timeout_ms} ms")
|
|
122
146
|
scored.sort(key=lambda item: -item[0])
|
|
123
|
-
hits = [{"id": f"{rel}:{number}", "source": kind, "title": _clean(unit[0])[:TITLE_CHARS],
|
|
147
|
+
hits = [{"id": rel if number is None else f"{rel}:{number}", "source": kind, "title": _clean(unit[0])[:TITLE_CHARS],
|
|
124
148
|
"snippet": _snippet(unit, terms)} for _score_value, rel, kind, number, unit in scored[:max_hits]]
|
|
125
149
|
return {"mode": "dossier", "degraded": bool(reasons), "reasons": reasons,
|
|
126
150
|
"elapsed_ms": round((time.monotonic() - started) * 1000), "scanned": scanned, "hits": hits}
|
|
@@ -138,7 +162,10 @@ def answer(arguments):
|
|
|
138
162
|
raise ValueError("smart_search mode=dossier requires 'query_identifiers' (or 'doc_query') with at least one word.")
|
|
139
163
|
timeout_ms = _bounded(arguments, "timeout_ms", DEFAULT_TIMEOUT_MS, 1, MAX_TIMEOUT_MS)
|
|
140
164
|
max_hits = _bounded(arguments, "max_hits", DEFAULT_MAX_HITS, 1, MAX_HITS)
|
|
141
|
-
|
|
165
|
+
history = arguments.get("history", False)
|
|
166
|
+
if not isinstance(history, bool):
|
|
167
|
+
raise ValueError("history must be true or false.")
|
|
168
|
+
return json.dumps(query(root, text, timeout_ms, max_hits, history), ensure_ascii=False, separators=(",", ":"))
|
|
142
169
|
|
|
143
170
|
|
|
144
171
|
def _bounded(arguments, name, default, low, high):
|
package/duplicates.py
CHANGED
|
@@ -184,7 +184,7 @@ def _functions(root, include_tests):
|
|
|
184
184
|
if inventory.get("message"):
|
|
185
185
|
notes.append(inventory["message"])
|
|
186
186
|
if graph.get("truncated"):
|
|
187
|
-
notes.append(
|
|
187
|
+
notes.append(code_graph.left_out_sentence(graph))
|
|
188
188
|
if graph.get("retryable"):
|
|
189
189
|
notes.append("Partial analysis: " + "; ".join(d.get("reason", "") for d in graph.get("diagnostics", [])[:2]))
|
|
190
190
|
return functions, notes, view
|
package/embedding_cache.py
CHANGED
|
@@ -12,13 +12,35 @@ import time
|
|
|
12
12
|
import project_identity
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
def _open(root, directory):
|
|
16
|
+
os.makedirs(directory, exist_ok=True)
|
|
17
|
+
db = sqlite3.connect(os.path.join(directory, project_identity.project_id(root) + '.vectors.sqlite3'), timeout=10)
|
|
18
|
+
db.execute('PRAGMA secure_delete=ON')
|
|
19
|
+
db.execute('CREATE TABLE IF NOT EXISTS vectors(key TEXT PRIMARY KEY, vector BLOB NOT NULL, used_at REAL NOT NULL, '
|
|
20
|
+
'model TEXT)')
|
|
21
|
+
if 'model' not in {row[1] for row in db.execute('PRAGMA table_info(vectors)')}:
|
|
22
|
+
db.execute('ALTER TABLE vectors ADD COLUMN model TEXT')
|
|
23
|
+
return db
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def drop_other_models(root, directory, model):
|
|
27
|
+
"""Deletes the cached vectors of every model but `model` and gives their space back to the disk. Rows written
|
|
28
|
+
before the cache recorded the model are deleted too, unless a build of `model` reused them since."""
|
|
29
|
+
db = _open(root, directory)
|
|
30
|
+
try:
|
|
31
|
+
dropped = db.execute('DELETE FROM vectors WHERE model IS NOT ?', (model,)).rowcount
|
|
32
|
+
db.commit()
|
|
33
|
+
if dropped:
|
|
34
|
+
db.execute('VACUUM')
|
|
35
|
+
return dropped
|
|
36
|
+
finally:
|
|
37
|
+
db.close()
|
|
38
|
+
|
|
39
|
+
|
|
15
40
|
class VectorCache:
|
|
16
41
|
def __init__(self, root, directory, contract):
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
self.db = sqlite3.connect(path, timeout=10)
|
|
20
|
-
self.db.execute('PRAGMA secure_delete=ON')
|
|
21
|
-
self.db.execute('CREATE TABLE IF NOT EXISTS vectors(key TEXT PRIMARY KEY, vector BLOB NOT NULL, used_at REAL NOT NULL)')
|
|
42
|
+
self.db = _open(root, directory)
|
|
43
|
+
self.model = contract.get('model')
|
|
22
44
|
self.contract = json.dumps(contract, sort_keys=True, ensure_ascii=False)
|
|
23
45
|
self.dimensions = contract.get('dimensions')
|
|
24
46
|
self.hits = self.misses = self.characters = 0
|
|
@@ -32,18 +54,20 @@ class VectorCache:
|
|
|
32
54
|
packed = array.array('f', vector)
|
|
33
55
|
if not all(math.isfinite(value) for value in packed):
|
|
34
56
|
raise ValueError('Embedding does not fit in float32; cache not written.')
|
|
35
|
-
self.db.execute('INSERT OR REPLACE INTO vectors VALUES(
|
|
36
|
-
(self.key(text), packed.tobytes(), time.time()))
|
|
57
|
+
self.db.execute('INSERT OR REPLACE INTO vectors VALUES(?,?,?,?)',
|
|
58
|
+
(self.key(text), packed.tobytes(), time.time(), self.model))
|
|
37
59
|
|
|
38
60
|
def embed(self, texts, embed_batch, validate, use_cache=True, split=None, workers=1):
|
|
39
61
|
"""Cache lookups and writes stay on the calling thread (sqlite); only embed_batch runs in workers."""
|
|
40
62
|
output = [None] * len(texts)
|
|
41
63
|
missing = {}
|
|
42
64
|
for i, text in enumerate(texts):
|
|
43
|
-
row = self.db.execute('SELECT vector FROM vectors WHERE key=?', (self.key(text),)).fetchone() if use_cache else None
|
|
65
|
+
row = self.db.execute('SELECT vector, model FROM vectors WHERE key=?', (self.key(text),)).fetchone() if use_cache else None
|
|
44
66
|
if row and self.dimensions:
|
|
45
67
|
vector = array.array('f'); vector.frombytes(row[0])
|
|
46
68
|
validate(vector, self.dimensions)
|
|
69
|
+
if row[1] is None:
|
|
70
|
+
self.db.execute('UPDATE vectors SET model=? WHERE key=?', (self.model, self.key(text)))
|
|
47
71
|
output[i] = list(vector)
|
|
48
72
|
self.hits += 1
|
|
49
73
|
else:
|
package/freshness.py
CHANGED
|
@@ -74,7 +74,8 @@ def npm_registry_evidence(package):
|
|
|
74
74
|
|
|
75
75
|
NPM_NAME_RE = re.compile(r"(?:@[a-z0-9._-]+/)?[a-z0-9._-]+")
|
|
76
76
|
NAME_RES = {"npm": NPM_NAME_RE, "PyPI": re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*"),
|
|
77
|
-
"Go": re.compile(r"[A-Za-z0-9][A-Za-z0-9._~/-]*")
|
|
77
|
+
"Go": re.compile(r"[A-Za-z0-9][A-Za-z0-9._~/-]*"), "crates.io": re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*"),
|
|
78
|
+
"Maven": re.compile(r"[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+")}
|
|
78
79
|
|
|
79
80
|
|
|
80
81
|
def _osv_query(ecosystem, package, package_version):
|
|
@@ -120,7 +121,7 @@ def npm_advisories(package, package_version):
|
|
|
120
121
|
def advisories_batch(ecosystem, packages, timeout):
|
|
121
122
|
"""Advisory IDs for each (package, version) of `packages` from one OSV `querybatch` call.
|
|
122
123
|
|
|
123
|
-
Parameters: ecosystem (str) "npm", "PyPI" or "
|
|
124
|
+
Parameters: ecosystem (str) "npm", "PyPI", "Go", "crates.io" or "Maven"; packages (list of (name, version)); timeout (float) seconds for
|
|
124
125
|
the request.
|
|
125
126
|
Returns: list in the same order, each the sorted ID list or None when that package's answer is incomplete
|
|
126
127
|
(next_page_token) or malformed, so it is unverified and not clean.
|