@allansantos-dev/smart-tool 0.9.79 → 0.9.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/CHANGELOG.md +132 -0
  2. package/README.md +82 -18
  3. package/browser_control.py +10 -3
  4. package/browser_service_client.py +3 -32
  5. package/chromium_flags.py +30 -0
  6. package/client_hooks.py +10 -8
  7. package/code_health.py +182 -0
  8. package/code_impact.py +14 -2
  9. package/config.py +76 -8
  10. package/credential_check.py +38 -0
  11. package/dependency_notice.py +262 -14
  12. package/document_converter.py +221 -0
  13. package/document_read.py +100 -25
  14. package/document_text.py +194 -6
  15. package/dossier.py +71 -9
  16. package/embedding_cache.py +32 -8
  17. package/freshness.py +3 -2
  18. package/hook_decision.py +289 -18
  19. package/hooks/pretool_router.py +5 -4
  20. package/indexer.py +92 -3
  21. package/install.py +66 -27
  22. package/install_runtime.py +24 -10
  23. package/layer_check.py +57 -0
  24. package/link_check.py +69 -0
  25. package/local_embedder.py +115 -58
  26. package/locales/pt-BR.json +122 -18
  27. package/low_priority.py +41 -0
  28. package/model_client.py +1 -1
  29. package/model_health.py +24 -6
  30. package/notice_action.py +9 -0
  31. package/package.json +1 -1
  32. package/pr_comment.py +94 -0
  33. package/pr_impact.py +292 -8
  34. package/process_limits.py +108 -0
  35. package/project_admission.py +17 -15
  36. package/project_monitor.py +33 -7
  37. package/project_scheduler.py +12 -4
  38. package/removed_callers.py +146 -0
  39. package/scout.py +6 -2
  40. package/setup_audit.py +34 -1
  41. package/setup_ui.py +241 -41
  42. package/smart_tool_daemon.py +403 -91
  43. package/tray.py +29 -9
  44. package/update_check.py +18 -0
  45. package/update_splash.py +2 -2
  46. package/usage_meter.py +78 -0
  47. package/version.py +1 -1
  48. package/web_adapters/browser/crawl4ai_search.py +7 -3
  49. package/web_document_graph.py +12 -1
  50. package/web_fetch.py +24 -0
  51. package/web_search_health.py +31 -0
package/document_read.py CHANGED
@@ -1,9 +1,10 @@
1
- """read_document: PDF, Word, PowerPoint and Excel files read as compact text, one page, slide or sheet range at a time.
1
+ """read_document: PDF, Word, PowerPoint, Excel, HTML, CSV and EPUB files read as compact text, one page, slide, sheet,
2
+ part or chapter range at a time.
2
3
 
3
4
  The agent gets the file facts first (type, size, pages, slides or sheets, estimated tokens) and then only the range it
4
5
  asks for, so a 200-page PDF is never loaded whole. Extraction is the one of document_text (pypdf in an isolated
5
- subprocess for PDF; zipfile and ElementTree with the shared guards for docx, pptx and xlsx); this module adds the unit
6
- split, the markdown shape, the compaction of PDF line wrapping and the range arithmetic."""
6
+ subprocess for PDF; zipfile and ElementTree with the shared guards for docx, pptx, xlsx and epub; the standard HTML
7
+ and CSV parsers for html and csv); this module adds the unit split, the markdown shape, the compaction of PDF line wrapping and the range arithmetic."""
7
8
  import collections
8
9
  import math
9
10
  import os
@@ -11,10 +12,14 @@ import re
11
12
  import threading
12
13
  import unicodedata
13
14
 
15
+ import document_converter
14
16
  import document_text
15
17
 
16
- KINDS = {'.pdf': 'pdf', '.docx': 'docx', '.pptx': 'pptx', '.xlsx': 'xlsx'}
17
- UNIT_NAMES = {'pdf': 'page', 'docx': 'part', 'pptx': 'slide', 'xlsx': 'sheet'}
18
+ KINDS = {'.pdf': 'pdf', '.docx': 'docx', '.pptx': 'pptx', '.xlsx': 'xlsx', '.xlsm': 'xlsx', '.html': 'html', '.htm': 'html',
19
+ '.csv': 'csv', '.epub': 'epub'}
20
+ UNIT_NAMES = {'pdf': 'page', 'docx': 'part', 'pptx': 'slide', 'xlsx': 'sheet', 'html': 'part', 'csv': 'sheet',
21
+ 'epub': 'chapter', 'converted': 'part'}
22
+ LABELLED_KINDS = ('pdf', 'docx', 'html', 'epub', 'converted')
18
23
  DEFAULT_MAX_CHARS = 20_000
19
24
  MAX_CHARS_LIMIT = 100_000
20
25
  DOCX_PART_CHARS = 6_000
@@ -26,6 +31,14 @@ _cache = collections.OrderedDict()
26
31
  _cache_lock = threading.Lock()
27
32
 
28
33
 
34
+ class NoTextLayer(document_text.DocumentError):
35
+ """A PDF whose pages carry no text: `pages` is its page count."""
36
+
37
+ def __init__(self, pages):
38
+ super().__init__(f'PDF of {pages} pages has no text layer (scanned pages are not read; there is no OCR).')
39
+ self.pages = pages
40
+
41
+
29
42
  class Unit:
30
43
  """One readable unit of a document: a PDF page, a docx part, a slide or a sheet."""
31
44
 
@@ -74,13 +87,12 @@ def compact_pdf_text(text):
74
87
 
75
88
 
76
89
  def _pdf_units(data):
77
- """Units and facts of a PDF, one unit per page; raises DocumentError for a password, a corrupt file or a file with no
78
- text layer at all (scanned pages are not read, there is no OCR)."""
90
+ """Units and facts of a PDF, one unit per page; raises DocumentError for a password or a corrupt file, and NoTextLayer
91
+ for a file with no text layer at all."""
79
92
  pages, count = document_text.pdf_pages(data, document_text.PDF_TIMEOUT_S, document_text.PDF_MAX_PAGES,
80
93
  document_text.PDF_MAX_CHARS)
81
94
  if not any(pages):
82
- raise document_text.DocumentError(
83
- f'PDF of {count} pages has no text layer (scanned pages are not read; there is no OCR).')
95
+ raise NoTextLayer(count)
84
96
  units = [Unit(number, compact_pdf_text(page) or '(no text on this page)') for number, page in enumerate(pages, 1)]
85
97
  facts = {'pages': count, 'pages without text': sum(1 for page in pages if not page)}
86
98
  if count > len(pages):
@@ -89,21 +101,47 @@ def _pdf_units(data):
89
101
  return units, facts
90
102
 
91
103
 
92
- def _docx_units(data):
93
- """Units and facts of a docx: the markdown blocks grouped in parts of about DOCX_PART_CHARS characters, a block never
94
- split (a docx has no fixed pages, they depend on the renderer)."""
104
+ def _parts(blocks):
105
+ """The markdown blocks grouped in parts of about DOCX_PART_CHARS characters, a block never split."""
95
106
  parts, current, size = [], [], 0
96
- for block in document_text.docx_blocks(data):
107
+ for block in blocks:
97
108
  if current and size + len(block) > DOCX_PART_CHARS:
98
109
  parts.append('\n'.join(current))
99
110
  current, size = [], 0
100
111
  current.append(block)
101
112
  size += len(block) + 1
102
113
  parts.append('\n'.join(current))
114
+ return parts
115
+
116
+
117
+ def _docx_units(data):
118
+ """Units and facts of a docx: the markdown blocks grouped in parts (a docx has no fixed pages, they depend on the
119
+ renderer)."""
120
+ parts = _parts(document_text.docx_blocks(data))
103
121
  return [Unit(number, text) for number, text in enumerate(parts, 1)], {
104
122
  'parts': len(parts), 'note': f'a docx has no fixed pages; a part is about {DOCX_PART_CHARS:,} characters'}
105
123
 
106
124
 
125
+ def _html_units(data):
126
+ """Units and facts of an html file: the markdown blocks grouped in parts, as a docx (an html file has no pages)."""
127
+ parts = _parts(document_text.html_blocks(data))
128
+ return [Unit(number, text) for number, text in enumerate(parts, 1)], {
129
+ 'parts': len(parts), 'note': f'a part is about {DOCX_PART_CHARS:,} characters of the page text'}
130
+
131
+
132
+ def _csv_units(data):
133
+ """Units and facts of a csv file: one unit, the sheet `csv`, one tab-separated line per row."""
134
+ rows = document_text.csv_rows(data)
135
+ return [Unit(1, '\n'.join(['[sheet csv]'] + rows), 'csv')], {'rows': len(rows)}
136
+
137
+
138
+ def _epub_units(data):
139
+ """Units and facts of an epub: one unit per chapter of the reading order."""
140
+ chapters = document_text.epub_chapters(data)
141
+ units = [Unit(number, '\n'.join(blocks) or '(no text)') for number, blocks in enumerate(chapters, 1)]
142
+ return units, {'chapters without text': sum(1 for blocks in chapters if not blocks)}
143
+
144
+
107
145
  def _pptx_units(data):
108
146
  """Units and facts of a pptx: one unit per slide under `[slide N]`, the speaker notes under `[notes N]`."""
109
147
  units = []
@@ -121,14 +159,51 @@ def _xlsx_units(data):
121
159
  return units, {'sheet names': ', '.join(f'{u.number}={u.name}' for u in units)}
122
160
 
123
161
 
124
- _BUILDERS = {'pdf': _pdf_units, 'docx': _docx_units, 'pptx': _pptx_units, 'xlsx': _xlsx_units}
162
+ def _converted_pdf_units(path, count):
163
+ """Units and facts of a PDF with no text layer, read by the external converter as page images."""
164
+ result = document_converter.convert(path, expected_pages=count, force_ocr=True)
165
+ units = [Unit(number, page.strip() or '(no text on this page)') for number, page in enumerate(result.pages, 1)]
166
+ facts = {'pages without text': sum(1 for page in result.pages if not page.strip()),
167
+ 'read by': f'{result.engine} (OCR of the page images)'}
168
+ if len(units) != count:
169
+ facts['note'] = f'the converter returned {len(units)} pages for a file of {count}'
170
+ return units, facts
171
+
172
+
173
+ def _converted_units(path):
174
+ """Units and facts of a file the native reader has no type for, read by the external converter: its text grouped in
175
+ parts as a docx."""
176
+ result = document_converter.convert(path)
177
+ text = '\n\n'.join(page.strip() for page in result.pages if page.strip())
178
+ parts = _parts(text.split('\n'))
179
+ return [Unit(number, part) for number, part in enumerate(parts, 1)], {'parts': len(parts),
180
+ 'read by': result.engine}
181
+
182
+
183
+ def _build(path, kind):
184
+ """Units and facts of the file at `path`: its native builder, or, for a kind the native reader has none for and for a
185
+ PDF without a text layer, the external converter when the setting turns it on."""
186
+ if kind == 'converted':
187
+ return _converted_units(path)
188
+ with open(path, 'rb') as stream:
189
+ data = stream.read()
190
+ try:
191
+ return _BUILDERS[kind](data)
192
+ except NoTextLayer as exc:
193
+ if not document_converter.enabled():
194
+ raise
195
+ return _converted_pdf_units(path, exc.pages)
196
+
197
+
198
+ _BUILDERS = {'pdf': _pdf_units, 'docx': _docx_units, 'pptx': _pptx_units, 'xlsx': _xlsx_units, 'html': _html_units,
199
+ 'csv': _csv_units, 'epub': _epub_units}
125
200
 
126
201
 
127
202
  def load(path):
128
203
  """The document at `path`, split into units. Parsed documents are kept for a few calls (keyed by path, size and
129
204
  modification time), so reading page after page parses the file once.
130
205
 
131
- Parameters: path (absolute path of a .pdf, .docx, .pptx or .xlsx file). Returns: a Document. Raises: ValueError for a
206
+ Parameters: path (absolute path of a .pdf, .docx, .pptx, .xlsx, .html, .htm, .csv or .epub file). Returns: a Document. Raises: ValueError for a
132
207
  relative path, a missing file, an unsupported type or a file over the size limit of its type; DocumentError with a
133
208
  readable reason for a file the extractor refuses (corrupt, encrypted, hostile, scanned, without text)."""
134
209
  if not isinstance(path, str) or not path.strip():
@@ -136,27 +211,27 @@ def load(path):
136
211
  if not os.path.isabs(path):
137
212
  raise ValueError(f'read_document needs an absolute path, got {path!r}')
138
213
  extension = os.path.splitext(path)[1].lower()
139
- if extension not in KINDS:
214
+ converted = extension not in KINDS and extension in document_converter.EXTENSIONS and document_converter.enabled()
215
+ if extension not in KINDS and not converted:
140
216
  raise ValueError(f"read_document reads {', '.join(KINDS)} files; {extension or 'this file'} is not supported "
141
- '(use the normal Read for plain text; legacy .doc, .xls and .ppt and macro files are not read)')
217
+ '(use the normal Read for plain text; legacy .doc, .xls, .xlsb and .ppt and macro files other than .xlsm are not read)')
142
218
  try:
143
219
  info = os.stat(path)
144
220
  except OSError as exc:
145
221
  raise ValueError(f'cannot read {path}: {exc.strerror or type(exc).__name__}') from None
146
222
  if not os.path.isfile(path):
147
223
  raise ValueError(f'{path} is not a file')
148
- if info.st_size > document_text.size_limit(path):
149
- raise ValueError(f'{path} is {info.st_size / 1048576:.1f} MiB, over the '
150
- f'{document_text.size_limit(path) // 1048576} MiB limit of a {extension} file')
224
+ limit = document_text.PDF_MAX_BYTES if converted else document_text.size_limit(path)
225
+ if info.st_size > limit:
226
+ raise ValueError(f'{path} is {info.st_size / 1048576:.1f} MiB, over the {limit // 1048576} MiB limit of a '
227
+ f'{extension} file')
151
228
  key = (os.path.normcase(path), info.st_size, info.st_mtime_ns)
152
229
  with _cache_lock:
153
230
  if key in _cache:
154
231
  _cache.move_to_end(key)
155
232
  return _cache[key]
156
- with open(path, 'rb') as stream:
157
- data = stream.read()
158
- kind = KINDS[extension]
159
- units, facts = _BUILDERS[kind](data)
233
+ kind = 'converted' if converted else KINDS[extension]
234
+ units, facts = _build(path, kind)
160
235
  document = Document(path, kind, info.st_size, units, facts)
161
236
  with _cache_lock:
162
237
  _cache[key] = document
@@ -241,7 +316,7 @@ def read(path, range_spec=None, max_chars=DEFAULT_MAX_CHARS, info_only=False):
241
316
  body, shown, size, cut = [], [], 0, None
242
317
  for number in numbers:
243
318
  text = document.units[number - 1].text
244
- label = f'[{unit} {number}]\n' if document.kind in ('pdf', 'docx') else ''
319
+ label = f'[{unit} {number}]\n' if document.kind in LABELLED_KINDS else ''
245
320
  piece = label + text
246
321
  if shown and size + len(piece) > max_chars:
247
322
  break
package/document_text.py CHANGED
@@ -1,5 +1,6 @@
1
1
  """Extração local de texto DOCX, PPTX, XLSX e PDF, sem Office, execução de macros ou acesso externo."""
2
2
  import contextlib
3
+ import csv
3
4
  import io
4
5
  import json
5
6
  import posixpath
@@ -9,6 +10,8 @@ import sys
9
10
  import zipfile
10
11
  import xml.etree.ElementTree as ET
11
12
  import zlib
13
+ from html.parser import HTMLParser
14
+ from urllib.parse import unquote
12
15
 
13
16
  VERSION = 3
14
17
  PDF_MAX_BYTES = 20 * 1024 * 1024
@@ -58,7 +61,9 @@ class _Package:
58
61
  self.label = label
59
62
  self.xml_bytes = 0
60
63
 
61
- def parse(self, name):
64
+ def read(self, name):
65
+ """The bytes of one part, charged to the XML budget; raises DocumentError for an encrypted or oddly compressed
66
+ part and for one that would pass the budget."""
62
67
  info = self.entries[name]
63
68
  if info.flag_bits & 1 or info.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
64
69
  raise DocumentError(f'Encrypted {self.label} or unsupported compression.')
@@ -67,7 +72,13 @@ class _Package:
67
72
  with self.archive.open(info) as stream:
68
73
  raw = stream.read(MAX_XML_BYTES - self.xml_bytes + 1)
69
74
  self.xml_bytes += len(raw)
70
- if self.xml_bytes > MAX_XML_BYTES or b'<!DOCTYPE' in raw.upper() or b'<!ENTITY' in raw.upper():
75
+ if self.xml_bytes > MAX_XML_BYTES:
76
+ raise DocumentError(f'{self.label} contains excessive XML or disallowed entities.')
77
+ return raw
78
+
79
+ def parse(self, name):
80
+ raw = self.read(name)
81
+ if b'<!DOCTYPE' in raw.upper() or b'<!ENTITY' in raw.upper():
71
82
  raise DocumentError(f'{self.label} contains excessive XML or disallowed entities.')
72
83
  builder = _DocumentTreeBuilder()
73
84
  builder.label = self.label
@@ -451,6 +462,182 @@ def extract_xlsx(data):
451
462
  return ('\n'.join(lines) + '\n').encode('utf-8')
452
463
 
453
464
 
465
+ _SKIPPED_HTML = frozenset(('script', 'style', 'noscript', 'template', 'svg', 'head'))
466
+ _BLOCK_HTML = frozenset(('p', 'div', 'br', 'section', 'article', 'header', 'footer', 'main', 'aside', 'nav', 'ul', 'ol',
467
+ 'blockquote', 'pre', 'hr', 'figure', 'figcaption', 'dl', 'dt', 'dd', 'table', 'body', 'form',
468
+ 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'))
469
+ _CHARSET = re.compile(rb'charset\s*=\s*["\']?([\w.:-]+)', re.I)
470
+
471
+
472
+ def _decode_text(data, label, declared=False):
473
+ """Text of a file's bytes: UTF-8 (a BOM is dropped), UTF-16 with a BOM, or, for markup, the charset it declares in
474
+ its first 2 KiB. A file that is none of those is refused, never decoded into wrong characters."""
475
+ if data.startswith((b'\xff\xfe', b'\xfe\xff')):
476
+ return data.decode('utf-16')
477
+ try:
478
+ return data.decode('utf-8-sig')
479
+ except UnicodeDecodeError:
480
+ found = _CHARSET.search(data[:2048]) if declared else None
481
+ if found:
482
+ try:
483
+ return data.decode(found.group(1).decode('ascii'))
484
+ except (LookupError, UnicodeDecodeError):
485
+ pass
486
+ raise DocumentError(f'{label} is not UTF-8 and declares no encoding the reader can use.')
487
+
488
+
489
+ class _HtmlText(HTMLParser):
490
+ """Markdown blocks of an HTML document: headings as `#`, list items as `- `, table rows as `a | b`, the other text
491
+ as lines. No tree is built and nothing is fetched or executed; script, style and the head except its title are
492
+ skipped. Output goes through a _Lines budget, so a page over MAX_TEXT_BYTES of text is refused."""
493
+
494
+ def __init__(self, label):
495
+ super().__init__(convert_charrefs=True)
496
+ self.blocks = _Lines(label)
497
+ self.pieces, self.prefix, self.skipped = [], '', 0
498
+ self.row, self.cell, self.title, self.in_title = [], 0, [], False
499
+
500
+ def _flush(self):
501
+ text = ' '.join(''.join(self.pieces).split())
502
+ self.pieces = []
503
+ if text:
504
+ self.blocks.add(self.prefix + text)
505
+ self.prefix = ''
506
+
507
+ def handle_starttag(self, tag, attrs):
508
+ if tag == 'body':
509
+ self.skipped = 0
510
+ if tag == 'title':
511
+ self.in_title = not self.title
512
+ elif tag in _SKIPPED_HTML:
513
+ self.skipped += 1
514
+ elif self.skipped:
515
+ return
516
+ elif tag in ('td', 'th'):
517
+ self.cell += 1
518
+ elif tag == 'tr':
519
+ self._flush()
520
+ elif tag in _BLOCK_HTML and not self.cell:
521
+ self._flush()
522
+ if tag[0] == 'h' and len(tag) == 2:
523
+ self.prefix = '#' * int(tag[1]) + ' '
524
+ elif tag == 'li':
525
+ self.prefix = '- '
526
+
527
+ def handle_endtag(self, tag):
528
+ if tag == 'title':
529
+ self.in_title = False
530
+ elif tag in _SKIPPED_HTML:
531
+ self.skipped = max(0, self.skipped - 1)
532
+ elif self.skipped:
533
+ return
534
+ elif tag in ('td', 'th') and self.cell:
535
+ self.cell -= 1
536
+ self.row.append(' '.join(''.join(self.pieces).split()))
537
+ self.pieces = []
538
+ elif tag == 'tr':
539
+ if any(self.row):
540
+ self.blocks.add(' | '.join(self.row))
541
+ self.row = []
542
+ elif tag in _BLOCK_HTML and not self.cell:
543
+ self._flush()
544
+
545
+ def handle_data(self, data):
546
+ if self.in_title:
547
+ self.title.append(data)
548
+ elif not self.skipped:
549
+ self.pieces.append(data)
550
+
551
+ def result(self):
552
+ self.close()
553
+ self._flush()
554
+ title = ' '.join(''.join(self.title).split())
555
+ return ([f'# {title}'] if title else []) + list(self.blocks)
556
+
557
+
558
+ def _html_blocks(data, label):
559
+ parser = _HtmlText(label)
560
+ parser.feed(_decode_text(data, label, declared=True))
561
+ return parser.result()
562
+
563
+
564
+ def html_blocks(data):
565
+ """Markdown blocks of an .html or .htm file in document order, for reading: the title as `#`, headings as `#`
566
+ levels, list items as `- `, table rows as `a | b`, other text as lines. Scripts, styles and the rest of the head
567
+ are not text; a link is its text.
568
+
569
+ Parameters: data (the file bytes). Returns: a list of blocks. Raises: DocumentError with a readable reason for an
570
+ encoding the reader cannot use, a page over MAX_TEXT_BYTES of text, or one without text."""
571
+ blocks = _html_blocks(data, 'HTML')
572
+ if not blocks:
573
+ raise DocumentError('HTML has no text.')
574
+ return blocks
575
+
576
+
577
+ def csv_rows(data):
578
+ """The rows of a .csv file, for reading: one tab-joined line per non-empty row (the delimiter is `,`, `;`, tab or
579
+ `|`, found from the first 4 KiB), a line break inside a cell turned into a space.
580
+
581
+ Parameters: data (the file bytes). Returns: a list of lines. Raises: DocumentError with a readable reason for an
582
+ encoding the reader cannot use, malformed quoting, a file over XLSX_MAX_ROWS rows or MAX_TEXT_BYTES of text, or one
583
+ without text."""
584
+ text = _decode_text(data, 'CSV')
585
+ try:
586
+ dialect = csv.Sniffer().sniff(text[:4096], delimiters=',;\t|')
587
+ except csv.Error:
588
+ dialect = csv.excel
589
+ rows = _Lines('CSV')
590
+ try:
591
+ for record in csv.reader(io.StringIO(text, newline=''), dialect):
592
+ values = [' '.join(cell.split()) for cell in record]
593
+ if any(values):
594
+ if len(rows) >= XLSX_MAX_ROWS:
595
+ raise DocumentError(f'CSV over {XLSX_MAX_ROWS} rows.')
596
+ rows.add('\t'.join(values))
597
+ except csv.Error as exc:
598
+ raise DocumentError(f'Could not read this CSV: {exc}.') from None
599
+ if not rows:
600
+ raise DocumentError('CSV has no text.')
601
+ return list(rows)
602
+
603
+
604
+ def _epub_chapter_paths(package):
605
+ """Zip names of the XHTML parts of an EPUB in reading order: the package document the container names, its spine
606
+ resolved through the manifest. A part outside the archive or a path leaving it is refused."""
607
+ container = package.parse('META-INF/container.xml')
608
+ rootfile = next((node.get('full-path') for node in container.iter() if _local(node.tag) == 'rootfile'), None)
609
+ opf = posixpath.normpath((rootfile or '').lstrip('/'))
610
+ if not rootfile or opf.startswith('..') or opf not in package.entries:
611
+ raise DocumentError('Invalid EPUB or excessive structure.')
612
+ package_root = package.parse(opf)
613
+ manifest = {node.get('id'): node for node in package_root.iter() if _local(node.tag) == 'item'}
614
+ base = posixpath.dirname(opf)
615
+ paths = []
616
+ for itemref in (node for node in package_root.iter() if _local(node.tag) == 'itemref'):
617
+ item = manifest.get(itemref.get('idref'))
618
+ if item is None or item.get('media-type') not in ('application/xhtml+xml', 'text/html'):
619
+ continue
620
+ part = posixpath.normpath(posixpath.join(base, unquote(item.get('href', '')).split('#')[0]))
621
+ if part.startswith('..') or part not in package.entries:
622
+ raise DocumentError('Invalid EPUB or excessive structure.')
623
+ paths.append(part)
624
+ return paths
625
+
626
+
627
+ def epub_chapters(data):
628
+ """The chapters of an .epub in reading order, for reading. Each XHTML part of the spine is read as HTML (see
629
+ html_blocks); the package is opened with the same guards as the Office formats.
630
+
631
+ Parameters: data (the file bytes). Returns: a list of block lists, one per chapter (a chapter without text is an
632
+ empty list). Raises: DocumentError with a readable reason for a corrupt, encrypted, oversized or hostile package, an
633
+ encoding the reader cannot use, or a book without any text."""
634
+ with _open_package(data, 'EPUB', 'META-INF/container.xml') as package:
635
+ chapters = [_html_blocks(package.read(path), 'EPUB') for path in _epub_chapter_paths(package)]
636
+ if not any(chapters):
637
+ raise DocumentError('EPUB has no text.')
638
+ return chapters
639
+
640
+
454
641
  PDF_SCRIPT = """
455
642
  import io, json, sys
456
643
  import pypdf
@@ -524,14 +711,15 @@ def extract_pdf(data):
524
711
  return (text + "\n").encode("utf-8", "replace")
525
712
 
526
713
 
527
- _EXTENSIONS = ('.docx', '.pptx', '.xlsx', '.pdf')
714
+ _EXTENSIONS = ('.docx', '.pptx', '.xlsx', '.xlsm', '.pdf')
528
715
 
529
716
 
530
717
  def is_extracted(path):
531
718
  """Whether the file type goes through a text extractor before indexing.
532
719
 
533
- Parameters: path (any file name or path). Returns: True for `.docx`, `.pptx`, `.xlsx` and `.pdf`, case-insensitive;
534
- macro and binary formats (`.xlsm`, `.xlsb`, `.xls`, `.ppt`, `.pptm`) are not extracted."""
720
+ Parameters: path (any file name or path). Returns: True for `.docx`, `.pptx`, `.xlsx`, `.xlsm` and `.pdf`,
721
+ case-insensitive; an `.xlsm` is read as an `.xlsx` (its XML parts only, no macro runs); the binary and other macro
722
+ formats (`.xlsb`, `.xls`, `.ppt`, `.pptm`) are not extracted."""
535
723
  return str(path).lower().endswith(_EXTENSIONS)
536
724
 
537
725
 
@@ -552,7 +740,7 @@ def extract(path, data):
552
740
  return extract_pdf(data)
553
741
  if name.endswith('.pptx'):
554
742
  return extract_pptx(data)
555
- if name.endswith('.xlsx'):
743
+ if name.endswith(('.xlsx', '.xlsm')):
556
744
  return extract_xlsx(data)
557
745
  return extract_docx(data)
558
746
 
package/dossier.py CHANGED
@@ -1,7 +1,10 @@
1
1
  """Dossier mode of smart_search: a narrow keyword query over the notes a project keeps about its own intent
2
- (docs/research, the backlog, CLAUDE.md and AGENTS.md), answered with ids and one short excerpt per hit, inside a time
3
- budget. It reads straight from disk, uses no index and no model, and never raises on a slow or unreadable file: it
4
- stops at the deadline and says so in `degraded`/`reasons`."""
2
+ (docs/research, the backlog, CLAUDE.md and AGENTS.md; with `history`, also the messages of its commits), answered with
3
+ ids and one short excerpt per hit, inside a time budget. It reads straight from disk, uses no index and no model, and
4
+ never raises on a slow or unreadable file: it stops at the deadline and says so in `degraded`/`reasons`. `notes_about`
5
+ narrows it to the notes that name one function. The commit
6
+ messages come from `doc_insights.history`, whose first read of a repository runs one `git log` that no deadline can
7
+ cut, so `history` is opt-in and a read that outlasts the budget is reported in `reasons`."""
5
8
  import json
6
9
  import os
7
10
  import re
@@ -11,6 +14,7 @@ DEFAULT_TIMEOUT_MS = 800
11
14
  MAX_TIMEOUT_MS = 5000
12
15
  DEFAULT_MAX_HITS = 5
13
16
  MAX_HITS = 10
17
+ MAX_FUNCTION_NOTES = 2
14
18
  SNIPPET_CHARS = 160
15
19
  TITLE_CHARS = 80
16
20
  MAX_FILE_BYTES = 512 * 1024
@@ -89,11 +93,21 @@ def _score(unit_lines, terms):
89
93
  return sum((term in low) + (term in title) for term in terms)
90
94
 
91
95
 
92
- def query(root, text, timeout_ms=DEFAULT_TIMEOUT_MS, max_hits=DEFAULT_MAX_HITS):
96
+ def _commit_units(root):
97
+ """`[(hash, subject, [lines])]` of the commits of `root`, newest first (a subject can carry the whole reason), and
98
+ the reason no history could be read (None when it was)."""
99
+ import doc_insights
100
+ commits, note = doc_insights.history(root)
101
+ return [(commit["hash"], commit["subject"], [commit["subject"], *commit["body"].splitlines()])
102
+ for commit in commits], note
103
+
104
+
105
+ def query(root, text, timeout_ms=DEFAULT_TIMEOUT_MS, max_hits=DEFAULT_MAX_HITS, history=False, keep=None):
93
106
  """The dossier answer for `text` over the notes of `root`: `{"mode", "degraded", "reasons", "elapsed_ms",
94
107
  "scanned", "hits": [{"id", "source", "title", "snippet"}]}`, best hit first. `id` is `path:line` of the unit's
95
- first line. `degraded` is true when the deadline cut the scan or a file could not be read; `hits` then holds what
96
- was found until then."""
108
+ first line, or `git:<hash>` for a commit message (`history` true; source `git`). `degraded` is true when the
109
+ deadline cut the scan or a file could not be read; `hits` then holds what was found until then. `keep`, when given,
110
+ is a predicate over the whole text of a unit that a unit must also pass to count."""
97
111
  started = time.monotonic()
98
112
  deadline = started + timeout_ms / 1000
99
113
  terms = focus_terms(text)
@@ -115,17 +129,62 @@ def query(root, text, timeout_ms=DEFAULT_TIMEOUT_MS, max_hits=DEFAULT_MAX_HITS):
115
129
  continue
116
130
  for number, unit in _units(lines):
117
131
  score = _score(unit, terms)
118
- if score:
132
+ if score and (keep is None or keep("\n".join(unit))):
119
133
  scored.append((score, rel, kind, number, unit))
120
134
  if unreadable:
121
135
  reasons.append("not read: " + "; ".join(unreadable[:5]))
136
+ if history and time.monotonic() <= deadline:
137
+ commits, note = _commit_units(root)
138
+ scanned += 1
139
+ if note:
140
+ reasons.append(f"git history: {note}")
141
+ if time.monotonic() > deadline:
142
+ reasons.append(f"git history read took longer than {timeout_ms} ms")
143
+ for short, _subject, unit in commits:
144
+ score = _score(unit, terms)
145
+ if score and (keep is None or keep("\n".join(unit))):
146
+ scored.append((score, f"git:{short}", "git", None, unit))
147
+ elif history:
148
+ reasons.append(f"git history not read: timeout after {timeout_ms} ms")
122
149
  scored.sort(key=lambda item: -item[0])
123
- hits = [{"id": f"{rel}:{number}", "source": kind, "title": _clean(unit[0])[:TITLE_CHARS],
150
+ hits = [{"id": rel if number is None else f"{rel}:{number}", "source": kind, "title": _clean(unit[0])[:TITLE_CHARS],
124
151
  "snippet": _snippet(unit, terms)} for _score_value, rel, kind, number, unit in scored[:max_hits]]
125
152
  return {"mode": "dossier", "degraded": bool(reasons), "reasons": reasons,
126
153
  "elapsed_ms": round((time.monotonic() - started) * 1000), "scanned": scanned, "hits": hits}
127
154
 
128
155
 
156
+ def mention_test(rel, qualified):
157
+ """A predicate over the text of one note: true when the note names the function `qualified` of the file `rel`.
158
+
159
+ A single word is also ordinary prose, so it counts only qualified, as `module.name` (the file's name without its
160
+ extension) or `Owner.name` (the class or function around it). A compound name (snake_case or camelCase, two words
161
+ or more) is distinctive enough as the bare identifier; it does not count when another name qualifies it
162
+ (`other.name`) or when it is itself a qualifier (`name.attribute`).
163
+ """
164
+ *owners, name = qualified.split(".")
165
+ stem = rel.rpartition("/")[2].rpartition(".")[0]
166
+ qualifiers = "|".join(re.escape(part) for part in (stem, *owners[-1:]) if part)
167
+ escaped = re.escape(name)
168
+ qualified_form = re.compile(rf"(?<![\w.])(?:{qualifiers})\.{escaped}(?!\w)")
169
+ if len([word for word in re.split(r"_|(?<=[a-z])(?=[A-Z])", name.strip("_")) if word]) < 2:
170
+ return lambda text: bool(qualified_form.search(text))
171
+ bare_form = re.compile(rf"(?<![\w.]){escaped}(?!\w|\.\w)")
172
+ return lambda text: bool(qualified_form.search(text) or bare_form.search(text))
173
+
174
+
175
+ def notes_about(root, rel, qualified, timeout_ms=DEFAULT_TIMEOUT_MS, history=True):
176
+ """The notes of `root`, and its commit messages when `history`, that name one function, at most `MAX_FUNCTION_NOTES`.
177
+
178
+ Parameters: root (str) project folder; rel (str) path of the file of the function, relative to root with forward
179
+ slashes; qualified (str) the function's name as `doc_check.functions` gives it (`Class.method` for a method);
180
+ timeout_ms (int) scan budget; history (bool) also read the commit messages, whose first read after a commit runs a
181
+ `git log` no deadline can cut.
182
+ Returns: the `query` answer, keeping only the units `mention_test` accepts.
183
+ """
184
+ name = qualified.rpartition(".")[2].strip("_")
185
+ return query(root, name, timeout_ms, MAX_FUNCTION_NOTES, history=history, keep=mention_test(rel, qualified))
186
+
187
+
129
188
  def answer(arguments):
130
189
  """The text smart_search returns for `mode=dossier`: the compact JSON of `query`. Raises ValueError for a missing
131
190
  folder or text, or a `timeout_ms`/`max_hits` out of range, so the caller sees the error instead of an empty dossier."""
@@ -138,7 +197,10 @@ def answer(arguments):
138
197
  raise ValueError("smart_search mode=dossier requires 'query_identifiers' (or 'doc_query') with at least one word.")
139
198
  timeout_ms = _bounded(arguments, "timeout_ms", DEFAULT_TIMEOUT_MS, 1, MAX_TIMEOUT_MS)
140
199
  max_hits = _bounded(arguments, "max_hits", DEFAULT_MAX_HITS, 1, MAX_HITS)
141
- return json.dumps(query(root, text, timeout_ms, max_hits), ensure_ascii=False, separators=(",", ":"))
200
+ history = arguments.get("history", False)
201
+ if not isinstance(history, bool):
202
+ raise ValueError("history must be true or false.")
203
+ return json.dumps(query(root, text, timeout_ms, max_hits, history), ensure_ascii=False, separators=(",", ":"))
142
204
 
143
205
 
144
206
  def _bounded(arguments, name, default, low, high):
@@ -12,13 +12,35 @@ import time
12
12
  import project_identity
13
13
 
14
14
 
15
+ def _open(root, directory):
16
+ os.makedirs(directory, exist_ok=True)
17
+ db = sqlite3.connect(os.path.join(directory, project_identity.project_id(root) + '.vectors.sqlite3'), timeout=10)
18
+ db.execute('PRAGMA secure_delete=ON')
19
+ db.execute('CREATE TABLE IF NOT EXISTS vectors(key TEXT PRIMARY KEY, vector BLOB NOT NULL, used_at REAL NOT NULL, '
20
+ 'model TEXT)')
21
+ if 'model' not in {row[1] for row in db.execute('PRAGMA table_info(vectors)')}:
22
+ db.execute('ALTER TABLE vectors ADD COLUMN model TEXT')
23
+ return db
24
+
25
+
26
+ def drop_other_models(root, directory, model):
27
+ """Deletes the cached vectors of every model but `model` and gives their space back to the disk. Rows written
28
+ before the cache recorded the model are deleted too, unless a build of `model` reused them since."""
29
+ db = _open(root, directory)
30
+ try:
31
+ dropped = db.execute('DELETE FROM vectors WHERE model IS NOT ?', (model,)).rowcount
32
+ db.commit()
33
+ if dropped:
34
+ db.execute('VACUUM')
35
+ return dropped
36
+ finally:
37
+ db.close()
38
+
39
+
15
40
  class VectorCache:
16
41
  def __init__(self, root, directory, contract):
17
- os.makedirs(directory, exist_ok=True)
18
- path = os.path.join(directory, project_identity.project_id(root) + '.vectors.sqlite3')
19
- self.db = sqlite3.connect(path, timeout=10)
20
- self.db.execute('PRAGMA secure_delete=ON')
21
- self.db.execute('CREATE TABLE IF NOT EXISTS vectors(key TEXT PRIMARY KEY, vector BLOB NOT NULL, used_at REAL NOT NULL)')
42
+ self.db = _open(root, directory)
43
+ self.model = contract.get('model')
22
44
  self.contract = json.dumps(contract, sort_keys=True, ensure_ascii=False)
23
45
  self.dimensions = contract.get('dimensions')
24
46
  self.hits = self.misses = self.characters = 0
@@ -32,18 +54,20 @@ class VectorCache:
32
54
  packed = array.array('f', vector)
33
55
  if not all(math.isfinite(value) for value in packed):
34
56
  raise ValueError('Embedding does not fit in float32; cache not written.')
35
- self.db.execute('INSERT OR REPLACE INTO vectors VALUES(?,?,?)',
36
- (self.key(text), packed.tobytes(), time.time()))
57
+ self.db.execute('INSERT OR REPLACE INTO vectors VALUES(?,?,?,?)',
58
+ (self.key(text), packed.tobytes(), time.time(), self.model))
37
59
 
38
60
  def embed(self, texts, embed_batch, validate, use_cache=True, split=None, workers=1):
39
61
  """Cache lookups and writes stay on the calling thread (sqlite); only embed_batch runs in workers."""
40
62
  output = [None] * len(texts)
41
63
  missing = {}
42
64
  for i, text in enumerate(texts):
43
- row = self.db.execute('SELECT vector FROM vectors WHERE key=?', (self.key(text),)).fetchone() if use_cache else None
65
+ row = self.db.execute('SELECT vector, model FROM vectors WHERE key=?', (self.key(text),)).fetchone() if use_cache else None
44
66
  if row and self.dimensions:
45
67
  vector = array.array('f'); vector.frombytes(row[0])
46
68
  validate(vector, self.dimensions)
69
+ if row[1] is None:
70
+ self.db.execute('UPDATE vectors SET model=? WHERE key=?', (self.model, self.key(text)))
47
71
  output[i] = list(vector)
48
72
  self.hits += 1
49
73
  else: