@allansantos-dev/smart-tool 0.9.43 → 0.9.46

package/document_read.py

@@ -0,0 +1,272 @@
+"""read_document: PDF, Word, PowerPoint and Excel files read as compact text, one page, slide or sheet range at a time.
+
+The agent gets the file facts first (type, size, pages, slides or sheets, estimated tokens) and then only the range it
+asks for, so a 200-page PDF is never loaded whole. Extraction is the one of document_text (pypdf in an isolated
+subprocess for PDF; zipfile and ElementTree with the shared guards for docx, pptx and xlsx); this module adds the unit
+split, the markdown shape, the compaction of PDF line wrapping and the range arithmetic."""
+import collections
+import math
+import os
+import re
+import threading
+import unicodedata
+
+import document_text
+
+KINDS = {'.pdf': 'pdf', '.docx': 'docx', '.pptx': 'pptx', '.xlsx': 'xlsx'}
+UNIT_NAMES = {'pdf': 'page', 'docx': 'part', 'pptx': 'slide', 'xlsx': 'sheet'}
+DEFAULT_MAX_CHARS = 20_000
+MAX_CHARS_LIMIT = 100_000
+DOCX_PART_CHARS = 6_000
+CHARS_PER_TOKEN = 4
+CACHE_DOCUMENTS = 4
+LONG_LINE = 60
+
+_cache = collections.OrderedDict()
+_cache_lock = threading.Lock()
+
+
+class Unit:
+    """One readable unit of a document: a PDF page, a docx part, a slide or a sheet."""
+
+    def __init__(self, number, text, name=''):
+        self.number = number
+        self.text = text
+        self.name = name
+
+
+class Document:
+    """A document split into units, with the facts the metadata block reports."""
+
+    def __init__(self, path, kind, size, units, facts):
+        self.path = path
+        self.kind = kind
+        self.size = size
+        self.units = units
+        self.facts = facts
+
+
+def estimate_tokens(text):
+    """Token estimate of a text: characters divided by four, rounded up (a rule of thumb, not a tokenizer count).
+
+    Parameters: text. Returns: the estimate as an int."""
+    return math.ceil(len(text) / CHARS_PER_TOKEN)
+
+
+def compact_pdf_text(text):
+    """PDF page text with the layout noise taken out: ligatures and compatibility forms normalised, a hyphen at a line
+    end joined to the word, a wrapped line (long, no closing punctuation, next one starting in lower case) joined to the
+    next, spaces collapsed and blank-line runs reduced to one.
+
+    Parameters: text (one page as pypdf returns it). Returns: the compact text."""
+    text = unicodedata.normalize('NFKC', text).replace('\r\n', '\n').replace('\r', '\n')
+    text = re.sub(r'(\w)-\n(?=[a-zà-ÿ])', r'\1', text)
+    lines = [re.sub(r'[ \t]+', ' ', line).strip() for line in text.split('\n')]
+    joined = []
+    for line in lines:
+        previous = joined[-1] if joined else ''
+        if (previous and len(previous) >= LONG_LINE and line[:1].islower()
+                and not previous.endswith(('.', ':', ';', '!', '?', ')', '|'))):
+            joined[-1] = previous + ' ' + line
+        else:
+            joined.append(line)
+    return re.sub(r'\n{3,}', '\n\n', '\n'.join(joined)).strip()
+
+
+def _pdf_units(data):
+    """Units and facts of a PDF, one unit per page; raises DocumentError for a password, a corrupt file or a file with no
+    text layer at all (scanned pages are not read, there is no OCR)."""
+    pages, count = document_text.pdf_pages(data, document_text.PDF_TIMEOUT_S, document_text.PDF_MAX_PAGES,
+                                           document_text.PDF_MAX_CHARS)
+    if not any(pages):
+        raise document_text.DocumentError(
+            f'PDF of {count} pages has no text layer (scanned pages are not read; there is no OCR).')
+    units = [Unit(number, compact_pdf_text(page) or '(no text on this page)') for number, page in enumerate(pages, 1)]
+    facts = {'pages': count, 'pages without text': sum(1 for page in pages if not page)}
+    if count > len(pages):
+        facts['not read'] = f'pages {len(pages) + 1}-{count} (over the {document_text.PDF_MAX_PAGES} page or ' \
+                            f'{document_text.PDF_MAX_CHARS:,} character cap)'
+    return units, facts
+
+
+def _docx_units(data):
+    """Units and facts of a docx: the markdown blocks grouped in parts of about DOCX_PART_CHARS characters, a block never
+    split (a docx has no fixed pages, they depend on the renderer)."""
+    parts, current, size = [], [], 0
+    for block in document_text.docx_blocks(data):
+        if current and size + len(block) > DOCX_PART_CHARS:
+            parts.append('\n'.join(current))
+            current, size = [], 0
+        current.append(block)
+        size += len(block) + 1
+    parts.append('\n'.join(current))
+    return [Unit(number, text) for number, text in enumerate(parts, 1)], {
+        'parts': len(parts), 'note': f'a docx has no fixed pages; a part is about {DOCX_PART_CHARS:,} characters'}
+
+
+def _pptx_units(data):
+    """Units and facts of a pptx: one unit per slide under `[slide N]`, the speaker notes under `[notes N]`."""
+    units = []
+    for number, body, notes in document_text.pptx_slides(data):
+        lines = [f'[slide {number}]'] + (body or ['(no text)']) + ([f'[notes {number}]'] + notes if notes else [])
+        units.append(Unit(number, '\n'.join(lines)))
+    return units, {'slides': len(units), 'slides without text': sum(1 for u in units if '\n(no text)' in u.text)}
+
+
+def _xlsx_units(data):
+    """Units and facts of an xlsx: one unit per sheet under `[sheet NAME]`, then one tab-separated line per row."""
+    units = []
+    for number, (name, rows) in enumerate(document_text.xlsx_sheets(data), 1):
+        units.append(Unit(number, '\n'.join([f'[sheet {name}]'] + (rows or ['(no text)'])), name))
+    return units, {'sheet names': ', '.join(f'{u.number}={u.name}' for u in units)}
+
+
+_BUILDERS = {'pdf': _pdf_units, 'docx': _docx_units, 'pptx': _pptx_units, 'xlsx': _xlsx_units}
+
+
+def load(path):
+    """The document at `path`, split into units. Parsed documents are kept for a few calls (keyed by path, size and
+    modification time), so reading page after page parses the file once.
+
+    Parameters: path (absolute path of a .pdf, .docx, .pptx or .xlsx file). Returns: a Document. Raises: ValueError for a
+    relative path, a missing file, an unsupported type or a file over the size limit of its type; DocumentError with a
+    readable reason for a file the extractor refuses (corrupt, encrypted, hostile, scanned, without text)."""
+    if not isinstance(path, str) or not path.strip():
+        raise ValueError("read_document requires 'path'")
+    if not os.path.isabs(path):
+        raise ValueError(f'read_document needs an absolute path, got {path!r}')
+    extension = os.path.splitext(path)[1].lower()
+    if extension not in KINDS:
+        raise ValueError(f"read_document reads {', '.join(KINDS)} files; {extension or 'this file'} is not supported "
+                         '(use the normal Read for plain text; legacy .doc, .xls and .ppt and macro files are not read)')
+    try:
+        info = os.stat(path)
+    except OSError as exc:
+        raise ValueError(f'cannot read {path}: {exc.strerror or type(exc).__name__}') from None
+    if not os.path.isfile(path):
+        raise ValueError(f'{path} is not a file')
+    if info.st_size > document_text.size_limit(path):
+        raise ValueError(f'{path} is {info.st_size / 1048576:.1f} MiB, over the '
+                         f'{document_text.size_limit(path) // 1048576} MiB limit of a {extension} file')
+    key = (os.path.normcase(path), info.st_size, info.st_mtime_ns)
+    with _cache_lock:
+        if key in _cache:
+            _cache.move_to_end(key)
+            return _cache[key]
+    with open(path, 'rb') as stream:
+        data = stream.read()
+    kind = KINDS[extension]
+    units, facts = _BUILDERS[kind](data)
+    document = Document(path, kind, info.st_size, units, facts)
+    with _cache_lock:
+        _cache[key] = document
+        while len(_cache) > CACHE_DOCUMENTS:
+            _cache.popitem(last=False)
+    return document
+
+
+def parse_range(spec, document):
+    """Unit numbers selected by a range text: `3`, `3-5`, `7-` (to the end), `-4` (the first four), a comma list of
+    those (`1,4-6`) and, for a workbook, a sheet name (case-insensitive).
+
+    Parameters: spec (the text), document (a Document). Returns: the sorted list of unit numbers. Raises: ValueError
+    naming the valid range for a malformed or out-of-range spec or an unknown sheet name."""
+    count = len(document.units)
+    unit = UNIT_NAMES[document.kind]
+    chosen = set()
+    for part in (piece.strip() for piece in spec.split(',')):
+        if document.kind == 'xlsx':
+            named = [u.number for u in document.units if u.name.lower() == part.lower()]
+            if named:
+                chosen.update(named)
+                continue
+        found = re.fullmatch(r'(\d*)\s*(-?)\s*(\d*)', part)
+        if not part or not found or not (found.group(1) or found.group(3)):
+            raise ValueError(f'range {spec!r} is not valid: use a {unit} number, a span like 3-5, 7- or -4, or a comma list'
+                             + ('; a sheet name also works' if document.kind == 'xlsx' else ''))
+        first, dash, last = found.groups()
+        if not dash and last:
+            raise ValueError(f'range {spec!r} is not valid: use a {unit} number, a span like 3-5, 7- or -4, or a comma list')
+        low = int(first) if first else 1
+        high = int(last) if last else (count if dash else low)
+        if low < 1 or high > count or low > high:
+            raise ValueError(f'range {spec!r} is outside the document: it has {count} {unit}s (1-{count})')
+        chosen.update(range(low, high + 1))
+    return sorted(chosen)
+
+
+def _spans(numbers):
+    """Compact text of unit numbers: `1-3, 7`."""
+    spans, start, previous = [], None, None
+    for number in numbers + [None]:
+        if start is None:
+            start = previous = number
+        elif number is not None and number == previous + 1:
+            previous = number
+        else:
+            spans.append(str(start) if start == previous else f'{start}-{previous}')
+            start = previous = number
+    return ', '.join(spans)
+
+
+def header(document):
+    """The metadata block that opens every answer: path, type, size, units and the estimated tokens of the whole text.
+
+    Parameters: document. Returns: the block as text."""
+    total = sum(estimate_tokens(unit.text) for unit in document.units)
+    lines = [f'document: {document.path}',
+             f'type: {document.kind} | size: {document.size / 1048576:.2f} MiB | '
+             f'{UNIT_NAMES[document.kind]}s: {len(document.units)} | est. tokens (all text): {total:,}']
+    lines += [f'{key}: {value}' for key, value in document.facts.items()
+              if value and key != UNIT_NAMES[document.kind] + 's']
+    return '\n'.join(lines)
+
+
+def read(path, range_spec=None, max_chars=DEFAULT_MAX_CHARS, info_only=False):
+    """The answer of read_document: the metadata block, then the chosen units as markdown, a note naming the next range
+    when the text was cut.
+
+    Parameters: path (see load), range_spec (see parse_range; None reads from the first unit up to max_chars),
+    max_chars (cap of the text returned, 1000-MAX_CHARS_LIMIT; a single unit over the cap is cut at a line with a
+    notice), info_only (only the metadata block). Returns: the text. Raises: as load and parse_range, ValueError for a
+    max_chars out of bounds."""
+    if isinstance(max_chars, bool) or not isinstance(max_chars, int) or not 1000 <= max_chars <= MAX_CHARS_LIMIT:
+        raise ValueError(f'max_chars must be an integer from 1000 to {MAX_CHARS_LIMIT}')
+    document = load(path)
+    head = header(document)
+    if info_only:
+        return head
+    numbers = parse_range(range_spec, document) if range_spec else list(range(1, len(document.units) + 1))
+    unit = UNIT_NAMES[document.kind]
+    body, shown, size, cut = [], [], 0, None
+    for number in numbers:
+        text = document.units[number - 1].text
+        label = f'[{unit} {number}]\n' if document.kind in ('pdf', 'docx') else ''
+        piece = label + text
+        if shown and size + len(piece) > max_chars:
+            break
+        if len(piece) > max_chars:
+            lines = piece.split('\n')
+            kept, used = [], 0
+            for line in lines:
+                if used + len(line) + 1 > max_chars and kept:
+                    break
+                kept.append(line)
+                used += len(line) + 1
+            piece = '\n'.join(kept)
+            cut = (f'{unit} {number} cut after {len(kept)} of {len(lines)} lines; raise max_chars '
+                   f'(up to {MAX_CHARS_LIMIT:,}) to read the rest')
+        body.append(piece)
+        shown.append(number)
+        size += len(piece) + 1
+        if cut:
+            break
+    returned = '\n\n'.join(body)
+    notes = [f'returned: {unit}s {_spans(shown)} (est. {estimate_tokens(returned):,} tokens)']
+    remaining = [number for number in numbers if number > shown[-1]]
+    if cut:
+        notes.append(cut)
+    if remaining:
+        notes.append(f'next: range="{remaining[0]}-{remaining[-1]}"' if not range_spec
+                     else f'not returned: {unit}s {_spans(remaining)} (max_chars); ask for them with range')
+    return '\n'.join([head] + notes + ['---', returned])

package/document_text.py

@@ -115,39 +115,157 @@ def _local(tag):
     return tag.rsplit('}', 1)[-1]
 
 
+def _docx_namespace(root):
+    """The WordprocessingML namespace prefix (transitional or strict) of a parsed part."""
+    return W if root.tag.startswith(W) else '{http://purl.oclc.org/ooxml/wordprocessingml/main}'
+
+
+def _docx_text(paragraph, namespace):
+    """Text of one `w:p`: runs, tabs and line breaks; a nested paragraph (text box) is not part of it."""
+    pending = list(reversed(paragraph))
+    pieces = []
+    while pending:
+        child = pending.pop()
+        if child.tag == namespace + 'p':
+            continue
+        if child.tag == namespace + 't':
+            pieces.append(child.text or '')
+        elif child.tag == namespace + 'tab':
+            pieces.append('\t')
+        elif child.tag in (namespace + 'br', namespace + 'cr'):
+            pieces.append('\n')
+        else:
+            pending.extend(reversed(child))
+    return ''.join(pieces)
+
+
+def _docx_extra_parts(package):
+    """Names of the header, footer, footnote and endnote parts of a .docx, in a stable order."""
+    return sorted(name for name in package.entries if re.fullmatch(
+        r'word/(?:header\d+|footer\d+|footnotes|endnotes)\.xml', name))
+
+
 def extract_docx(data):
     """Text of a .docx: body, headers, footers, footnotes and endnotes, one line per non-empty paragraph.
 
     Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
     corrupt, encrypted or oversized package, or one with a DOCTYPE or entity."""
     with _open_package(data, 'DOCX', 'word/document.xml') as package:
-        names = ['word/document.xml'] + sorted(name for name in package.entries if re.fullmatch(
-            r'word/(?:header\d+|footer\d+|footnotes|endnotes)\.xml', name))
         paragraphs = _Lines('DOCX')
-        for name in names:
+        for name in ['word/document.xml'] + _docx_extra_parts(package):
             root = package.parse(name)
-            namespace = W if root.tag.startswith(W) else '{http://purl.oclc.org/ooxml/wordprocessingml/main}'
+            namespace = _docx_namespace(root)
             for paragraph in root.iter(namespace + 'p'):
-                def parts(node):
-                    pending = list(reversed(node))
-                    while pending:
-                        child = pending.pop()
-                        if child.tag == namespace + 'p':
-                            continue
-                        if child.tag == namespace + 't':
-                            yield child.text or ''
-                        elif child.tag == namespace + 'tab':
-                            yield '\t'
-                        elif child.tag in (namespace + 'br', namespace + 'cr'):
-                            yield '\n'
-                        else:
-                            pending.extend(reversed(child))
-                text = ''.join(parts(paragraph))
+                text = _docx_text(paragraph, namespace)
                 if text.strip():
                     paragraphs.add(text)
         return ('\n'.join(paragraphs) + '\n' if paragraphs else '').encode('utf-8')
 
 
+def _docx_heading_levels(package, namespace):
+    """`{styleId: level}` of the heading styles of word/styles.xml (the style name is `heading N` in every language,
+    `Title` is level 1); empty when the part is missing."""
+    if 'word/styles.xml' not in package.entries:
+        return {}
+    levels = {}
+    for style in package.parse('word/styles.xml').iter(namespace + 'style'):
+        name = next((node.get(namespace + 'val', '') for node in style if node.tag == namespace + 'name'), '').lower()
+        found = re.fullmatch(r'heading (\d)', name)
+        if found or name == 'title':
+            levels[style.get(namespace + 'styleId')] = int(found.group(1)) if found else 1
+    return levels
+
+
+def _docx_paragraph_line(paragraph, namespace, levels):
+    """Markdown line of a paragraph: `#` heading by style or outline level, `- ` item (indented by level) for a list
+    paragraph, else the plain text; an empty string for a paragraph without text."""
+    text = ' '.join(_docx_text(paragraph, namespace).split())
+    if not text:
+        return ''
+    properties = next((node for node in paragraph if node.tag == namespace + 'pPr'), None)
+    if properties is None:
+        return text
+    style = next((node.get(namespace + 'val') for node in properties if node.tag == namespace + 'pStyle'), None)
+    outline = next((node.get(namespace + 'val') for node in properties if node.tag == namespace + 'outlineLvl'), None)
+    level = levels.get(style) or (int(outline) + 1 if outline and outline.isdigit() and int(outline) < 6 else 0)
+    if level:
+        return '#' * min(level, 6) + ' ' + text
+    numbering = next((node for node in properties if node.tag == namespace + 'numPr'), None)
+    if numbering is not None:
+        depth = next((node.get(namespace + 'val') for node in numbering if node.tag == namespace + 'ilvl'), '0')
+        return '  ' * (int(depth) if depth.isdigit() else 0) + '- ' + text
+    return text
+
+
+def _docx_table_lines(table, namespace):
+    """GFM table lines of a `w:tbl`: one row per `w:tr`, the first row as the header, cell text on one line."""
+    rows = []
+    for row in table:
+        if row.tag != namespace + 'tr':
+            continue
+        cells = []
+        for cell in row:
+            if cell.tag == namespace + 'tc':
+                text = ' '.join(' '.join(_docx_text(p, namespace).split()) for p in cell.iter(namespace + 'p'))
+                cells.append(' '.join(text.split()).replace('|', '\\|'))
+        if any(cells):
+            rows.append('| ' + ' | '.join(cells) + ' |')
+    if len(rows) > 1:
+        rows.insert(1, '|' + ' --- |' * (rows[0].count(' | ') + 1))
+    return rows
+
+
+def _docx_body_items(node, namespace):
+    """The paragraphs and tables of a body (or of a content control inside it) in document order."""
+    for child in node:
+        if child.tag in (namespace + 'p', namespace + 'tbl'):
+            yield child
+        elif child.tag == namespace + 'sdt':
+            for content in child:
+                if content.tag == namespace + 'sdtContent':
+                    yield from _docx_body_items(content, namespace)
+
+
+def docx_blocks(data):
+    """Markdown blocks of a .docx in document order, for reading: headings as `#`, list items as `- `, tables as GFM
+    tables, other paragraphs as text. Footnotes, endnotes and the distinct header and footer lines follow under
+    `## Notes` and `## Headers and footers`.
+
+    Parameters: data (the file bytes). Returns: a list of blocks (a block is a paragraph line or a whole table, lines
+    joined by newline). Raises: DocumentError with a readable reason for a corrupt, encrypted, oversized or hostile
+    package, or one without text."""
+    with _open_package(data, 'DOCX', 'word/document.xml') as package:
+        root = package.parse('word/document.xml')
+        namespace = _docx_namespace(root)
+        levels = _docx_heading_levels(package, namespace)
+        blocks = _Lines('DOCX')
+        body = next((node for node in root if node.tag == namespace + 'body'), root)
+        for item in _docx_body_items(body, namespace):
+            if item.tag == namespace + 'tbl':
+                table = _docx_table_lines(item, namespace)
+                if table:
+                    blocks.add('\n'.join(table))
+            else:
+                line = _docx_paragraph_line(item, namespace, levels)
+                if line:
+                    blocks.add(line)
+        notes, frame = [], []
+        for name in _docx_extra_parts(package):
+            part = package.parse(name)
+            target = frame if re.search(r'header|footer', name) else notes
+            for paragraph in part.iter(_docx_namespace(part) + 'p'):
+                text = ' '.join(_docx_text(paragraph, _docx_namespace(part)).split())
+                if text and text not in target:
+                    target.append(text)
+        if notes:
+            blocks.add('## Notes', *notes)
+        if frame:
+            blocks.add('## Headers and footers', *frame)
+    if not blocks:
+        raise DocumentError('DOCX has no text.')
+    return list(blocks)
+
+
 def _drawing_paragraphs(root):
     """Non-empty paragraphs (`a:p`) of a slide or notes part, from the `a:t` runs; the slide-number field is not text."""
     lines = []
@@ -173,26 +291,40 @@ def _drawing_paragraphs(root):
     return lines
 
 
-def extract_pptx(data):
-    """Text of a .pptx: each slide under `[slide N]`, then its speaker notes under `[notes N]`, in slide order.
+def pptx_slides(data):
+    """The slides of a .pptx in slide order, for extraction and for reading.
 
-    Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
-    corrupt, encrypted, oversized or hostile package, or a deck without any text."""
+    Parameters: data (the file bytes). Returns: a list of `(number, body lines, notes lines)`, a slide without text
+    included with empty lists. Raises: DocumentError with a readable reason for a corrupt, encrypted, oversized or hostile
+    package or one over MAX_TEXT_BYTES of text."""
     def has_slides(names):
         return any(re.fullmatch(r'ppt/slides/slide\d+\.xml', name) for name in names)
 
     with _open_package(data, 'PPTX', has_slides) as package:
-        slides = sorted((int(m.group(1)), name) for name in package.entries
-                        if (m := re.fullmatch(r'ppt/slides/slide(\d+)\.xml', name)))
-        lines = _Lines('PPTX')
-        for number, name in slides:
+        numbered = sorted((int(m.group(1)), name) for name in package.entries
+                          if (m := re.fullmatch(r'ppt/slides/slide(\d+)\.xml', name)))
+        budget = _Lines('PPTX')
+        slides = []
+        for number, name in numbered:
             body = _drawing_paragraphs(package.parse(name))
-            if body:
-                lines.add(f'[slide {number}]', *body)
             notes_name = f'ppt/notesSlides/notesSlide{number}.xml'
             notes = _drawing_paragraphs(package.parse(notes_name)) if notes_name in package.entries else []
-            if notes:
-                lines.add(f'[notes {number}]', *notes)
+            budget.add(*body, *notes)
+            slides.append((number, body, notes))
+    return slides
+
+
+def extract_pptx(data):
+    """Text of a .pptx: each slide under `[slide N]`, then its speaker notes under `[notes N]`, in slide order.
+
+    Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
+    corrupt, encrypted, oversized or hostile package, or a deck without any text."""
+    lines = _Lines('PPTX')
+    for number, body, notes in pptx_slides(data):
+        if body:
+            lines.add(f'[slide {number}]', *body)
+        if notes:
+            lines.add(f'[notes {number}]', *notes)
     if not lines:
         raise DocumentError('PPTX has no text.')
     return ('\n'.join(lines) + '\n').encode('utf-8')
@@ -264,14 +396,13 @@ def _sheet_rows(root, strings):
             yield '\t'.join(values)
 
 
-def extract_xlsx(data):
-    """Text of an .xlsx: each sheet, in workbook order, under `[sheet NAME]`, then one line per row with the cells
-    separated by tabs. Shared strings are resolved; numbers and inline strings are kept; a formula contributes its cached
-    value, nothing is calculated.
+def xlsx_sheets(data):
+    """The sheets of an .xlsx in workbook order, for extraction and for reading. Shared strings are resolved; numbers
+    and inline strings are kept; a formula contributes its cached value, nothing is calculated.
 
-    Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
-    corrupt, encrypted, oversized or hostile package, a workbook over XLSX_MAX_ROWS rows or MAX_TEXT_BYTES of text, or
-    one without any text."""
+    Parameters: data (the file bytes). Returns: a list of `(name, rows)`, `rows` being the tab-joined text of each
+    non-empty row (empty for a sheet without text). Raises: DocumentError with a readable reason for a corrupt,
+    encrypted, oversized or hostile package, or a workbook over XLSX_MAX_ROWS rows or MAX_TEXT_BYTES of text."""
     with _open_package(data, 'XLSX', 'xl/workbook.xml') as package:
         workbook = package.parse('xl/workbook.xml')
         targets = {}
@@ -281,22 +412,40 @@ def extract_xlsx(data):
                 targets[rel.get('Id')] = posixpath.normpath(target.lstrip('/') if target.startswith('/')
                                                             else posixpath.join('xl', target))
         strings = _shared_strings(package)
-        lines = _Lines('XLSX')
-        rows = 0
+        budget = _Lines('XLSX')
+        sheets = []
+        total = 0
         for sheet in (node for node in workbook.iter() if _local(node.tag) == 'sheet'):
             rid = next((value for key, value in sheet.attrib.items() if _local(key) == 'id'), None)
             part = targets.get(rid)
             if part not in package.entries:
                 raise DocumentError('Invalid XLSX or excessive structure.')
-            header = f"[sheet {' '.join(sheet.get('name', '').split())}]"
+            name = ' '.join(sheet.get('name', '').split())
+            rows = []
             for line in _sheet_rows(package.parse(part), strings):
-                rows += 1
-                if rows > XLSX_MAX_ROWS:
+                total += 1
+                if total > XLSX_MAX_ROWS:
                     raise DocumentError(f'XLSX over {XLSX_MAX_ROWS} rows.')
-                if header:
-                    lines.add(header)
-                    header = ''
-                lines.add(line)
+                if not rows:
+                    budget.add(f'[sheet {name}]')
+                budget.add(line)
+                rows.append(line)
+            sheets.append((name, rows))
+    return sheets
+
+
+def extract_xlsx(data):
+    """Text of an .xlsx: each sheet, in workbook order, under `[sheet NAME]`, then one line per row with the cells
+    separated by tabs. Shared strings are resolved; numbers and inline strings are kept; a formula contributes its cached
+    value, nothing is calculated.
+
+    Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
+    corrupt, encrypted, oversized or hostile package, a workbook over XLSX_MAX_ROWS rows or MAX_TEXT_BYTES of text, or
+    one without any text."""
+    lines = _Lines('XLSX')
+    for name, rows in xlsx_sheets(data):
+        if rows:
+            lines.add(f'[sheet {name}]', *rows)
     if not lines:
         raise DocumentError('XLSX has no text.')
     return ('\n'.join(lines) + '\n').encode('utf-8')
@@ -320,14 +469,15 @@ print(json.dumps({"pages": pages, "count": len(reader.pages)}))
 """
 
 
-def run_pdf(data, timeout, max_pages, max_chars):
-    """Text of a PDF (pypdf, pure Python), page by page, with `[page N]` markers. A hostile PDF can make pypdf parse for
-    minutes, so it runs in an isolated subprocess (`-I`) killed at the deadline.
+def pdf_pages(data, timeout, max_pages, max_chars):
+    """Text of each page of a PDF (pypdf, pure Python). A hostile PDF can make pypdf parse for minutes, so it runs in an
+    isolated subprocess (`-I`) killed at the deadline.
 
     Parameters: data (PDF bytes), timeout (seconds before the subprocess is killed), max_pages, max_chars (cap on the
-    text read, over all pages). Returns: the marked text, with a `[pages A-B not read]` line when a cap cut it; empty
-    pages are left out. Raises: PdfTimeout, PdfEncrypted (a password is needed), PdfUnreadable (corrupt; `detail` has
-    the last line pypdf reported), all DocumentError."""
+    text read, over all pages). Returns: `(pages, count)`: the text of the pages read (an empty string for a page
+    without text) and the page count of the file, which is larger when a cap cut the reading. Raises: PdfTimeout,
+    PdfEncrypted (a password is needed), PdfUnreadable (corrupt; `detail` has the last line pypdf reported), all
+    DocumentError."""
     try:
         done = subprocess.run([sys.executable, "-I", "-c", PDF_SCRIPT, str(max_pages), str(max_chars)],
                               input=data, capture_output=True, timeout=timeout,
@@ -341,11 +491,19 @@ def run_pdf(data, timeout, max_pages, max_chars):
         result = json.loads(done.stdout)
         if result.get("encrypted"):
             raise PdfEncrypted("PDF protected by a password.")
-        pages, count = result["pages"], result["count"]
+        return result["pages"], result["count"]
     except (ValueError, KeyError, AttributeError) as exc:
         if isinstance(exc, PdfEncrypted):
             raise
         raise PdfUnreadable("PDF could not be read.", f"unexpected extractor output ({type(exc).__name__})") from None
+
+
+def run_pdf(data, timeout, max_pages, max_chars):
+    """Text of a PDF through pdf_pages, with `[page N]` markers.
+
+    Parameters: as pdf_pages. Returns: the marked text, with a `[pages A-B not read]` line when a cap cut it; empty
+    pages are left out. Raises: as pdf_pages."""
+    pages, count = pdf_pages(data, timeout, max_pages, max_chars)
     text = "\n\n".join(f"[page {number}]\n{page}" for number, page in enumerate(pages, 1) if page)
     if count > len(pages):
         text += f"\n\n[pages {len(pages) + 1}-{count} not read]"

package/folder_picker.py

@@ -6,14 +6,16 @@ import threading
 import urllib.request
 import webbrowser
 
+import i18n
+
 _PICKER_LOCK = threading.Lock()
 
 
 def choose_folder():
     if os.name != "nt":
-        raise RuntimeError("Native folder selection is only available on Windows; enter the folder path.")
+        raise RuntimeError(i18n.t("Native folder selection is only available on Windows; enter the folder path."))
     if not _PICKER_LOCK.acquire(blocking=False):
-        raise ValueError("A folder picker is already open.")
+        raise ValueError(i18n.t("A folder picker is already open."))
     import ctypes
     from ctypes import wintypes
 
@@ -37,14 +39,14 @@ def choose_folder():
         label = ctypes.create_unicode_buffer(32768)
         info = BrowseInfo()
         info.pszDisplayName = ctypes.cast(label, wintypes.LPWSTR)
-        info.lpszTitle = "Smart Tool — select the folder to index"
+        info.lpszTitle = i18n.t("Smart Tool — select the folder to index")
         info.ulFlags = 0x0001 | 0x0040  # Diretórios reais; diálogo redimensionável.
         selected = shell.SHBrowseForFolderW(ctypes.byref(info))
         if not selected:
             return None
         path = ctypes.create_unicode_buffer(32768)
         if not shell.SHGetPathFromIDListEx(selected, path, len(path), 0):
-            raise ValueError("The selection is not an accessible folder.")
+            raise ValueError(i18n.t("The selection is not an accessible folder."))
         return path.value
     finally:
         if selected:
@@ -64,7 +66,7 @@ def register_from_tray():
         page = response.read().decode("utf-8")
     match = re.search(r'<meta name="setup-token" content="([a-zA-Z0-9_-]+)">', page)
     if not match:
-        raise RuntimeError("Reopen Smart Tool to load the updated projects page.")
+        raise RuntimeError(i18n.t("Reopen Smart Tool to load the updated projects page."))
     request = urllib.request.Request(url + "/setup/projects", method="POST",
         headers={"Content-Type": "application/json", "X-Setup-Token": match.group(1)},
         data=json.dumps({"action": "register", "project_root": root}).encode("utf-8"))