@allansantos-dev/smart-tool 0.9.40 → 0.9.42

package/document_text.py

@@ -1,13 +1,16 @@
-"""Extração local de texto DOCX e PDF, sem Office, execução de macros ou acesso externo."""
+"""Extração local de texto DOCX, PPTX, XLSX e PDF, sem Office, execução de macros ou acesso externo."""
+import contextlib
 import io
 import json
+import posixpath
 import re
 import subprocess
 import sys
 import zipfile
 import xml.etree.ElementTree as ET
+import zlib
 
-VERSION = 2
+VERSION = 3
 PDF_MAX_BYTES = 20 * 1024 * 1024
 PDF_TIMEOUT_S = 20
 PDF_MAX_PAGES = 300
@@ -15,6 +18,8 @@ PDF_MAX_CHARS = 1_000_000
 MAX_ARCHIVE_BYTES = 10 * 1024 * 1024
 MAX_XML_BYTES = 8 * 1024 * 1024
 MAX_TEXT_BYTES = 1024 * 1024
+XLSX_MAX_ROWS = 100_000
+XLSX_MAX_COLUMNS = 16_384
 W = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}'
 
 
@@ -37,53 +42,264 @@ class PdfUnreadable(DocumentError):
 
 
 class _DocumentTreeBuilder(ET.TreeBuilder):
+    label = 'DOCX'
+
     def doctype(self, name, pubid, system):
         # O callback também recusa DTD em XML UTF-16, antes de expandir entidades.
-        raise DocumentError('DOCX contains disallowed entities.')
+        raise DocumentError(f'{self.label} contains disallowed entities.')
+
+
+class _Package:
+    """An open OOXML zip: the parts it holds and a guarded parser that charges every part to one XML budget."""
+
+    def __init__(self, archive, entries, label):
+        self.archive = archive
+        self.entries = entries
+        self.label = label
+        self.xml_bytes = 0
+
+    def parse(self, name):
+        info = self.entries[name]
+        if info.flag_bits & 1 or info.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
+            raise DocumentError(f'Encrypted {self.label} or unsupported compression.')
+        if info.file_size > MAX_XML_BYTES - self.xml_bytes:
+            raise DocumentError(f'{self.label} XML content over the limit.')
+        with self.archive.open(info) as stream:
+            raw = stream.read(MAX_XML_BYTES - self.xml_bytes + 1)
+        self.xml_bytes += len(raw)
+        if self.xml_bytes > MAX_XML_BYTES or b'<!DOCTYPE' in raw.upper() or b'<!ENTITY' in raw.upper():
+            raise DocumentError(f'{self.label} contains excessive XML or disallowed entities.')
+        builder = _DocumentTreeBuilder()
+        builder.label = self.label
+        return ET.fromstring(raw, parser=ET.XMLParser(target=builder))
+
+
+@contextlib.contextmanager
+def _open_package(data, label, required):
+    """The one place that opens an OOXML zip (docx, pptx, xlsx) with the guards: archive size, entry count, a required
+    part, no encryption, one XML budget, no DOCTYPE or ENTITY, and every low-level failure turned into a DocumentError.
+
+    Parameters: data (zip bytes), label (`DOCX`, `PPTX` or `XLSX`, used in the reasons), required (a part name, or a
+    predicate over the part names, that must exist). Yields: a _Package. Raises: DocumentError with a readable reason."""
+    if len(data) > MAX_ARCHIVE_BYTES:
+        raise DocumentError(f'{label} over the 10 MiB limit.')
+    try:
+        with zipfile.ZipFile(io.BytesIO(data)) as archive:
+            listing = archive.infolist()
+            entries = {info.filename: info for info in listing}
+            present = required(entries) if callable(required) else required in entries
+            if len(listing) > 2000 or not present:
+                raise DocumentError(f'Invalid {label} or excessive structure.')
+            yield _Package(archive, entries, label)
+    except (zipfile.BadZipFile, ET.ParseError, RuntimeError, NotImplementedError, OSError, zlib.error, EOFError) as exc:
+        raise DocumentError(f'Could not extract text from this {label}.') from exc
+
+
+class _Lines(list):
+    """Output lines of an extractor; `add` refuses to grow past MAX_TEXT_BYTES of UTF-8 text."""
+
+    def __init__(self, label):
+        super().__init__()
+        self.label = label
+        self.size = 0
+
+    def add(self, *lines):
+        for line in lines:
+            self.size += len(line.encode('utf-8')) + 1
+            if self.size > MAX_TEXT_BYTES:
+                raise DocumentError(f'Text extracted from {self.label} exceeds 1 MiB.')
+            self.append(line)
+
+
+def _local(tag):
+    return tag.rsplit('}', 1)[-1]
 
 
 def extract_docx(data):
-    if len(data)>MAX_ARCHIVE_BYTES:
-        raise DocumentError('DOCX over the 10 MiB limit.')
-    try:
-        with zipfile.ZipFile(io.BytesIO(data)) as archive:
-            listing=archive.infolist()
-            entries={info.filename:info for info in listing}
-            if len(listing)>2000 or 'word/document.xml' not in entries:
-                raise DocumentError('Invalid DOCX or excessive structure.')
-            names=['word/document.xml']+sorted(name for name in entries if re.fullmatch(r'word/(?:header\d+|footer\d+|footnotes|endnotes)\.xml',name))
-            paragraphs=[];xml_bytes=0;text_bytes=0
-            for name in names:
-                info=entries[name]
-                if info.flag_bits&1 or info.compress_type not in (zipfile.ZIP_STORED,zipfile.ZIP_DEFLATED):
-                    raise DocumentError('Encrypted DOCX or unsupported compression.')
-                if info.file_size>MAX_XML_BYTES-xml_bytes:
-                    raise DocumentError('DOCX XML content over the limit.')
-                with archive.open(info) as stream:
-                    raw=stream.read(MAX_XML_BYTES-xml_bytes+1)
-                xml_bytes+=len(raw)
-                if xml_bytes>MAX_XML_BYTES or b'<!DOCTYPE' in raw.upper() or b'<!ENTITY' in raw.upper():
-                    raise DocumentError('DOCX contains excessive XML or disallowed entities.')
-                root=ET.fromstring(raw,parser=ET.XMLParser(target=_DocumentTreeBuilder()))
-                namespace=W if root.tag.startswith(W) else '{http://purl.oclc.org/ooxml/wordprocessingml/main}'
-                for paragraph in root.iter(namespace+'p'):
-                    def parts(node):
-                        pending=list(reversed(node))
-                        while pending:
-                            child=pending.pop()
-                            if child.tag==namespace+'p':continue
-                            if child.tag==namespace+'t':yield child.text or ''
-                            elif child.tag==namespace+'tab':yield '\t'
-                            elif child.tag in (namespace+'br',namespace+'cr'):yield '\n'
-                            else:pending.extend(reversed(child))
-                    text=''.join(parts(paragraph))
-                    if text.strip():
-                        text_bytes+=len(text.encode('utf-8'))+1
-                        if text_bytes>MAX_TEXT_BYTES:raise DocumentError('Text extracted from DOCX exceeds 1 MiB.')
-                        paragraphs.append(text)
-            return ('\n'.join(paragraphs)+'\n' if paragraphs else '').encode('utf-8')
-    except (zipfile.BadZipFile,ET.ParseError,RuntimeError,NotImplementedError,OSError) as exc:
-        raise DocumentError('Could not extract text from this DOCX.') from exc
+    """Text of a .docx: body, headers, footers, footnotes and endnotes, one line per non-empty paragraph.
+
+    Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
+    corrupt, encrypted or oversized package, or one with a DOCTYPE or entity."""
+    with _open_package(data, 'DOCX', 'word/document.xml') as package:
+        names = ['word/document.xml'] + sorted(name for name in package.entries if re.fullmatch(
+            r'word/(?:header\d+|footer\d+|footnotes|endnotes)\.xml', name))
+        paragraphs = _Lines('DOCX')
+        for name in names:
+            root = package.parse(name)
+            namespace = W if root.tag.startswith(W) else '{http://purl.oclc.org/ooxml/wordprocessingml/main}'
+            for paragraph in root.iter(namespace + 'p'):
+                def parts(node):
+                    pending = list(reversed(node))
+                    while pending:
+                        child = pending.pop()
+                        if child.tag == namespace + 'p':
+                            continue
+                        if child.tag == namespace + 't':
+                            yield child.text or ''
+                        elif child.tag == namespace + 'tab':
+                            yield '\t'
+                        elif child.tag in (namespace + 'br', namespace + 'cr'):
+                            yield '\n'
+                        else:
+                            pending.extend(reversed(child))
+                text = ''.join(parts(paragraph))
+                if text.strip():
+                    paragraphs.add(text)
+        return ('\n'.join(paragraphs) + '\n' if paragraphs else '').encode('utf-8')
+
+
+def _drawing_paragraphs(root):
+    """Non-empty paragraphs (`a:p`) of a slide or notes part, from the `a:t` runs; the slide-number field is not text."""
+    lines = []
+    for paragraph in root.iter():
+        if _local(paragraph.tag) != 'p':
+            continue
+        pending = list(reversed(paragraph))
+        pieces = []
+        while pending:
+            child = pending.pop()
+            kind = _local(child.tag)
+            if kind == 'fld' and child.get('type', '').startswith('slidenum'):
+                continue
+            if kind == 't':
+                pieces.append(child.text or '')
+            elif kind == 'br':
+                pieces.append('\n')
+            elif kind != 'p':
+                pending.extend(reversed(child))
+        text = ''.join(pieces)
+        if text.strip():
+            lines.append(text)
+    return lines
+
+
+def extract_pptx(data):
+    """Text of a .pptx: each slide under `[slide N]`, then its speaker notes under `[notes N]`, in slide order.
+
+    Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
+    corrupt, encrypted, oversized or hostile package, or a deck without any text."""
+    def has_slides(names):
+        return any(re.fullmatch(r'ppt/slides/slide\d+\.xml', name) for name in names)
+
+    with _open_package(data, 'PPTX', has_slides) as package:
+        slides = sorted((int(m.group(1)), name) for name in package.entries
+                        if (m := re.fullmatch(r'ppt/slides/slide(\d+)\.xml', name)))
+        lines = _Lines('PPTX')
+        for number, name in slides:
+            body = _drawing_paragraphs(package.parse(name))
+            if body:
+                lines.add(f'[slide {number}]', *body)
+            notes_name = f'ppt/notesSlides/notesSlide{number}.xml'
+            notes = _drawing_paragraphs(package.parse(notes_name)) if notes_name in package.entries else []
+            if notes:
+                lines.add(f'[notes {number}]', *notes)
+    if not lines:
+        raise DocumentError('PPTX has no text.')
+    return ('\n'.join(lines) + '\n').encode('utf-8')
+
+
+def _shared_strings(package):
+    if 'xl/sharedStrings.xml' not in package.entries:
+        return []
+    strings = []
+    for item in package.parse('xl/sharedStrings.xml'):
+        if _local(item.tag) != 'si':
+            continue
+        pending = list(reversed(item))
+        pieces = []
+        while pending:
+            child = pending.pop()
+            kind = _local(child.tag)
+            if kind == 't':
+                pieces.append(child.text or '')
+            elif kind != 'rPh':
+                pending.extend(reversed(child))
+        strings.append(''.join(pieces))
+    return strings
+
+
+def _column_index(ref):
+    letters = re.match(r'[A-Za-z]+', ref or '')
+    if not letters:
+        return None
+    index = 0
+    for char in letters.group().upper():
+        index = index * 26 + ord(char) - 64
+    return index - 1
+
+
+def _sheet_rows(root, strings):
+    """Tab-joined text of each non-empty row of a worksheet; an empty cell keeps the columns after it aligned."""
+    for row in root.iter():
+        if _local(row.tag) != 'row':
+            continue
+        values = []
+        for cell in row:
+            if _local(cell.tag) != 'c':
+                continue
+            kind = cell.get('t')
+            value = ''
+            if kind == 'inlineStr':
+                value = ''.join(node.text or '' for node in cell.iter() if _local(node.tag) == 't')
+            else:
+                raw = next((node.text or '' for node in cell if _local(node.tag) == 'v'), '')
+                if kind == 's':
+                    position = int(raw) if raw.strip().isdigit() else -1
+                    if not 0 <= position < len(strings):
+                        raise DocumentError('Could not extract text from this XLSX.')
+                    value = strings[position]
+                else:
+                    value = raw
+            column = _column_index(cell.get('r'))
+            if column is None:
+                column = len(values)
+            if column >= XLSX_MAX_COLUMNS:
+                raise DocumentError(f'XLSX over {XLSX_MAX_COLUMNS} columns.')
+            values += [''] * (column - len(values))
+            if column == len(values):
+                values.append(value)
+            else:
+                values[column] = value
+        if any(value.strip() for value in values):
+            yield '\t'.join(values)
+
+
+def extract_xlsx(data):
+    """Text of an .xlsx: each sheet, in workbook order, under `[sheet NAME]`, then one line per row with the cells
+    separated by tabs. Shared strings are resolved; numbers and inline strings are kept; a formula contributes its cached
+    value, nothing is calculated.
+
+    Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
+    corrupt, encrypted, oversized or hostile package, a workbook over XLSX_MAX_ROWS rows or MAX_TEXT_BYTES of text, or
+    one without any text."""
+    with _open_package(data, 'XLSX', 'xl/workbook.xml') as package:
+        workbook = package.parse('xl/workbook.xml')
+        targets = {}
+        if 'xl/_rels/workbook.xml.rels' in package.entries:
+            for rel in package.parse('xl/_rels/workbook.xml.rels'):
+                target = rel.get('Target', '')
+                targets[rel.get('Id')] = posixpath.normpath(target.lstrip('/') if target.startswith('/')
+                                                            else posixpath.join('xl', target))
+        strings = _shared_strings(package)
+        lines = _Lines('XLSX')
+        rows = 0
+        for sheet in (node for node in workbook.iter() if _local(node.tag) == 'sheet'):
+            rid = next((value for key, value in sheet.attrib.items() if _local(key) == 'id'), None)
+            part = targets.get(rid)
+            if part not in package.entries:
+                raise DocumentError('Invalid XLSX or excessive structure.')
+            header = f"[sheet {' '.join(sheet.get('name', '').split())}]"
+            for line in _sheet_rows(package.parse(part), strings):
+                rows += 1
+                if rows > XLSX_MAX_ROWS:
+                    raise DocumentError(f'XLSX over {XLSX_MAX_ROWS} rows.')
+                if header:
+                    lines.add(header)
+                    header = ''
+                lines.add(line)
+    if not lines:
+        raise DocumentError('XLSX has no text.')
+    return ('\n'.join(lines) + '\n').encode('utf-8')
 
 
 PDF_SCRIPT = """
@@ -150,11 +366,15 @@ def extract_pdf(data):
     return (text + "\n").encode("utf-8", "replace")
 
 
+_EXTENSIONS = ('.docx', '.pptx', '.xlsx', '.pdf')
+
+
 def is_extracted(path):
     """Whether the file type goes through a text extractor before indexing.
 
-    Parameters: path (any file name or path). Returns: True for `.docx` and `.pdf`, case-insensitive."""
-    return str(path).lower().endswith(('.docx', '.pdf'))
+    Parameters: path (any file name or path). Returns: True for `.docx`, `.pptx`, `.xlsx` and `.pdf`, case-insensitive;
+    macro and binary formats (`.xlsm`, `.xlsb`, `.xls`, `.ppt`, `.pptm`) are not extracted."""
+    return str(path).lower().endswith(_EXTENSIONS)
 
 
 def size_limit(path):
@@ -169,7 +389,14 @@ def extract(path, data):
 
     Parameters: path (decides the type, see is_extracted), data (file bytes). Returns: UTF-8 text bytes. Raises:
     DocumentError with a readable reason."""
-    return extract_pdf(data) if str(path).lower().endswith('.pdf') else extract_docx(data)
+    name = str(path).lower()
+    if name.endswith('.pdf'):
+        return extract_pdf(data)
+    if name.endswith('.pptx'):
+        return extract_pptx(data)
+    if name.endswith('.xlsx'):
+        return extract_xlsx(data)
+    return extract_docx(data)
 
 
 def location_kind(path):

package/freshness.py

@@ -73,11 +73,13 @@ def npm_registry_evidence(package):
 
 
 NPM_NAME_RE = re.compile(r"(?:@[a-z0-9._-]+/)?[a-z0-9._-]+")
+NAME_RES = {"npm": NPM_NAME_RE, "PyPI": re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*"),
+            "Go": re.compile(r"[A-Za-z0-9][A-Za-z0-9._~/-]*")}
 
 
-def _osv_query(package, package_version):
-    """One OSV query object: the package (npm) and the version asked about."""
-    return {"package": {"name": package, "ecosystem": "npm"}, "version": package_version}
+def _osv_query(ecosystem, package, package_version):
+    """One OSV query object: the package in `ecosystem` and the version asked about."""
+    return {"package": {"name": package, "ecosystem": ecosystem}, "version": package_version}
 
 
 def _osv_request(endpoint, payload):
@@ -107,7 +109,7 @@ def npm_advisories(package, package_version):
     """IDs conhecidos pela OSV para a versão; None significa checagem indisponível."""
     if not package or not package_version or not NPM_NAME_RE.fullmatch(package):
         return None
-    request = _osv_request("query", _osv_query(package, package_version))
+    request = _osv_request("query", _osv_query("npm", package, package_version))
     try:
         with urllib.request.urlopen(request, timeout=7) as response:
             return _osv_ids(json.load(response))
@@ -115,17 +117,21 @@ def npm_advisories(package, package_version):
         return None
 
 
-def npm_advisories_batch(packages, timeout):
+def advisories_batch(ecosystem, packages, timeout):
     """Advisory IDs for each (package, version) of `packages` from one OSV `querybatch` call.
 
-    Parameters: packages (list of (name, version)); timeout (float) seconds for the request.
+    Parameters: ecosystem (str) "npm", "PyPI" or "Go"; packages (list of (name, version)); timeout (float) seconds for
+    the request.
     Returns: list in the same order, each the sorted ID list or None when that package's answer is incomplete
     (next_page_token) or malformed, so it is unverified and not clean.
-    Raises: RuntimeError when the service cannot be reached, answers badly or answers a different number of results.
+    Raises: ValueError for an unknown ecosystem or an invalid name or version; RuntimeError when the service cannot be
+    reached, answers badly or answers a different number of results.
     """
-    if any(not NPM_NAME_RE.fullmatch(name) or not package_version for name, package_version in packages):
-        raise ValueError(f"Invalid npm package query: {packages!r}")
-    request = _osv_request("querybatch", {"queries": [_osv_query(name, package_version)
+    if ecosystem not in NAME_RES:
+        raise ValueError(f"Unknown OSV ecosystem: {ecosystem!r}")
+    if any(not NAME_RES[ecosystem].fullmatch(name) or not package_version for name, package_version in packages):
+        raise ValueError(f"Invalid {ecosystem} package query: {packages!r}")
+    request = _osv_request("querybatch", {"queries": [_osv_query(ecosystem, name, package_version)
                                                       for name, package_version in packages]})
     try:
         with urllib.request.urlopen(request, timeout=timeout) as response:

package/hook_decision.py

@@ -230,20 +230,21 @@ def _edit_review(payload, doc_mode, duplicate_mode, dependency_mode):
     """Checks of an edit before it runs. Docstrings follow doc_mode, in code files of registered projects only (tests,
     docs and files outside a project are left out, as in the docs coverage): missing ones deny the edit in require mode and become a reminder in remind mode; a
     documented function whose signature changed always gets a reminder. Copies of indexed functions follow
-    duplicate_mode and only add context, never deny. An edit of a package.json follows dependency_mode: advisories
+    duplicate_mode and only add context, never deny. An edit of a package.json, requirements*.txt or go.mod follows dependency_mode: advisories
     of the added or changed dependencies are added as context, never a denial."""
     with code_graph.pending_scope():
         return _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode)
 
 
-def _dependency_part(before, after):
-    """The dependency notice of one package.json edit, or the "dependency check skipped" line when it cannot run.
+def _dependency_part(before, after, ecosystem):
+    """The dependency notice of one manifest edit, or the "dependency check skipped" line when it cannot run.
 
-    Parameters: before (str or None) text before the edit; after (str) text after it.
+    Parameters: before (str or None) text before the edit; after (str) text after it; ecosystem (str) from
+    `dependency_notice.ecosystem_of`.
     Returns: str, empty when there is nothing to say.
     """
     try:
-        return dependency_notice.notice(before, after)
+        return dependency_notice.notice(before, after, ecosystem)
     except dependency_notice.DependencyCheckError as exc:
         return f"{dependency_notice.PREFIX}dependency check skipped: {exc}"
 
@@ -252,8 +253,9 @@ def _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode):
     """`_edit_review` inside one `code_graph.pending_scope`, so the pending-changes scan runs once per review."""
     doc_findings, duplicate_findings, failures, dependency_parts = [], [], [], []
     for path, before, after in edit_preview.preview(payload.get("tool_name"), payload.get("tool_input"), payload.get("cwd")):
-        if dependency_mode != "off" and os.path.basename(path) == "package.json":
-            dependency_parts += [part for part in (_dependency_part(before, after),) if part]
+        ecosystem = dependency_notice.ecosystem_of(path)
+        if dependency_mode != "off" and ecosystem:
+            dependency_parts += [part for part in (_dependency_part(before, after, ecosystem),) if part]
             continue
         if not doc_check.supported(path):
             continue

package/hooks/session_audit.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""SessionStart hook of Claude Code (opt-in, from the setup screen): forwards the payload to the Smart Tool daemon
+"""SessionStart hook of Claude Code and Codex (opt-in, from the setup screen): forwards the payload to the Smart Tool daemon
 (POST /hooks/session-audit) and prints its answer, the stored setup audit notice of the session folder as additional
 context. Never starts the daemon or an audit: daemon down or any error means empty output, so a session start is
 never delayed or blocked.

package/index_profile.py

@@ -11,7 +11,7 @@ PROFILE_VERSION = 2
 KINDS = ("code", "test", "doc")
 RETRY_AFTER_S = 3600
 MAX_DIRS = 90
-PROSE = {"md", "txt", "docx", "pdf", "rst", "adoc"}
+PROSE = {"md", "txt", "docx", "pptx", "xlsx", "pdf", "rst", "adoc"}
 # Data and markup inside a documentation, design or data-dump folder are reports and mockups, not code (GitHub Linguist
 # treats docs/ as documentation; a design/ HTML or a probe's JSON result pushed hook_decision.py out of the code block).
 DOC_DATA = {"html", "htm", "json", "csv", "xml", "yaml", "yml", "svg"}

package/index_scope.py

@@ -40,10 +40,13 @@ _PROJECT_TYPE_EXCLUDES = (
     (("Podfile",), ("Pods",)),
 )
 
+# Files a tool writes while it works and replaces or deletes right after (atomic-write temp, backup, editor swap).
+TRANSIENT_FILES = {"*.tmp", "*.bak", "*.swp", "*~"}
+
 # Mesmo critério do `_ALWAYS_EXCLUDE`, para arquivo: o significado não varia por
 # projeto. Lockfile é grafo de dependência resolvido por ferramenta — nunca é fonte de
 # resposta numa busca semântica, e é grande o bastante pra dominar o índice.
-_ALWAYS_EXCLUDE_FILES = {
+_ALWAYS_EXCLUDE_FILES = TRANSIENT_FILES | {
     "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "npm-shrinkwrap.json",
     "poetry.lock", "pdm.lock", "uv.lock", "Pipfile.lock", "Cargo.lock",
     "composer.lock", "go.sum", "Gemfile.lock", "packages.lock.json",
@@ -110,6 +113,11 @@ def normalize_include(include):
     return bases, dropped
 
 
+def is_transient(name):
+    """Whether a file name is a temporary, backup or editor-swap file (`*.tmp`, `*.bak`, `*.swp`, `*~`)."""
+    return any(fnmatch.fnmatch(name.casefold(), pattern) for pattern in TRANSIENT_FILES)
+
+
 def always_excluded(name):
     """Directory name in _ALWAYS_EXCLUDE, entries may be globs (e.g. *.egg-info)."""
     folded = name.casefold()

package/indexer.py

@@ -31,8 +31,10 @@ CHUNK_MAX_CHARS = 8000
 CHUNK_OVERLAP_LINES = 12
 CHUNKER_VERSION = 3
 SCHEMA_VERSION = 2
-REINDEX_BATCH_MAX_CHARS = 32_000
-REINDEX_BATCH_MAX_ITEMS = 32
+# An embeddings request holds up to 128 items and 120,000 characters (about 30,000 tokens): far under the 2048 items and
+# 300,000 tokens per request that OpenAI and Azure accept (docs/research/2026-10-27-excesso-de-chamadas-embed-rerank.md).
+REINDEX_BATCH_MAX_CHARS = 120_000
+REINDEX_BATCH_MAX_ITEMS = 128
 # Embedding requests in flight per flush. Measured 2026-10-03 (docs/spikes/embed_parallel_spike.py, 30 batches,
 # 3 interleaved rounds): 1 worker 33-50 s, 2 workers 19-27 s, 4 workers 11-20 s (~2.8x), no 429 from the gateway.
 EMBED_WORKERS = 4

package/model_breaker.py

@@ -0,0 +1,70 @@
+"""Protection of the model gateway, per model id: a circuit that opens after consecutive 429/503 answers and a cap on
+the requests in flight.
+
+Pattern: a circuit breaker (closed, open for a pause, then one trial call decides). Retrying a gateway that answers
+busy only adds load (AWS Bedrock throughput guide: honor Retry-After, back off with jitter); once it keeps answering
+busy the caller stops asking for a while and serves what it can without the model.
+"""
+import contextlib
+import threading
+import time
+import urllib.error
+
+FAILURES_TO_OPEN = 3
+PAUSE_S = 30
+MAX_IN_FLIGHT = 6
+_BUSY_CODES = (429, 503)
+_lock = threading.Lock()
+_failures = {}
+_open_until = {}
+_slots = {}
+
+
+class CircuitOpen(RuntimeError):
+    """The model is paused after repeated busy answers; no request was sent."""
+
+
+def guard(model):
+    """Raises CircuitOpen while `model` is paused. Nothing happens for a call without a model id."""
+    if not model:
+        return
+    with _lock:
+        left = _open_until.get(model, 0) - time.monotonic()
+    if left > 0:
+        raise CircuitOpen(f"Model {model} is paused for {int(left) + 1} s after {FAILURES_TO_OPEN} consecutive "
+                          "HTTP 429/503 answers from the gateway; no request was sent.")
+
+
+def note(model, failure=None):
+    """Records the outcome of a request: a success closes the circuit, a 429/503 counts toward opening it, any other
+    failure changes nothing. After the pause the first failure opens it again."""
+    if not model:
+        return
+    busy = isinstance(failure, urllib.error.HTTPError) and failure.code in _BUSY_CODES
+    with _lock:
+        if failure is None:
+            _failures[model] = 0
+        elif busy:
+            _failures[model] = _failures.get(model, 0) + 1
+            if _failures[model] >= FAILURES_TO_OPEN:
+                _open_until[model] = time.monotonic() + PAUSE_S
+
+
+@contextlib.contextmanager
+def slot(model):
+    """Holds one of the `MAX_IN_FLIGHT` places of `model` for the duration of a request."""
+    if not model:
+        yield
+        return
+    with _lock:
+        semaphore = _slots.setdefault(model, threading.BoundedSemaphore(MAX_IN_FLIGHT))
+    with semaphore:
+        yield
+
+
+def reset():
+    """Forgets every circuit and cap (tests)."""
+    with _lock:
+        _failures.clear()
+        _open_until.clear()
+        _slots.clear()

package/model_client.py

@@ -9,6 +9,7 @@ import usage_meter
 
 import config
 import gateway
+import model_breaker
 import model_health
 
 OFFLINE = contextvars.ContextVar("model_offline", default=None)
@@ -26,18 +27,24 @@ def get_token():
 
 
 def fetch(path, token, method="GET", body=None, timeout=15):
+    """One gateway request; returns the parsed JSON. A call that names a model first passes that model's circuit and
+    request cap (model_breaker, CircuitOpen when paused), and every attempt is metered. HTTP errors propagate."""
     if OFFLINE.get():
         raise RuntimeError(OFFLINE.get())
-    started=time.monotonic();result=None;headers=None;failure=None
-    try:
-        result,headers=_fetch(path,token,method,body,timeout)
-        return result
-    except Exception as exc:
-        failure=exc
-        raise
-    finally:
-        usage_meter.record(path,body,result,headers,failure,time.monotonic()-started)
-        model_health.record(path,body,failure)
+    model=body.get("model") if isinstance(body,dict) else None
+    model_breaker.guard(model)
+    with model_breaker.slot(model):
+        started=time.monotonic();result=None;headers=None;failure=None
+        try:
+            result,headers=_fetch(path,token,method,body,timeout)
+            return result
+        except Exception as exc:
+            failure=exc
+            raise
+        finally:
+            model_breaker.note(model,failure)
+            usage_meter.record(path,body,result,headers,failure,time.monotonic()-started)
+            model_health.record(path,body,failure)
 
 
 def _fetch(path, _token, method="GET", body=None, timeout=15):