bps-kit 1.0.1 → 1.0.2

package/templates/vault/junta-leiloeiros/scripts/run_all.py

@@ -0,0 +1,190 @@
+"""
+Orquestrador de scraping — coleta dados de todas as 27 Juntas Comerciais do Brasil.
+
+Uso:
+    python scripts/run_all.py                      # todos os estados
+    python scripts/run_all.py --estado SP RJ MG    # estados específicos
+    python scripts/run_all.py --concurrency 5      # paralelismo (default: 5)
+    python scripts/run_all.py --dry-run            # mostra o que seria coletado
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List, Optional
+
+# Ajusta PYTHONPATH para imports relativos funcionarem
+sys.path.insert(0, str(Path(__file__).parent))
+
+from scraper.states import SCRAPERS, get_all_scrapers, get_scraper
+from db import Database
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    datefmt="%H:%M:%S",
+)
+logger = logging.getLogger(__name__)
+
+LOG_FILE = Path(__file__).parent.parent / "data" / "scraping_log.json"
+
+
+async def scrape_state(estado: str, semaphore: asyncio.Semaphore) -> dict:
+    """Scrapa um estado e retorna resultado com metadados."""
+    async with semaphore:
+        scraper = get_scraper(estado)
+        if not scraper:
+            return {
+                "estado": estado,
+                "status": "SCRAPER_NAO_ENCONTRADO",
+                "count": 0,
+                "records": [],
+                "error": None,
+                "scraped_at": datetime.now(timezone.utc).isoformat(),
+            }
+
+        try:
+            records = await scraper.scrape()
+            return {
+                "estado": estado,
+                "junta": scraper.junta,
+                "url": scraper.url,
+                "status": "OK" if records else "VAZIO",
+                "count": len(records),
+                "records": [r.to_dict() for r in records],
+                "error": None,
+                "scraped_at": datetime.now(timezone.utc).isoformat(),
+            }
+        except Exception as exc:
+            logger.exception("[%s] Erro no scraping: %s", estado, exc)
+            return {
+                "estado": estado,
+                "status": "ERRO",
+                "count": 0,
+                "records": [],
+                "error": str(exc),
+                "scraped_at": datetime.now(timezone.utc).isoformat(),
+            }
+
+
+async def run(estados: Optional[List[str]], concurrency: int, dry_run: bool) -> None:
+    estados_alvo = [e.upper() for e in estados] if estados else list(SCRAPERS.keys())
+
+    if dry_run:
+        print(f"\n[DRY-RUN] Estados que seriam coletados ({len(estados_alvo)}):")
+        for uf in estados_alvo:
+            s = get_scraper(uf)
+            if s:
+                print(f"  {uf}: {s.junta} -> {s.url}")
+            else:
+                print(f"  {uf}: scraper nao encontrado")
+        return
+
+    logger.info("Iniciando coleta de %d estados (concurrency=%d)", len(estados_alvo), concurrency)
+
+    semaphore = asyncio.Semaphore(concurrency)
+    tasks = [scrape_state(uf, semaphore) for uf in estados_alvo]
+    results = await asyncio.gather(*tasks)
+
+    # Persistir no banco
+    db = Database()
+    db.init()
+
+    total_coletados = 0
+    total_salvos = 0
+    log_entries = []
+
+    for res in results:
+        estado = res["estado"]
+        status = res["status"]
+        count = res["count"]
+        total_coletados += count
+
+        if res["records"]:
+            saved = db.upsert_many(res["records"])
+            total_salvos += saved
+            logger.info("[%s] %d leiloeiros coletados, %d salvos", estado, count, saved)
+        else:
+            if status == "OK":
+                status = "VAZIO"
+            logger.warning("[%s] STATUS=%s error=%s", estado, status, res.get("error"))
+
+        log_entries.append({
+            "estado": estado,
+            "junta": res.get("junta", "?"),
+            "url": res.get("url", "?"),
+            "status": status,
+            "count": count,
+            "error": res.get("error"),
+            "scraped_at": res["scraped_at"],
+        })
+
+    # Salvar log
+    LOG_FILE.parent.mkdir(parents=True, exist_ok=True)
+    with open(LOG_FILE, "w", encoding="utf-8") as f:
+        json.dump(
+            {
+                "run_at": datetime.now(timezone.utc).isoformat(),
+                "total_estados": len(estados_alvo),
+                "total_coletados": total_coletados,
+                "total_salvos": total_salvos,
+                "estados": log_entries,
+            },
+            f,
+            ensure_ascii=False,
+            indent=2,
+        )
+
+    # Resumo final
+    print("\n" + "=" * 60)
+    print(f"RESUMO DA COLETA")
+    print("=" * 60)
+    ok = [e for e in log_entries if e["status"] == "OK"]
+    vazios = [e for e in log_entries if e["status"] == "VAZIO"]
+    erros = [e for e in log_entries if e["status"] not in ("OK", "VAZIO")]
+
+    print(f"  OK:     {len(ok)} estados | {total_coletados} leiloeiros coletados")
+    print(f"  VAZIO:  {len(vazios)} estados | {[e['estado'] for e in vazios]}")
+    print(f"  ERRO:   {len(erros)} estados | {[e['estado'] for e in erros]}")
+    print(f"  TOTAL SALVO NO BANCO: {total_salvos}")
+    print(f"  LOG: {LOG_FILE}")
+    print("=" * 60)
+
+    # Estatísticas do banco
+    stats = db.get_stats()
+    if stats:
+        print("\nESTATÍSTICAS POR ESTADO:")
+        print(f"{'UF':<5} {'Junta':<12} {'Total':>6} {'Ativos':>7}")
+        print("-" * 35)
+        for s in stats:
+            print(f"{s['estado']:<5} {s['junta']:<12} {s['total']:>6} {s['ativos']:>7}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Coleta dados de leiloeiros de todas as Juntas Comerciais do Brasil"
+    )
+    parser.add_argument(
+        "--estado", nargs="*", metavar="UF",
+        help="Estados específicos (ex: SP RJ MG). Padrão: todos os 27."
+    )
+    parser.add_argument(
+        "--concurrency", type=int, default=5,
+        help="Número de scrapers em paralelo (default: 5)"
+    )
+    parser.add_argument(
+        "--dry-run", action="store_true",
+        help="Mostra o que seria coletado sem executar"
+    )
+    args = parser.parse_args()
+
+    asyncio.run(run(args.estado, args.concurrency, args.dry_run))
+
+
+if __name__ == "__main__":
+    main()

package/templates/vault/junta-leiloeiros/scripts/scraper/__init__.py

@@ -0,0 +1,4 @@
+"""Scrapers de Juntas Comerciais do Brasil."""
+from .base_scraper import AbstractJuntaScraper, Leiloeiro
+
+__all__ = ["AbstractJuntaScraper", "Leiloeiro"]

package/templates/vault/junta-leiloeiros/scripts/scraper/base_scraper.py

@@ -0,0 +1,209 @@
+"""
+Base abstrata para scrapers de leiloeiros das Juntas Comerciais do Brasil.
+Cada estado herda desta classe e implementa parse_leiloeiros().
+Suporta httpx (sites estáticos) e Playwright (sites com JavaScript).
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import List, Optional
+
+import httpx
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Leiloeiro:
+    estado: str
+    junta: str
+    nome: str
+    matricula: Optional[str] = None
+    cpf_cnpj: Optional[str] = None
+    situacao: Optional[str] = None
+    endereco: Optional[str] = None
+    municipio: Optional[str] = None
+    telefone: Optional[str] = None
+    email: Optional[str] = None
+    data_registro: Optional[str] = None
+    data_atualizacao: Optional[str] = None
+    url_fonte: Optional[str] = None
+    scraped_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+
+    def to_dict(self) -> dict:
+        return {
+            "estado": self.estado,
+            "junta": self.junta,
+            "matricula": self.matricula,
+            "nome": self.nome,
+            "cpf_cnpj": self.cpf_cnpj,
+            "situacao": self.situacao,
+            "endereco": self.endereco,
+            "municipio": self.municipio,
+            "telefone": self.telefone,
+            "email": self.email,
+            "data_registro": self.data_registro,
+            "data_atualizacao": self.data_atualizacao,
+            "url_fonte": self.url_fonte,
+            "scraped_at": self.scraped_at,
+        }
+
+
+class AbstractJuntaScraper(ABC):
+    """Classe base para todos os scrapers de Juntas Comerciais."""
+
+    estado: str        # UF ex: "SP"
+    junta: str         # nome da junta ex: "JUCESP"
+    url: str           # URL da página de leiloeiros
+    rate_limit: float = 2.0   # segundos entre requests
+    max_retries: int = 3
+    timeout: float = 30.0
+
+    HEADERS = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/121.0.0.0 Safari/537.36"
+        ),
+        "Accept-Language": "pt-BR,pt;q=0.9",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    }
+
+    async def fetch_page(
+        self,
+        url: Optional[str] = None,
+        params: Optional[dict] = None,
+        data: Optional[dict] = None,
+        method: str = "GET",
+    ) -> Optional[BeautifulSoup]:
+        """Faz o request HTTP com retry e retorna BeautifulSoup ou None."""
+        target = url or self.url
+        for attempt in range(1, self.max_retries + 1):
+            try:
+                async with httpx.AsyncClient(
+                    headers=self.HEADERS,
+                    timeout=self.timeout,
+                    follow_redirects=True,
+                    verify=False,  # alguns sites gov têm cert self-signed
+                ) as client:
+                    if method.upper() == "POST":
+                        resp = await client.post(target, data=data, params=params)
+                    else:
+                        resp = await client.get(target, params=params)
+
+                    resp.raise_for_status()
+                    return BeautifulSoup(resp.text, "lxml")
+
+            except httpx.HTTPStatusError as exc:
+                logger.warning(
+                    "[%s] HTTP %s em %s (tentativa %d/%d)",
+                    self.estado, exc.response.status_code, target, attempt, self.max_retries,
+                )
+            except (httpx.RequestError, httpx.TimeoutException) as exc:
+                logger.warning(
+                    "[%s] Erro de request em %s: %s (tentativa %d/%d)",
+                    self.estado, target, exc, attempt, self.max_retries,
+                )
+
+            if attempt < self.max_retries:
+                await asyncio.sleep(2 ** attempt)  # exponential backoff
+
+        logger.error("[%s] Falha após %d tentativas em %s", self.estado, self.max_retries, target)
+        return None
+
+    @abstractmethod
+    async def parse_leiloeiros(self) -> List[Leiloeiro]:
+        """Coleta e retorna a lista de leiloeiros do estado."""
+        ...
+
+    async def scrape(self) -> List[Leiloeiro]:
+        """Ponto de entrada principal — respeita rate limit e loga resultado."""
+        logger.info("[%s] Iniciando scraping de %s", self.estado, self.url)
+        await asyncio.sleep(self.rate_limit)
+        try:
+            results = await self.parse_leiloeiros()
+            logger.info("[%s] %d leiloeiros coletados", self.estado, len(results))
+            return results
+        except Exception as exc:
+            logger.exception("[%s] Erro inesperado: %s", self.estado, exc)
+            return []
+
+    # ── helpers comuns ──────────────────────────────────────────────────────
+
+    @staticmethod
+    def clean(text: Optional[str]) -> Optional[str]:
+        """Remove espaços extras e retorna None se vazio."""
+        if text is None:
+            return None
+        s = " ".join(text.split()).strip()
+        return s if s else None
+
+    @staticmethod
+    def normalize_situacao(raw: Optional[str]) -> Optional[str]:
+        """Normaliza status para ATIVO / CANCELADO / SUSPENSO / IRREGULAR."""
+        if raw is None:
+            return None
+        r = raw.upper().strip()
+        if any(x in r for x in ("ATIV", "REGULAR", "HABILITAD")):
+            return "ATIVO"
+        if any(x in r for x in ("CANCEL", "BAIXAD", "EXTINT")):
+            return "CANCELADO"
+        if "SUSPEND" in r:
+            return "SUSPENSO"
+        if "IRREG" in r:
+            return "IRREGULAR"
+        return raw.strip()
+
+    def make_leiloeiro(self, **kwargs) -> Leiloeiro:
+        """Factory que preenche estado/junta/url_fonte automaticamente."""
+        kwargs.setdefault("estado", self.estado)
+        kwargs.setdefault("junta", self.junta)
+        kwargs.setdefault("url_fonte", self.url)
+        if "situacao" in kwargs:
+            kwargs["situacao"] = self.normalize_situacao(kwargs["situacao"])
+        return Leiloeiro(**kwargs)
+
+    async def fetch_page_js(
+        self,
+        url: Optional[str] = None,
+        wait_selector: Optional[str] = None,
+        wait_ms: int = 3000,
+    ) -> Optional[BeautifulSoup]:
+        """Renderiza página com JavaScript usando Playwright. Retorna BeautifulSoup ou None."""
+        target = url or self.url
+        try:
+            from playwright.async_api import async_playwright
+        except ImportError:
+            logger.error("[%s] Playwright não instalado. Execute: playwright install chromium", self.estado)
+            return None
+
+        try:
+            async with async_playwright() as pw:
+                browser = await pw.chromium.launch(headless=True)
+                ctx = await browser.new_context(
+                    user_agent=self.HEADERS["User-Agent"],
+                    locale="pt-BR",
+                    ignore_https_errors=True,
+                )
+                page = await ctx.new_page()
+                await page.goto(target, timeout=60000, wait_until="networkidle")
+
+                if wait_selector:
+                    try:
+                        await page.wait_for_selector(wait_selector, timeout=15000)
+                    except Exception:
+                        pass  # Continua mesmo sem o seletor
+                else:
+                    await page.wait_for_timeout(wait_ms)
+
+                html = await page.content()
+                await browser.close()
+                return BeautifulSoup(html, "lxml")
+        except Exception as exc:
+            logger.error("[%s] Erro Playwright em %s: %s", self.estado, target, exc)
+            return None

package/templates/vault/junta-leiloeiros/scripts/scraper/generic_scraper.py

@@ -0,0 +1,110 @@
+"""
+Scraper genérico para juntas que usam formato padrão de tabela HTML.
+Estados sem scraper customizado herdam deste.
+"""
+from __future__ import annotations
+
+from typing import List, Optional
+
+from .base_scraper import AbstractJuntaScraper, Leiloeiro
+
+
+class GenericJuntaScraper(AbstractJuntaScraper):
+    """
+    Scraper genérico para juntas com tabela HTML padrão.
+    Subclasses definem apenas estado, junta e url.
+    """
+
+    estado: str
+    junta: str
+    url: str
+    municipio_default: Optional[str] = None  # para estados com capital única dominante
+
+    async def parse_leiloeiros(self) -> List[Leiloeiro]:
+        soup = await self.fetch_page()
+        if not soup:
+            return []
+
+        results: List[Leiloeiro] = []
+
+        # Tentativa 1: tabela HTML
+        tables = soup.find_all("table")
+        for table in tables:
+            rows = table.find_all("tr")
+            if len(rows) < 2:
+                continue
+
+            headers = [self.clean(th.get_text()) for th in rows[0].find_all(["th", "td"])]
+            if not headers:
+                continue
+
+            col = {(h or "").lower(): i for i, h in enumerate(headers)}
+
+            # Verificar se parece uma tabela de leiloeiros
+            has_name_col = any(
+                "nome" in k or "leiloeiro" in k or "auxiliar" in k
+                for k in col.keys()
+            )
+            if not has_name_col and len(headers) < 2:
+                continue
+
+            def gcol(cells, frags):
+                for k, i in col.items():
+                    if any(f in k for f in frags) and i < len(cells):
+                        return self.clean(cells[i].get_text())
+                return None
+
+            for row in rows[1:]:
+                cells = row.find_all(["td", "th"])
+                if not cells:
+                    continue
+                nome = gcol(cells, ["nome", "leiloeiro"]) or self.clean(cells[0].get_text())
+                if not nome or len(nome) < 3:
+                    continue
+
+                results.append(self.make_leiloeiro(
+                    nome=nome,
+                    matricula=gcol(cells, ["matr", "registro", "núm", "numero", "nº"]),
+                    cpf_cnpj=gcol(cells, ["cpf", "cnpj", "documento"]),
+                    situacao=gcol(cells, ["situ", "status"]),
+                    municipio=gcol(cells, ["munic", "cidade"]) or self.municipio_default,
+                    telefone=gcol(cells, ["tel", "fone", "contato"]),
+                    email=gcol(cells, ["email", "e-mail"]),
+                    endereco=gcol(cells, ["ender", "logr", "rua"]),
+                    data_registro=gcol(cells, ["data", "cadastr"]),
+                ))
+
+            if results:
+                break  # Parar na primeira tabela com resultados
+
+        # Tentativa 2: listas (ul/ol li)
+        if not results:
+            list_items = soup.select("ul.leiloeiros li, ol.leiloeiros li, .lista-leiloeiros li")
+            if not list_items:
+                list_items = soup.select("ul li, ol li")
+
+            for li in list_items:
+                text = self.clean(li.get_text(" | "))
+                if not text or len(text) < 5:
+                    continue
+                results.append(self.make_leiloeiro(nome=text, municipio=self.municipio_default))
+
+        # Tentativa 3: divs/articles com conteúdo textual
+        if not results:
+            content = soup.select_one(
+                ".conteudo-pagina, .page-content, .entry-content, article, main .content"
+            )
+            if content:
+                import re
+                for p in content.find_all(["p", "div", "li"]):
+                    text = self.clean(p.get_text())
+                    if not text or len(text) < 5:
+                        continue
+                    # Filtrar parágrafos que parecem ser registros de pessoas
+                    if re.search(r"\b[A-ZÁÉÍÓÚÀÃÕÇ][a-záéíóúàãõç]{2,}", text):
+                        results.append(self.make_leiloeiro(
+                            nome=text,
+                            municipio=self.municipio_default,
+                        ))
+
+        return results

package/templates/vault/junta-leiloeiros/scripts/scraper/jucap.py

@@ -0,0 +1,110 @@
+"""
+Scraper JUCAP — Junta Comercial do Amapa
+URL: https://jucap.portal.ap.gov.br/pagina/informacoes/leiloleiros
+     (ATENCAO: typo oficial — "leiloleiros" com L extra, URL exata do site)
+Metodo: httpx + BeautifulSoup
+Estrutura: h4/h5 com nome + paragrafos com detalhes (Laravel/Livewire SSR)
+Registros: ~12 leiloeiros
+Nota: www.jucap.ap.gov.br falha por DNS — usar jucap.portal.ap.gov.br
+"""
+from __future__ import annotations
+
+import re
+from typing import List
+
+from .base_scraper import AbstractJuntaScraper, Leiloeiro
+
+
+class JucapScraper(AbstractJuntaScraper):
+    estado = "AP"
+    junta = "JUCAP"
+    url = "https://jucap.portal.ap.gov.br/pagina/informacoes/leiloleiros"
+
+    async def parse_leiloeiros(self) -> List[Leiloeiro]:
+        soup = await self.fetch_page()
+        if not soup:
+            soup = await self.fetch_page(url="https://jucap.portal.ap.gov.br/pagina/informacoes/leiloeiros")
+        if not soup:
+            soup = await self.fetch_page_js(url=self.url, wait_ms=3000)
+        if not soup:
+            return []
+
+        results: List[Leiloeiro] = []
+
+        # Tenta tabela primeiro
+        for table in soup.find_all("table"):
+            rows = table.find_all("tr")
+            if len(rows) < 2:
+                continue
+            headers = [self.clean(th.get_text()) for th in rows[0].find_all(["th", "td"])]
+            col = {(h or "").lower(): i for i, h in enumerate(headers)}
+
+            def gcol(cells, frags):
+                for k, i in col.items():
+                    if any(f in k for f in frags) and i < len(cells):
+                        return self.clean(cells[i].get_text())
+                return None
+
+            for row in rows[1:]:
+                cells = row.find_all(["td", "th"])
+                if not cells:
+                    continue
+                nome = gcol(cells, ["nome", "leiloeiro"]) or self.clean(cells[0].get_text())
+                if not nome or len(nome) < 3:
+                    continue
+                results.append(self.make_leiloeiro(
+                    nome=nome,
+                    matricula=gcol(cells, ["matr", "registro"]),
+                    situacao=gcol(cells, ["situ", "status"]),
+                    municipio=gcol(cells, ["munic", "cidade"]) or "Macapa",
+                    telefone=gcol(cells, ["tel", "fone"]),
+                    email=gcol(cells, ["email"]),
+                    endereco=gcol(cells, ["ender", "logr"]),
+                    data_registro=gcol(cells, ["data", "posse"]),
+                ))
+            if results:
+                return results
+
+        # Estrutura Laravel: h4/h5 como nome + p como detalhes
+        content = soup.select_one("main, .content, article, #content, .page-content")
+        if not content:
+            content = soup.body
+
+        if content:
+            current: dict = {}
+            for el in content.find_all(["h3", "h4", "h5", "p", "li", "strong"]):
+                tag = el.name
+                text = self.clean(el.get_text())
+                if not text:
+                    continue
+
+                if tag in ("h3", "h4", "h5"):
+                    if current.get("nome"):
+                        results.append(self.make_leiloeiro(**current))
+                    current = {"nome": text, "municipio": "Macapa"}
+                elif current.get("nome"):
+                    text_lower = text.lower()
+                    if "matrícula" in text_lower or "matricula" in text_lower:
+                        m = re.search(r"\d+", text)
+                        if m:
+                            current["matricula"] = m.group()
+                    elif re.search(r"\d{4,5}[-\s]\d{4}", text):
+                        current.setdefault("telefone", text)
+                    elif "@" in text:
+                        current["email"] = text
+                    elif any(uf in text for uf in ["/AP", "Macapa", "Macapá", "Santana"]):
+                        current["endereco"] = text
+                    elif re.search(r"ativ|regular|cancel|suspen", text_lower):
+                        current["situacao"] = text
+
+            if current.get("nome"):
+                results.append(self.make_leiloeiro(**current))
+
+        # Fallback texto plano
+        if not results:
+            for line in soup.get_text("\n").split("\n"):
+                line = self.clean(line)
+                if line and len(line) > 5 and re.match(r"^[A-ZÁÉÍÓÚÀÃÕÇ][A-ZÁÉÍÓÚÀÃÕÇ\s]{4,}$", line):
+                    results.append(self.make_leiloeiro(nome=line, municipio="Macapa"))
+
+        return results