bps-kit 1.0.1 → 1.0.2

package/templates/vault/junta-leiloeiros/scripts/scraper/juceac.py

@@ -0,0 +1,72 @@
+"""
+Scraper JUCEAC — Junta Comercial do Estado do Acre
+URL: https://juceac.ac.gov.br/leiloeiro/
+Método: httpx + BeautifulSoup
+Nota: URL correta é /leiloeiro/ (singular), não /leiloeiros/.
+      Lista ~30 leiloeiros com situação (CANCELADO, SUSPENSO, ativo),
+      data de posse, endereço e contatos. Lista atualizada.
+"""
+from __future__ import annotations
+
+from typing import List
+
+from .base_scraper import AbstractJuntaScraper, Leiloeiro
+
+
+class JuceacScraper(AbstractJuntaScraper):
+    estado = "AC"
+    junta = "JUCEAC"
+    url = "https://juceac.ac.gov.br/leiloeiro/"
+
+    async def parse_leiloeiros(self) -> List[Leiloeiro]:
+        soup = await self.fetch_page()
+        if not soup:
+            # Tenta sem trailing slash e com www
+            soup = await self.fetch_page(url="https://www.juceac.ac.gov.br/leiloeiro/")
+        if not soup:
+            soup = await self.fetch_page_js(wait_ms=3000)
+        if not soup:
+            return []
+
+        results: List[Leiloeiro] = []
+
+        for table in soup.find_all("table"):
+            rows = table.find_all("tr")
+            if len(rows) < 2:
+                continue
+            headers = [self.clean(th.get_text()) for th in rows[0].find_all(["th", "td"])]
+            col = {(h or "").lower(): i for i, h in enumerate(headers)}
+
+            def gcol(cells, frags):
+                for k, i in col.items():
+                    if any(f in k for f in frags) and i < len(cells):
+                        return self.clean(cells[i].get_text())
+                return None
+
+            for row in rows[1:]:
+                cells = row.find_all(["td", "th"])
+                if not cells:
+                    continue
+                nome = gcol(cells, ["nome", "leiloeiro"]) or self.clean(cells[0].get_text())
+                if not nome or len(nome) < 3:
+                    continue
+                results.append(self.make_leiloeiro(
+                    nome=nome,
+                    matricula=gcol(cells, ["matr", "registro", "nº"]),
+                    situacao=gcol(cells, ["situ", "status", "cancel", "suspen"]),
+                    municipio=gcol(cells, ["munic", "cidade"]) or "Rio Branco",
+                    telefone=gcol(cells, ["tel", "fone"]),
+                    email=gcol(cells, ["email"]),
+                    endereco=gcol(cells, ["ender", "logr"]),
+                    data_registro=gcol(cells, ["data", "posse"]),
+                ))
+            if results:
+                break
+
+        if not results:
+            for el in soup.select("li, p, .leiloeiro, article"):
+                text = self.clean(el.get_text(" | "))
+                if text and len(text) > 10:
+                    results.append(self.make_leiloeiro(nome=text, municipio="Rio Branco"))
+
+        return results

package/templates/vault/junta-leiloeiros/scripts/scraper/juceal.py

@@ -0,0 +1,72 @@
+"""
+Scraper JUCEAL — Junta Comercial do Estado de Alagoas
+URL: http://www.juceal.al.gov.br/servicos/leiloeiros
+Método: httpx + BeautifulSoup
+Nota: URL /leiloeiros retornava 404 — URL correta é /servicos/leiloeiros.
+      Lista 28 leiloeiros com matrícula, data de posse, situação, contatos e redes sociais.
+      Registros atualizados até 2025.
+"""
+from __future__ import annotations
+
+from typing import List
+
+from .base_scraper import AbstractJuntaScraper, Leiloeiro
+
+
+class JucealScraper(AbstractJuntaScraper):
+    estado = "AL"
+    junta = "JUCEAL"
+    url = "http://www.juceal.al.gov.br/servicos/leiloeiros"
+
+    async def parse_leiloeiros(self) -> List[Leiloeiro]:
+        soup = await self.fetch_page()
+        if not soup:
+            # Tenta HTTPS também
+            soup = await self.fetch_page(url="https://www.juceal.al.gov.br/servicos/leiloeiros")
+        if not soup:
+            soup = await self.fetch_page_js(wait_ms=3000)
+        if not soup:
+            return []
+
+        results: List[Leiloeiro] = []
+
+        for table in soup.find_all("table"):
+            rows = table.find_all("tr")
+            if len(rows) < 2:
+                continue
+            headers = [self.clean(th.get_text()) for th in rows[0].find_all(["th", "td"])]
+            col = {(h or "").lower(): i for i, h in enumerate(headers)}
+
+            def gcol(cells, frags):
+                for k, i in col.items():
+                    if any(f in k for f in frags) and i < len(cells):
+                        return self.clean(cells[i].get_text())
+                return None
+
+            for row in rows[1:]:
+                cells = row.find_all(["td", "th"])
+                if not cells:
+                    continue
+                nome = gcol(cells, ["nome", "leiloeiro"]) or self.clean(cells[0].get_text())
+                if not nome or len(nome) < 3:
+                    continue
+                results.append(self.make_leiloeiro(
+                    nome=nome,
+                    matricula=gcol(cells, ["matr", "registro", "nº"]),
+                    situacao=gcol(cells, ["situ", "status"]),
+                    municipio=gcol(cells, ["munic", "cidade"]) or "Maceió",
+                    telefone=gcol(cells, ["tel", "fone"]),
+                    email=gcol(cells, ["email"]),
+                    endereco=gcol(cells, ["ender", "logr"]),
+                    data_registro=gcol(cells, ["data", "posse"]),
+                ))
+            if results:
+                break
+
+        if not results:
+            for el in soup.select("li, p, .leiloeiro, article"):
+                text = self.clean(el.get_text(" | "))
+                if text and len(text) > 10:
+                    results.append(self.make_leiloeiro(nome=text, municipio="Maceió"))
+
+        return results

package/templates/vault/junta-leiloeiros/scripts/scraper/juceb.py

@@ -0,0 +1,68 @@
+"""
+Scraper JUCEB — Junta Comercial do Estado da Bahia
+URL: https://www.ba.gov.br/juceb/home/matriculas-e-carteira-profissional/leiloeiros
+Método: httpx + BeautifulSoup
+Nota: Site migrou de juceb.ba.gov.br para ba.gov.br/juceb
+      Lista inclui: nome, matrícula, portaria, nomeação, contatos, situação (Regular/Irregular)
+"""
+from __future__ import annotations
+
+from typing import List
+
+from .base_scraper import AbstractJuntaScraper, Leiloeiro
+
+
+class JucebScraper(AbstractJuntaScraper):
+    estado = "BA"
+    junta = "JUCEB"
+    url = "https://www.ba.gov.br/juceb/home/matriculas-e-carteira-profissional/leiloeiros"
+
+    async def parse_leiloeiros(self) -> List[Leiloeiro]:
+        soup = await self.fetch_page()
+        if not soup:
+            soup = await self.fetch_page_js(wait_ms=3000)
+        if not soup:
+            return []
+
+        results: List[Leiloeiro] = []
+
+        for table in soup.find_all("table"):
+            rows = table.find_all("tr")
+            if len(rows) < 2:
+                continue
+            headers = [self.clean(th.get_text()) for th in rows[0].find_all(["th", "td"])]
+            col = {(h or "").lower(): i for i, h in enumerate(headers)}
+
+            def gcol(cells, frags):
+                for k, i in col.items():
+                    if any(f in k for f in frags) and i < len(cells):
+                        return self.clean(cells[i].get_text())
+                return None
+
+            for row in rows[1:]:
+                cells = row.find_all(["td", "th"])
+                if not cells:
+                    continue
+                nome = gcol(cells, ["nome", "leiloeiro"]) or self.clean(cells[0].get_text())
+                if not nome or len(nome) < 3:
+                    continue
+                results.append(self.make_leiloeiro(
+                    nome=nome,
+                    matricula=gcol(cells, ["matr", "registro", "nº"]),
+                    situacao=gcol(cells, ["situ", "status", "regular", "irregular"]),
+                    municipio=gcol(cells, ["munic", "cidade"]) or "Salvador",
+                    telefone=gcol(cells, ["tel", "fone", "contato"]),
+                    email=gcol(cells, ["email"]),
+                    endereco=gcol(cells, ["ender", "logr"]),
+                    data_registro=gcol(cells, ["portaria", "nomea", "data", "posse"]),
+                ))
+            if results:
+                break
+
+        if not results:
+            for li in soup.select("li, p, .item-leiloeiro, article"):
+                text = self.clean(li.get_text(" | "))
+                if text and len(text) > 10:
+                    results.append(self.make_leiloeiro(nome=text, municipio="Salvador"))
+
+        return results

package/templates/vault/junta-leiloeiros/scripts/scraper/jucec.py

@@ -0,0 +1,63 @@
+"""
+Scraper JUCEC — Junta Comercial do Estado do Ceará
+URL: https://www.jucec.ce.gov.br/leiloeiros/
+Método: httpx + BeautifulSoup
+"""
+from __future__ import annotations
+
+from typing import List
+
+from .base_scraper import AbstractJuntaScraper, Leiloeiro
+
+
+class JucecScraper(AbstractJuntaScraper):
+    estado = "CE"
+    junta = "JUCEC"
+    url = "https://www.jucec.ce.gov.br/leiloeiros/"
+
+    async def parse_leiloeiros(self) -> List[Leiloeiro]:
+        soup = await self.fetch_page()
+        if not soup:
+            return []
+
+        results: List[Leiloeiro] = []
+
+        tables = soup.find_all("table")
+        for table in tables:
+            rows = table.find_all("tr")
+            if len(rows) < 2:
+                continue
+            headers = [self.clean(th.get_text()) for th in rows[0].find_all(["th", "td"])]
+            col = {(h or "").lower(): i for i, h in enumerate(headers)}
+
+            def gcol(cells, frags):
+                for k, i in col.items():
+                    if any(f in k for f in frags) and i < len(cells):
+                        return self.clean(cells[i].get_text())
+                return None
+
+            for row in rows[1:]:
+                cells = row.find_all(["td", "th"])
+                if not cells:
+                    continue
+                nome = gcol(cells, ["nome"]) or self.clean(cells[0].get_text())
+                if not nome or len(nome) < 3:
+                    continue
+                results.append(self.make_leiloeiro(
+                    nome=nome,
+                    matricula=gcol(cells, ["matr", "registro", "núm"]),
+                    cpf_cnpj=gcol(cells, ["cpf", "cnpj"]),
+                    situacao=gcol(cells, ["situ", "status"]),
+                    municipio=gcol(cells, ["munic", "cidade", "fortaleza"]),
+                    telefone=gcol(cells, ["tel", "fone"]),
+                    email=gcol(cells, ["email"]),
+                    endereco=gcol(cells, ["ender", "logr"]),
+                ))
+
+        if not results:
+            for el in soup.select("ul li, ol li, .leiloeiro"):
+                text = self.clean(el.get_text(" | "))
+                if text and len(text) > 5:
+                    results.append(self.make_leiloeiro(nome=text))
+
+        return results

package/templates/vault/junta-leiloeiros/scripts/scraper/jucema.py

@@ -0,0 +1,211 @@
+"""
+Scraper JUCEMA — Junta Comercial do Estado do Maranhao
+URL: https://portal.jucema.ma.gov.br/ (React SPA - dados via REST API)
+Metodo: httpx GET para API REST do CMS
+Mecanismo real descoberto em 2026-02-25:
+  - Portal e React SPA (Create React App), sem conteudo server-side
+  - API base: https://api.jucema.ma.gov.br/api/public/
+  - Leiloeiros estao em: GET /api/public/posts/11
+    (post_id 11 = "Leiloeiro" no menu /api/public/menus?with=menus.menus)
+  - Conteudo e HTML dentro do campo data.content
+  - Ultima atualizacao: 2025-10-20 18:29:59
+Total: 53 leiloeiros (39 Regular + 12 Irregular + 2 Cancelada)
+"""
+from __future__ import annotations
+
+import logging
+import re
+from typing import List
+
+import httpx
+from bs4 import BeautifulSoup
+
+from .base_scraper import AbstractJuntaScraper, Leiloeiro
+
+logger = logging.getLogger(__name__)
+
+RE_MATRICULA = re.compile(r"[Mm]atr[íi]cula\s+N[º°o]?\s*(\d+(?:/\d+)?)\s*[–\-]\s*[Ee]m:\s*(\d{2}/\d{2}/\d{4})")
+RE_SITUACAO = re.compile(r"SITUA[ÇC][ÃA]O:\s*(.+)", re.IGNORECASE)
+RE_CONTATO = re.compile(r"[Cc]ontato:\s*(.+)")
+RE_EMAIL = re.compile(r"[Ee]-?[Mm]ail:\s*(.+)")
+RE_ENDERECO = re.compile(r"[Ee]ndere[çc]o:\s*(.+)")
+
+
+class JucemaScraper(AbstractJuntaScraper):
+    estado = "MA"
+    junta = "JUCEMA"
+    url = "https://portal.jucema.ma.gov.br/"
+    _API_URL = "https://api.jucema.ma.gov.br/api/public/posts/11"
+
+    async def _fetch_api(self) -> List[dict]:
+        """
+        Busca dados do post de leiloeiros via API REST do CMS.
+        GET /api/public/posts/11 retorna JSON com campo 'content' em HTML.
+        """
+        try:
+            async with httpx.AsyncClient(
+                headers={**self.HEADERS, "Accept": "application/json"},
+                verify=True,
+                follow_redirects=True,
+                timeout=30.0,
+            ) as client:
+                resp = await client.get(self._API_URL)
+                if resp.status_code >= 400:
+                    logger.warning("[MA] API post/11 retornou HTTP %d", resp.status_code)
+                    return []
+
+                data = resp.json()
+                # O campo pode estar em data.content ou diretamente em content
+                # API retorna { success: true, data: { content: "..." }, message: "..." }
+                inner = data.get("data") or {}
+                content_html = (
+                    inner.get("content") or
+                    data.get("content") or
+                    ""
+                )
+                if not content_html:
+                    logger.warning("[MA] Campo 'content' vazio na API")
+                    return []
+
+                logger.info("[MA] API retornou %d bytes de HTML", len(content_html))
+                soup = BeautifulSoup(content_html, "lxml")
+                return self._parse_cms_content(soup)
+
+        except Exception as exc:
+            logger.error("[MA] Erro na API: %s", exc)
+            return []
+
+    def _parse_cms_content(self, soup) -> List[dict]:
+        """
+        Parseia conteudo HTML do CMS da JUCEMA.
+        Formato dos paragrafos:
+          <p>NOME COMPLETO</p>
+          <p>SITUACAO: REGULAR</p>
+          <p>Matricula No 010/1993 - Em: 29/04/1993</p>
+          <p>Endereco: ...</p>
+          <p>Contato: ...</p>
+          <p>E-mail: ...</p>
+        """
+        records = []
+        paragraphs = [self.clean(p.get_text()) for p in soup.find_all("p") if self.clean(p.get_text())]
+
+        # Tambem tentar com outros elementos se nao houver <p>
+        if len(paragraphs) < 3:
+            paragraphs = [
+                self.clean(el.get_text())
+                for el in soup.find_all(["p", "li", "div", "span"])
+                if self.clean(el.get_text()) and len(self.clean(el.get_text())) > 3
+            ]
+
+        current: dict | None = None
+
+        for text in paragraphs:
+            if not text:
+                continue
+
+            m_matr = RE_MATRICULA.search(text)
+            m_sit = RE_SITUACAO.search(text)
+            m_cont = RE_CONTATO.search(text)
+            m_email = RE_EMAIL.search(text)
+            m_end = RE_ENDERECO.search(text)
+
+            if m_matr:
+                if current:
+                    current["matricula"] = m_matr.group(1)
+                    current["data_registro"] = m_matr.group(2)
+                continue
+            if m_sit:
+                if current:
+                    current["situacao"] = self.clean(m_sit.group(1))
+                continue
+            if m_cont:
+                if current:
+                    current.setdefault("telefone", self.clean(m_cont.group(1)))
+                continue
+            if m_email:
+                if current:
+                    current["email"] = self.clean(m_email.group(1))
+                continue
+            if m_end:
+                if current:
+                    current["endereco"] = self.clean(m_end.group(1))
+                    m_cidade = re.search(r"([A-ZÁÉÍÓÚÀÃÕÇ][A-Za-záéíóúàãõç\s]+)/MA", text)
+                    if m_cidade:
+                        current["municipio"] = m_cidade.group(1).strip()
+                continue
+
+            # Detectar inicio de nova entrada: nome em maiusculas
+            # Excluir titulos de secao como "RELACAO DOS LEILOEIROS", "CEP:", linhas curtas
+            is_nome = (
+                len(text) > 8 and
+                not re.match(r"(SITUA|Matr|MATR|[Ee]ndere|[Cc]ontato|[Ee]-?mail|www\.|http|^\d|Site:|CEP:|RELA[ÇC])", text) and
+                not re.search(r"^(RELA[ÇC][ÃA]O|LISTA|CADASTRO|JUNTA|COMERCIAL|LEILOEIROS\s*$)", text, re.IGNORECASE) and
+                sum(1 for c in text if c.isupper()) > len(text) * 0.3 and
+                " " in text and
+                len(text.split()) >= 2 and
+                len(text) < 120
+            )
+
+            if is_nome:
+                if current and current.get("nome"):
+                    records.append(current)
+                current = {"nome": text, "municipio": "Sao Luis"}
+            elif current and text and re.search(r"\(\d{2}\)", text):
+                current.setdefault("telefone", text)
+
+        if current and current.get("nome"):
+            records.append(current)
+
+        return records
+
+    async def fetch_insecure(self, url: str):
+        """Fetch com verify=False para sites com SSL problematico."""
+        try:
+            async with httpx.AsyncClient(
+                headers=self.HEADERS,
+                timeout=30.0,
+                follow_redirects=True,
+                verify=False,
+            ) as client:
+                resp = await client.get(url)
+                if resp.status_code < 400:
+                    return BeautifulSoup(resp.text, "lxml")
+        except Exception:
+            pass
+        return None
+
+    async def parse_leiloeiros(self) -> List[Leiloeiro]:
+        # Estrategia 1: API REST (direto ao dado, sem renderizacao JS)
+        records = await self._fetch_api()
+
+        if not records:
+            # Estrategia 2: Playwright para renderizar o SPA React
+            logger.info("[MA] API falhou, tentando Playwright no SPA")
+            for spa_url in [
+                "https://portal.jucema.ma.gov.br/leiloeiro",
+                "https://portal.jucema.ma.gov.br/leiloeiros",
+            ]:
+                soup = await self.fetch_page_js(url=spa_url, wait_ms=5000)
+                if soup:
+                    records = self._parse_cms_content(soup)
+                    if records:
+                        break
+
+        if not records:
+            # Estrategia 3: URLs alternativas com httpx
+            logger.info("[MA] Tentando URLs alternativas")
+            for url in [
+                "http://www.jucema.ma.gov.br/leiloeiros",
+                "https://www.jucema.ma.gov.br/leiloeiros",
+                "http://portal.jucema.ma.gov.br/pagina/11",
+            ]:
+                soup = await self.fetch_insecure(url)
+                if soup:
+                    text = soup.get_text()
+                    if any(kw in text.lower() for kw in ["leiloeiro", "matr", "nome"]):
+                        records = self._parse_cms_content(soup)
+                        if records:
+                            break
+
+        logger.info("[MA] Total de registros encontrados: %d", len(records))
+        return [self.make_leiloeiro(**r) for r in records if r.get("nome")]