@danhachuel/thunderbolt 0.6.52 → 0.6.54

package/integrations/instagram_public.py

@@ -25,6 +25,7 @@ except ImportError:  # pragma: no cover - Streamlit is a runtime dependency
 
 LOGGER = logging.getLogger(__name__)
 _WINDOWS_PLAYWRIGHT_CHECKED = False
+_INSTALOADER_INSTALL_ATTEMPTED = False
 
 
 ABOUT_ACCOUNT_URL = 'https://i.instagram.com/api/v1/bloks/apps/com.instagram.interactions.about_this_account/'
@@ -124,6 +125,138 @@ def _instagram_headers() -> dict[str, str]:
     }
 
 
+def _instagram_playwright_headless() -> bool:
+    """Use headed mode for Windows diagnostics unless explicitly overridden."""
+    value = os.getenv('INSTAGRAM_PLAYWRIGHT_HEADLESS', 'false').strip().casefold()
+    return value not in {'0', 'false', 'no', 'off'}
+
+
+def _save_instagram_debug_html(username: str, document: str) -> None:
+    storage_dir = Path(__file__).resolve().parents[1] / 'storage'
+    path = Path(os.getenv('INSTAGRAM_DEBUG_PROFILE_HTML', str(storage_dir / 'debug_instagram.html')))
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(document, encoding='utf-8')
+        LOGGER.info('HTML salvo em %s', path)
+    except OSError as exc:
+        LOGGER.warning('Não foi possível salvar o HTML de diagnóstico Instagram: %s', exc)
+
+
+def _instagram_instaloader():
+    """Load the anonymous fallback, installing it once when the requirement is missing."""
+    global _INSTALOADER_INSTALL_ATTEMPTED
+    try:
+        import instaloader
+        return instaloader
+    except ImportError:
+        if _INSTALOADER_INSTALL_ATTEMPTED:
+            LOGGER.warning('Fallback instaloader indisponível; instale com: python -m pip install instaloader')
+            return None
+        _INSTALOADER_INSTALL_ATTEMPTED = True
+        try:
+            LOGGER.info('Instaloader ausente; a tentar instalar a dependência automaticamente.')
+            completed = subprocess.run(
+                [sys.executable, '-m', 'pip', 'install', 'instaloader>=4.14,<5'],
+                capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=180, check=False,
+            )
+            if completed.returncode == 0:
+                import importlib
+                importlib.invalidate_caches()
+                try:
+                    import instaloader
+                    return instaloader
+                except ImportError:
+                    LOGGER.warning('Instaloader foi instalado, mas não ficou disponível neste interpretador.')
+            LOGGER.warning('Instalação automática do instaloader falhou: %s', (completed.stderr or completed.stdout)[-1000:])
+        except (OSError, subprocess.SubprocessError) as exc:
+            LOGGER.warning('Não foi possível instalar instaloader automaticamente: %s', exc)
+        _streamlit_message('error', 'O fallback instaloader não está disponível. Instale com: python -m pip install instaloader', 'instaloader_missing')
+        return None
+
+
+def _fetch_profile_instaloader(username: str) -> dict[str, Any] | None:
+    """Fetch a public profile anonymously through Instaloader when Playwright fails."""
+    instaloader = _instagram_instaloader()
+    if instaloader is None:
+        return None
+    try:
+        loader = instaloader.Instaloader(
+            quiet=True,
+            request_timeout=30,
+            max_connection_attempts=1,
+            download_pictures=False,
+            download_videos=False,
+            download_video_thumbnails=False,
+            save_metadata=False,
+        )
+        profile = instaloader.Profile.from_username(loader.context, username)
+        LOGGER.info('Fallback instaloader activado para o perfil @%s', username)
+        return {
+            'id': str(getattr(profile, 'userid', '') or ''),
+            'username': str(getattr(profile, 'username', '') or username),
+            'full_name': str(getattr(profile, 'full_name', '') or ''),
+            'biography': str(getattr(profile, 'biography', '') or ''),
+            'follower_count': getattr(profile, 'follower_count', getattr(profile, 'followers', None)),
+            'following_count': getattr(profile, 'following_count', getattr(profile, 'followees', None)),
+            'edge_owner_to_timeline_media': {'count': getattr(profile, 'mediacount', None)},
+            'is_private': bool(getattr(profile, 'is_private', False)),
+            '_instagram_source': 'instaloader',
+        }
+    except Exception as exc:
+        LOGGER.warning('Fallback instaloader falhou para @%s: %s', username, exc, exc_info=True)
+        return None
+
+
+def _fetch_posts_instaloader(username: str, limit: int = 10) -> list[dict[str, Any]]:
+    """Fetch public posts anonymously through Instaloader when Playwright fails."""
+    instaloader = _instagram_instaloader()
+    if instaloader is None:
+        return []
+    posts: list[dict[str, Any]] = []
+    try:
+        loader = instaloader.Instaloader(
+            quiet=True,
+            request_timeout=30,
+            max_connection_attempts=1,
+            download_pictures=False,
+            download_videos=False,
+            download_video_thumbnails=False,
+            save_metadata=False,
+        )
+        profile = instaloader.Profile.from_username(loader.context, username)
+        for post in profile.get_posts():
+            shortcode = str(getattr(post, 'shortcode', '') or '')
+            if not shortcode:
+                continue
+            image_url = str(getattr(post, 'url', '') or '')
+            timestamp = getattr(post, 'date_utc', '') or ''
+            posts.append({
+                'id': str(getattr(post, 'mediaid', '') or shortcode),
+                'shortcode': shortcode,
+                'url': f'https://www.instagram.com/p/{shortcode}/',
+                'image_url': image_url,
+                'display_url': image_url,
+                'caption': str(getattr(post, 'caption', '') or '').strip(),
+                'published_at': timestamp,
+                'timestamp': timestamp,
+                'is_video': bool(getattr(post, 'is_video', False)),
+            })
+            if len(posts) >= max(1, int(limit)):
+                break
+        LOGGER.info('Fallback instaloader carregou %s posts para @%s', len(posts), username)
+    except Exception as exc:
+        LOGGER.warning('Fallback instaloader falhou nos posts de @%s: %s', username, exc, exc_info=True)
+    return posts
+
+
+def fetch_profile_instaloader(username: str) -> dict[str, Any] | None:
+    return _fetch_profile_instaloader(username)
+
+
+def fetch_posts_instaloader(username: str, limit: int = 10) -> list[dict[str, Any]]:
+    return _fetch_posts_instaloader(username, limit)
+
+
 @dataclass
 class IntegrationResult:
     ok: bool
@@ -291,14 +424,17 @@ def _fetch_profile_with_playwright(username: str, headers: Mapping[str, str] | N
     try:
         from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
         with sync_playwright() as playwright:
-            browser = playwright.chromium.launch(headless=True)
+            browser = playwright.chromium.launch(headless=_instagram_playwright_headless())
             try:
-                page = browser.new_page(
-                    user_agent=headers['User-Agent'],
-                    extra_http_headers={'x-ig-app-id': headers['x-ig-app-id'], 'Accept-Language': 'en-US,en;q=0.9'},
-                )
+                headers_for_context = {'x-ig-app-id': headers['x-ig-app-id'], 'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7', 'Accept': 'application/json, text/plain, */*', 'Origin': 'https://www.instagram.com', 'Referer': 'https://www.instagram.com/'}
+                state_path = Path(__file__).resolve().parents[1] / 'storage' / 'instagram_storage_state.json'
+                context_kwargs = {'user_agent': headers['User-Agent'], 'extra_http_headers': headers_for_context}
+                if state_path.is_file():
+                    context_kwargs['storage_state'] = str(state_path)
+                context = browser.new_context(**context_kwargs)
                 if cookies:
-                    page.context.add_cookies([{'name': name, 'value': value, 'domain': '.instagram.com', 'path': '/'} for name, value in cookies.items()])
+                    context.add_cookies([{'name': name, 'value': value, 'domain': '.instagram.com', 'path': '/'} for name, value in cookies.items()])
+                page = context.new_page()
 
                 def capture_profile_response(response: Any) -> None:
                     nonlocal captured_user
@@ -314,16 +450,29 @@ def _fetch_profile_with_playwright(username: str, headers: Mapping[str, str] | N
                         LOGGER.debug('Não foi possível interpretar resposta web_profile_info: %s', exc)
 
                 page.on('response', capture_profile_response)
+                try:
+                    page.goto('https://www.instagram.com/', wait_until='domcontentloaded', timeout=30000)
+                    state_path.parent.mkdir(parents=True, exist_ok=True)
+                    context.storage_state(path=str(state_path))
+                    LOGGER.debug('Instagram Playwright página inicial carregada; storage_state salvo em %s.', state_path)
+                except Exception as exc:
+                    LOGGER.debug('Instagram Playwright não carregou a página inicial: %s', exc)
                 url = f'https://www.instagram.com/{username}/'
                 response = page.goto(url, wait_until='domcontentloaded', timeout=30000)
                 LOGGER.debug('Instagram Playwright profile URL=%s status=%s', url, response.status if response else None)
                 if response is None or response.status >= 400:
-                    return None
+                    LOGGER.warning('Instagram Playwright perfil HTTP status=%s URL=%s', response.status if response else None, url)
+                    return captured_user
+                try:
+                    _save_instagram_debug_html(username, page.content())
+                except Exception as exc:
+                    LOGGER.debug('Não foi possível guardar HTML de diagnóstico: %s', exc)
+                page.wait_for_timeout(5000)
                 try:
                     page.wait_for_load_state('networkidle', timeout=10000)
                 except (PlaywrightTimeoutError, TimeoutError):
                     LOGGER.debug('Instagram Playwright networkidle excedido; usando resposta API já capturada.')
-                LOGGER.debug('Instagram Playwright API profile user=%s quality=%s', bool(captured_user), _profile_quality(captured_user or {}))
+                LOGGER.info('Instagram Playwright API profile captured=%s quality=%s URL=%s', bool(captured_user), _profile_quality(captured_user or {}), url)
             finally:
                 browser.close()
     except Exception as exc:
@@ -339,7 +488,16 @@ def _fetch_web_profile_user(username: str) -> dict[str, Any] | None:
     headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0 Safari/537.36'
     if platform.system() == 'Windows':
         LOGGER.debug('Windows detectado: perfil Instagram exclusivamente via Playwright.')
-        return _fetch_profile_with_playwright(username, headers, cookies)
+        playwright_user = _fetch_profile_with_playwright(username, headers, cookies)
+        if playwright_user:
+            return playwright_user
+        LOGGER.warning('Fallback para instaloader activado no perfil @%s.', username)
+        fallback_user = fetch_profile_instaloader(username)
+        if fallback_user:
+            _streamlit_message('info', 'Dados obtidos via fallback (instaloader). O país pode não estar disponível.', 'profile_instaloader_fallback')
+        else:
+            _streamlit_message('error', 'Não foi possível obter os dados do Instagram. Verifique sua conexão ou tente novamente mais tarde.', 'profile_instagram_failed')
+        return fallback_user
 
     endpoints = (
         f'https://www.instagram.com/api/v1/users/web_profile_info/?username={username}',
@@ -570,7 +728,10 @@ def fetch_public_instagram_profile(source: str) -> IntegrationResult:
         return IntegrationResult(False, str(exc), {})
     api_user = _fetch_web_profile_user(reference['username'])
     if api_user:
-        return IntegrationResult(True, 'Perfil Instagram encontrado publicamente.', _profile_data_from_api(api_user, reference))
+        data = _profile_data_from_api(api_user, reference)
+        if api_user.get('_instagram_source') == 'instaloader':
+            return IntegrationResult(True, 'Perfil encontrado via fallback instaloader. O país pode não estar disponível.', data)
+        return IntegrationResult(True, 'Perfil Instagram encontrado publicamente.', data)
     if platform.system() == 'Windows':
         return IntegrationResult(False, 'Não foi possível extrair os dados públicos do Instagram com o Playwright.', reference)
     try:
@@ -674,14 +835,17 @@ def _fetch_posts_with_playwright(reference: Mapping[str, str], limit: int, cooki
     try:
         from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
         with sync_playwright() as playwright:
-            browser = playwright.chromium.launch(headless=True)
+            browser = playwright.chromium.launch(headless=_instagram_playwright_headless())
             try:
-                page = browser.new_page(
-                    user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0 Safari/537.36',
-                    extra_http_headers={'x-ig-app-id': '936619743392459', 'Accept-Language': 'en-US,en;q=0.9'},
-                )
+                state_path = Path(__file__).resolve().parents[1] / 'storage' / 'instagram_storage_state.json'
+                headers_for_context = {'x-ig-app-id': '936619743392459', 'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7', 'Accept': 'application/json, text/plain, */*', 'Origin': 'https://www.instagram.com', 'Referer': 'https://www.instagram.com/'}
+                context_kwargs = {'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0 Safari/537.36', 'extra_http_headers': headers_for_context}
+                if state_path.is_file():
+                    context_kwargs['storage_state'] = str(state_path)
+                context = browser.new_context(**context_kwargs)
                 if cookies:
-                    page.context.add_cookies([{'name': name, 'value': value, 'domain': '.instagram.com', 'path': '/'} for name, value in cookies.items()])
+                    context.add_cookies([{'name': name, 'value': value, 'domain': '.instagram.com', 'path': '/'} for name, value in cookies.items()])
+                page = context.new_page()
 
                 def capture_profile_posts(response: Any) -> None:
                     if '/api/v1/users/web_profile_info/' not in response.url:
@@ -703,10 +867,17 @@ def _fetch_posts_with_playwright(reference: Mapping[str, str], limit: int, cooki
                         LOGGER.debug('Não foi possível interpretar posts web_profile_info: %s', exc)
 
                 page.on('response', capture_profile_posts)
+                try:
+                    page.goto('https://www.instagram.com/', wait_until='domcontentloaded', timeout=30000)
+                    state_path.parent.mkdir(parents=True, exist_ok=True)
+                    context.storage_state(path=str(state_path))
+                except Exception as exc:
+                    LOGGER.debug('Instagram Playwright não carregou a página inicial para posts: %s', exc)
                 response = page.goto(reference['url'], wait_until='domcontentloaded', timeout=30000)
                 LOGGER.debug('Instagram Playwright posts URL=%s status=%s', reference['url'], response.status if response else None)
                 if response is None or response.status >= 400:
                     return []
+                page.wait_for_timeout(5000)
                 try:
                     page.wait_for_load_state('networkidle', timeout=10000)
                 except (PlaywrightTimeoutError, TimeoutError):
@@ -731,6 +902,13 @@ def fetch_public_instagram_posts(source: str, limit: int = 10) -> IntegrationRes
     if platform.system() == 'Windows':
         LOGGER.debug('Windows detectado: posts Instagram exclusivamente via Playwright.')
         posts = _fetch_posts_with_playwright(reference, limit, _instagram_cookies())
+        if not posts:
+            LOGGER.warning('Fallback para instaloader activado nos posts de @%s.', reference['username'])
+            posts = fetch_posts_instaloader(reference['username'], limit)
+            if posts:
+                _streamlit_message('info', 'Dados obtidos via fallback (instaloader). O país pode não estar disponível.', 'posts_instaloader_fallback')
+            else:
+                _streamlit_message('error', 'Não foi possível obter os dados do Instagram. Verifique sua conexão ou tente novamente mais tarde.', 'posts_instagram_failed')
         return IntegrationResult(bool(posts), 'Posts públicos encontrados.' if posts else 'Não foi possível encontrar posts públicos nesta página do Instagram.', reference | {'posts': posts[:max(1, int(limit))]})
     api_user = _fetch_web_profile_user(reference['username'])
     if api_user:
@@ -773,4 +951,4 @@ def fetch_public_instagram_posts(source: str, limit: int = 10) -> IntegrationRes
     return IntegrationResult(bool(posts), 'Posts públicos encontrados.' if posts else 'Não foi possível encontrar posts públicos nesta página do Instagram.', reference | {'posts': posts[:max(1, int(limit))]})
 
 
-__all__ = ['IntegrationResult', 'extract_public_instagram_country', 'fetch_public_instagram_posts', 'fetch_public_instagram_profile', 'normalize_instagram_bio', 'normalize_instagram_metric', 'normalize_instagram_reference']
+__all__ = ['IntegrationResult', 'extract_public_instagram_country', 'fetch_posts_instaloader', 'fetch_profile_instaloader', 'fetch_public_instagram_posts', 'fetch_public_instagram_profile', 'normalize_instagram_bio', 'normalize_instagram_metric', 'normalize_instagram_reference']

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@danhachuel/thunderbolt",
-  "version": "0.6.52",
+  "version": "0.6.54",
   "description": "Thunderbolt — interface local para operação de canais faceless e motor MoneyPrinterTurbo",
   "license": "MIT",
   "main": "scripts/cli.mjs",

package/requirements.txt

@@ -26,5 +26,6 @@ kagglehub>=0.3,<1
 supabase>=2.4,<3
 bilibili-api-python>=17.4,<18
 playwright>=1.45,<2
+instaloader>=4.14,<5
 composio>=0.21,<1
 justokenmax @ https://github.com/Kalmantic/jusTokenMax/archive/refs/heads/main.zip#subdirectory=python