@danhachuel/thunderbolt 0.6.57 → 0.6.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/social_networks_ui.py +23 -123
- package/integrations/instagram_public.py +133 -421
- package/package.json +1 -1
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import io
|
|
4
|
-
import logging
|
|
5
4
|
import re
|
|
6
5
|
import uuid
|
|
7
6
|
import zipfile
|
|
8
7
|
import unicodedata
|
|
9
|
-
import hashlib
|
|
10
8
|
from typing import Any, Callable, Mapping
|
|
11
9
|
|
|
12
10
|
import requests
|
|
@@ -17,19 +15,10 @@ from hermes_ui.influencers import InfluencerBackendError, STANDALONE_CONTENT_INF
|
|
|
17
15
|
from hermes_ui.storage import read_json, write_json
|
|
18
16
|
from hermes_ui.countries import COUNTRY_OPTIONS
|
|
19
17
|
from hermes_ui.languages import LANGUAGE_CODES, language_code, language_label
|
|
20
|
-
from integrations.instagram_public import
|
|
21
|
-
extract_public_instagram_country,
|
|
22
|
-
fetch_public_instagram_posts,
|
|
23
|
-
fetch_public_instagram_profile,
|
|
24
|
-
normalize_instagram_bio,
|
|
25
|
-
normalize_instagram_metric,
|
|
26
|
-
)
|
|
18
|
+
from integrations.instagram_public import fetch_public_instagram_posts, fetch_public_instagram_profile, normalize_instagram_bio, normalize_instagram_metric
|
|
27
19
|
from integrations.meta_social import test_facebook_pages_api_card, test_instagram_api_card
|
|
28
20
|
|
|
29
21
|
|
|
30
|
-
LOGGER = logging.getLogger(__name__)
|
|
31
|
-
|
|
32
|
-
|
|
33
22
|
def _clean(value: Any) -> str:
|
|
34
23
|
return str(value or "").strip()
|
|
35
24
|
|
|
@@ -78,35 +67,11 @@ def _country_key(value: Any) -> str:
|
|
|
78
67
|
COUNTRY_ALIASES = {
|
|
79
68
|
"brazil": "Brasil",
|
|
80
69
|
"brasil": "Brasil",
|
|
81
|
-
"br": "Brasil",
|
|
82
70
|
"usa": "Estados Unidos",
|
|
83
71
|
"us": "Estados Unidos",
|
|
84
72
|
"united states": "Estados Unidos",
|
|
85
73
|
"united states of america": "Estados Unidos",
|
|
86
|
-
"pt": "Portugal",
|
|
87
74
|
"portugal": "Portugal",
|
|
88
|
-
"uk": "Reino Unido",
|
|
89
|
-
"gb": "Reino Unido",
|
|
90
|
-
"united kingdom": "Reino Unido",
|
|
91
|
-
"england": "Reino Unido",
|
|
92
|
-
"germany": "Alemanha",
|
|
93
|
-
"de": "Alemanha",
|
|
94
|
-
"france": "França",
|
|
95
|
-
"fr": "França",
|
|
96
|
-
"italy": "Itália",
|
|
97
|
-
"it": "Itália",
|
|
98
|
-
"spain": "Espanha",
|
|
99
|
-
"es": "Espanha",
|
|
100
|
-
"canada": "Canadá",
|
|
101
|
-
"ca": "Canadá",
|
|
102
|
-
"australia": "Austrália",
|
|
103
|
-
"au": "Austrália",
|
|
104
|
-
"japan": "Japão",
|
|
105
|
-
"jp": "Japão",
|
|
106
|
-
"mexico": "México",
|
|
107
|
-
"mx": "México",
|
|
108
|
-
"argentina": "Argentina",
|
|
109
|
-
"ar": "Argentina",
|
|
110
75
|
}
|
|
111
76
|
|
|
112
77
|
|
|
@@ -124,17 +89,11 @@ def _normalise_country(value: Any) -> str:
|
|
|
124
89
|
|
|
125
90
|
|
|
126
91
|
def _profile_country(profile: Mapping[str, Any]) -> str:
|
|
127
|
-
for key in ("country", "country_name", "country_of_registration", "account_country", "account_based_in"
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
for candidate in candidates:
|
|
133
|
-
country = _normalise_country(candidate)
|
|
134
|
-
if country:
|
|
135
|
-
return country
|
|
136
|
-
extracted = _normalise_country(extract_public_instagram_country(profile))
|
|
137
|
-
return extracted
|
|
92
|
+
for key in ("country", "country_name", "country_of_registration", "account_country", "account_based_in"):
|
|
93
|
+
country = _normalise_country(profile.get(key))
|
|
94
|
+
if country:
|
|
95
|
+
return country
|
|
96
|
+
return ""
|
|
138
97
|
|
|
139
98
|
|
|
140
99
|
def _language_index(value: Any) -> int:
|
|
@@ -165,41 +124,10 @@ def _instagram_profile_storage_id(profile: Mapping[str, Any]) -> str:
|
|
|
165
124
|
return re.sub(r"[^A-Za-z0-9_.-]+", "_", value) or "unknown"
|
|
166
125
|
|
|
167
126
|
|
|
168
|
-
def _clear_instagram_result_widget_state(profile: Mapping[str, Any]) -> None:
|
|
169
|
-
"""Clear every previous widget key for this profile before rebuilding the form."""
|
|
170
|
-
widget_id = _instagram_profile_storage_id(profile)
|
|
171
|
-
for field in ("name", "bio", "country", "language", "posts", "followers", "following", "character"):
|
|
172
|
-
prefix = f"social_instagram_result_{field}_{widget_id}"
|
|
173
|
-
for key in list(st.session_state.keys()):
|
|
174
|
-
if str(key).startswith(prefix):
|
|
175
|
-
st.session_state.pop(key, None)
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
def _instagram_result_widget_id(profile: Mapping[str, Any]) -> str:
|
|
179
|
-
base_id = _instagram_profile_storage_id(profile)
|
|
180
|
-
signature = "|".join(_clean(profile.get(key)) for key in ("bio", "bio_raw", "following_count", "subscriber_count", "post_count", "country"))
|
|
181
|
-
digest = hashlib.sha1(signature.encode("utf-8")).hexdigest()[:10]
|
|
182
|
-
return f"{base_id}_{digest}"
|
|
183
|
-
|
|
184
|
-
|
|
185
127
|
def _normalise_instagram_posts(value: Any) -> list[dict[str, Any]]:
|
|
186
|
-
if isinstance(value, Mapping):
|
|
187
|
-
value = value.get("posts") or value.get("items") or value.get("edges") or []
|
|
188
128
|
if not isinstance(value, list):
|
|
189
129
|
return []
|
|
190
|
-
|
|
191
|
-
seen: set[str] = set()
|
|
192
|
-
for item in value:
|
|
193
|
-
post = item.get("node") if isinstance(item, Mapping) and isinstance(item.get("node"), Mapping) else item
|
|
194
|
-
if not isinstance(post, Mapping):
|
|
195
|
-
continue
|
|
196
|
-
data = dict(post)
|
|
197
|
-
identifier = _clean(data.get("id") or data.get("shortcode") or data.get("url"))
|
|
198
|
-
if not identifier or identifier in seen:
|
|
199
|
-
continue
|
|
200
|
-
seen.add(identifier)
|
|
201
|
-
normalised.append(data)
|
|
202
|
-
return normalised
|
|
130
|
+
return [dict(post) for post in value if isinstance(post, Mapping) and _clean(post.get("id") or post.get("url"))]
|
|
203
131
|
|
|
204
132
|
|
|
205
133
|
def _stored_instagram_posts(profile: Mapping[str, Any]) -> list[dict[str, Any]]:
|
|
@@ -248,8 +176,8 @@ def _instagram_posts_archive(posts: list[dict[str, Any]]) -> bytes:
|
|
|
248
176
|
elif "webp" in content_type:
|
|
249
177
|
extension = ".webp"
|
|
250
178
|
bundle.writestr(f"post-{index:02d}{extension}", response.content)
|
|
251
|
-
except requests.RequestException
|
|
252
|
-
|
|
179
|
+
except requests.RequestException:
|
|
180
|
+
pass
|
|
253
181
|
manifest.append(f"{index}. {_clean(post.get('url'))}\n{_clean(post.get('caption'))}\n")
|
|
254
182
|
bundle.writestr("posts.txt", "\n".join(manifest))
|
|
255
183
|
return archive.getvalue()
|
|
@@ -270,14 +198,13 @@ def _render_instagram_posts(profile: Mapping[str, Any]) -> None:
|
|
|
270
198
|
if load_clicked or refresh_clicked:
|
|
271
199
|
target = max(10, len(posts)) if refresh_clicked else 10
|
|
272
200
|
posts_ok, posts_message, loaded_posts = _load_instagram_posts(profile, limit=target)
|
|
273
|
-
# DEBUG WINDOWS INSTAGRAM: manter até a validação final do carregamento público.
|
|
274
|
-
st.write({"instagram_posts_ok": posts_ok, "instagram_posts_message": posts_message, "instagram_posts_count": len(loaded_posts)})
|
|
275
201
|
if posts_ok:
|
|
276
|
-
posts =
|
|
202
|
+
posts = loaded_posts
|
|
277
203
|
_persist_instagram_posts(profile, posts)
|
|
278
204
|
st.success(f"{len(posts)} posts carregados.")
|
|
205
|
+
st.rerun()
|
|
279
206
|
else:
|
|
280
|
-
st.
|
|
207
|
+
st.warning(posts_message)
|
|
281
208
|
posts = st.session_state.get(state_key, posts)
|
|
282
209
|
if not posts:
|
|
283
210
|
st.info("Clique em Carregar últimos 10 para consultar os posts públicos desta conta.")
|
|
@@ -296,10 +223,9 @@ def _render_instagram_posts(profile: Mapping[str, Any]) -> None:
|
|
|
296
223
|
if st.button("Mostrar + 10", key=f"more_instagram_posts_{profile_id}", use_container_width=True):
|
|
297
224
|
posts_ok, posts_message, loaded_posts = _load_instagram_posts(profile, limit=len(posts) + 10)
|
|
298
225
|
if posts_ok:
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
st.error(posts_message)
|
|
226
|
+
_persist_instagram_posts(profile, loaded_posts)
|
|
227
|
+
st.rerun()
|
|
228
|
+
st.warning(posts_message)
|
|
303
229
|
|
|
304
230
|
|
|
305
231
|
def _api_card_defaults(kind: str) -> tuple[str, list[str], Callable[[Mapping[str, Any]], dict[str, Any]]]:
|
|
@@ -446,8 +372,7 @@ def _characters(settings: Mapping[str, Any]) -> tuple[list[dict[str, Any]], Any
|
|
|
446
372
|
repository = get_repository(settings)
|
|
447
373
|
items = [item for item in repository.list_influencers() if _clean(item.get("id")) != STANDALONE_CONTENT_INFLUENCER_ID]
|
|
448
374
|
return items, repository
|
|
449
|
-
except Exception
|
|
450
|
-
LOGGER.warning("Não foi possível carregar as personagens: %s", exc, exc_info=True)
|
|
375
|
+
except (InfluencerBackendError, Exception):
|
|
451
376
|
return [], None
|
|
452
377
|
|
|
453
378
|
|
|
@@ -559,39 +484,19 @@ def _render_instagram_card(profile: dict[str, Any], characters: list[dict[str, A
|
|
|
559
484
|
refresh_col, edit_col = st.columns(2)
|
|
560
485
|
with refresh_col:
|
|
561
486
|
if st.button("↻", help="Actualizar posts, seguidores e seguindo", key=f"refresh_instagram_{profile_id}"):
|
|
562
|
-
|
|
563
|
-
refreshed_ok, refreshed_message, refreshed = _refresh_instagram_profile(profile)
|
|
564
|
-
except Exception as exc:
|
|
565
|
-
st.error(f"Falha ao actualizar o perfil Instagram: {exc}")
|
|
566
|
-
refreshed_ok, refreshed_message, refreshed = False, str(exc), {}
|
|
487
|
+
refreshed_ok, refreshed_message, refreshed = _refresh_instagram_profile(profile)
|
|
567
488
|
if refreshed_ok:
|
|
568
489
|
current_posts = _stored_instagram_posts(profile)
|
|
569
|
-
|
|
570
|
-
posts_ok, posts_message, refreshed_posts = _load_instagram_posts(refreshed, limit=max(10, len(current_posts)))
|
|
571
|
-
except Exception as exc:
|
|
572
|
-
posts_ok, posts_message, refreshed_posts = False, str(exc), []
|
|
490
|
+
posts_ok, _, refreshed_posts = _load_instagram_posts(refreshed, limit=max(10, len(current_posts)))
|
|
573
491
|
if posts_ok:
|
|
574
492
|
_persist_instagram_posts(profile, refreshed_posts)
|
|
575
|
-
|
|
576
|
-
st.warning(f"Perfil actualizado, mas os posts não foram actualizados: {posts_message}")
|
|
577
|
-
# Debug temporário removido após validação; manter para reactivar se necessário:
|
|
578
|
-
# st.write({"profile_ok": refreshed_ok, "profile_message": refreshed_message, "profile_data": refreshed})
|
|
579
|
-
# st.write({"posts_ok": posts_ok, "posts_message": posts_message, "posts_count": len(refreshed_posts)})
|
|
580
|
-
try:
|
|
581
|
-
updated = update_channel(profile_id, {**refreshed, "id": profile_id, "country": _profile_country(refreshed) or _profile_country(profile), "language": profile.get("language", ""), "character_id": profile.get("character_id", "")})
|
|
582
|
-
except (OSError, PermissionError) as exc:
|
|
583
|
-
st.error(f"Não foi possível guardar os dados no Windows: {exc}")
|
|
584
|
-
return
|
|
585
|
-
except Exception as exc:
|
|
586
|
-
st.error(f"Falha ao guardar os dados da conta Instagram: {exc}")
|
|
587
|
-
return
|
|
493
|
+
updated = update_channel(profile_id, {**refreshed, "id": profile_id, "country": _profile_country(refreshed) or _profile_country(profile), "language": profile.get("language", ""), "character_id": profile.get("character_id", "")})
|
|
588
494
|
if updated is None:
|
|
589
|
-
st.error("A conta Instagram já não existe no armazenamento local
|
|
495
|
+
st.error("A conta Instagram já não existe no armazenamento local.")
|
|
590
496
|
return
|
|
591
497
|
st.success("Métricas Instagram actualizadas.")
|
|
592
498
|
st.rerun()
|
|
593
|
-
|
|
594
|
-
st.error(f"Não foi possível actualizar o perfil Instagram: {refreshed_message}")
|
|
499
|
+
st.warning(refreshed_message)
|
|
595
500
|
with edit_col:
|
|
596
501
|
if st.button("Editar", key=f"edit_instagram_button_{profile_id}", use_container_width=True):
|
|
597
502
|
for field in ("name", "handle", "bio", "country", "language", "posts", "followers", "following"):
|
|
@@ -678,20 +583,15 @@ def render_social_networks(settings: dict[str, Any]) -> None:
|
|
|
678
583
|
if search_clicked:
|
|
679
584
|
result = fetch_public_instagram_profile(source)
|
|
680
585
|
result_data = dict(result.data) if isinstance(result.data, dict) else {}
|
|
681
|
-
if result_data:
|
|
682
|
-
_clear_instagram_result_widget_state(result_data)
|
|
683
586
|
st.session_state["social_instagram_result"] = result_data
|
|
684
587
|
st.session_state["social_instagram_ok"] = result.ok
|
|
685
588
|
st.session_state["social_instagram_message"] = result.message
|
|
686
589
|
if st.session_state.get("social_instagram_message"):
|
|
687
|
-
if st.session_state.get("social_instagram_ok")
|
|
688
|
-
st.success(st.session_state["social_instagram_message"])
|
|
689
|
-
else:
|
|
690
|
-
st.error(st.session_state["social_instagram_message"])
|
|
590
|
+
(st.success if st.session_state.get("social_instagram_ok") else st.warning)(st.session_state["social_instagram_message"])
|
|
691
591
|
data = st.session_state.get("social_instagram_result", {}) if st.session_state.get("social_instagram_ok") else {}
|
|
692
592
|
if data:
|
|
693
593
|
characters, _ = _characters(settings)
|
|
694
|
-
result_widget_id =
|
|
594
|
+
result_widget_id = _instagram_profile_storage_id(data)
|
|
695
595
|
with st.container(border=True):
|
|
696
596
|
st.subheader("Conta Instagram encontrada")
|
|
697
597
|
preview_cols = st.columns([0.8, 2.2, 1.2, 1.2, 1.2])
|
|
@@ -2,107 +2,23 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import re
|
|
4
4
|
import json
|
|
5
|
-
import logging
|
|
6
5
|
import os
|
|
7
6
|
import platform
|
|
8
|
-
import sys
|
|
9
7
|
import shutil
|
|
10
8
|
import subprocess
|
|
11
9
|
from dataclasses import dataclass
|
|
12
10
|
from datetime import datetime, timezone
|
|
13
11
|
from html import unescape
|
|
14
|
-
from pathlib import Path
|
|
15
12
|
from typing import Any, Mapping
|
|
16
13
|
from urllib.parse import urlparse
|
|
17
14
|
|
|
18
15
|
import requests
|
|
19
16
|
|
|
20
|
-
try:
|
|
21
|
-
import streamlit as st
|
|
22
|
-
except ImportError: # pragma: no cover - Streamlit is a runtime dependency
|
|
23
|
-
st = None
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
LOGGER = logging.getLogger(__name__)
|
|
27
|
-
_WINDOWS_PLAYWRIGHT_CHECKED = False
|
|
28
|
-
|
|
29
17
|
|
|
30
18
|
ABOUT_ACCOUNT_URL = 'https://i.instagram.com/api/v1/bloks/apps/com.instagram.interactions.about_this_account/'
|
|
31
19
|
ABOUT_ACCOUNT_BLOKS_VERSION = '8ca96ca267e30c02cf90888d91eeff09627f0e3fd2bd9df472278c9a6c022cbb'
|
|
32
20
|
|
|
33
21
|
|
|
34
|
-
def _load_dotenv_compat() -> None:
|
|
35
|
-
"""Load .env when python-dotenv exists, otherwise parse simple KEY=VALUE lines."""
|
|
36
|
-
try:
|
|
37
|
-
from dotenv import load_dotenv
|
|
38
|
-
load_dotenv()
|
|
39
|
-
return
|
|
40
|
-
except ImportError:
|
|
41
|
-
LOGGER.debug('python-dotenv não está instalado; será usado o parser .env compatível.')
|
|
42
|
-
env_path = Path(__file__).resolve().parents[1] / '.env'
|
|
43
|
-
if not env_path.is_file():
|
|
44
|
-
return
|
|
45
|
-
try:
|
|
46
|
-
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
|
|
47
|
-
line = raw_line.strip()
|
|
48
|
-
if not line or line.startswith('#') or '=' not in line:
|
|
49
|
-
continue
|
|
50
|
-
key, value = line.split('=', 1)
|
|
51
|
-
value = value.strip().strip('"').strip("'")
|
|
52
|
-
os.environ.setdefault(key.strip(), value)
|
|
53
|
-
except OSError as exc:
|
|
54
|
-
LOGGER.warning('Não foi possível ler %s: %s', env_path, exc)
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def _streamlit_message(level: str, message: str, key: str) -> None:
|
|
58
|
-
"""Show a runtime diagnostic once when Streamlit is available."""
|
|
59
|
-
if st is None:
|
|
60
|
-
return
|
|
61
|
-
try:
|
|
62
|
-
shown_key = f'_instagram_debug_{key}'
|
|
63
|
-
if st.session_state.get(shown_key):
|
|
64
|
-
return
|
|
65
|
-
st.session_state[shown_key] = True
|
|
66
|
-
getattr(st, level)(message)
|
|
67
|
-
except Exception as exc: # Streamlit may be imported outside an active app context.
|
|
68
|
-
LOGGER.debug('Não foi possível mostrar diagnóstico Streamlit: %s', exc)
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
def _ensure_windows_playwright() -> None:
|
|
72
|
-
"""Ensure Chromium exists on Windows before using the Playwright fallback."""
|
|
73
|
-
global _WINDOWS_PLAYWRIGHT_CHECKED
|
|
74
|
-
if os.name != 'nt' or _WINDOWS_PLAYWRIGHT_CHECKED:
|
|
75
|
-
return
|
|
76
|
-
_WINDOWS_PLAYWRIGHT_CHECKED = True
|
|
77
|
-
_load_dotenv_compat()
|
|
78
|
-
try:
|
|
79
|
-
completed = subprocess.run(
|
|
80
|
-
[sys.executable, '-m', 'playwright', 'install', 'chromium'],
|
|
81
|
-
capture_output=True,
|
|
82
|
-
text=True,
|
|
83
|
-
encoding='utf-8',
|
|
84
|
-
errors='replace',
|
|
85
|
-
timeout=180,
|
|
86
|
-
check=False,
|
|
87
|
-
)
|
|
88
|
-
if completed.returncode != 0:
|
|
89
|
-
detail = (completed.stderr or completed.stdout or 'sem detalhes').strip()[-1000:]
|
|
90
|
-
message = f'Não foi possível instalar o Chromium do Playwright no Windows: {detail}'
|
|
91
|
-
LOGGER.warning(message)
|
|
92
|
-
_streamlit_message('error', message + ' Execute: python -m playwright install chromium', 'playwright_install_error')
|
|
93
|
-
else:
|
|
94
|
-
LOGGER.info('Chromium do Playwright verificado/instalado no Windows.')
|
|
95
|
-
except (OSError, subprocess.SubprocessError) as exc:
|
|
96
|
-
message = f'Falha ao executar a instalação do Chromium no Windows: {exc}'
|
|
97
|
-
LOGGER.warning(message)
|
|
98
|
-
_streamlit_message('error', message + ' Execute: python -m playwright install chromium', 'playwright_install_exception')
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
_load_dotenv_compat()
|
|
102
|
-
if os.name == 'nt':
|
|
103
|
-
_ensure_windows_playwright()
|
|
104
|
-
|
|
105
|
-
|
|
106
22
|
def _instagram_cookies() -> dict[str, str]:
|
|
107
23
|
cookies = {}
|
|
108
24
|
sessionid = os.getenv('INSTAGRAM_SESSIONID') or os.getenv('IG_SESSIONID')
|
|
@@ -124,21 +40,34 @@ def _instagram_headers() -> dict[str, str]:
|
|
|
124
40
|
}
|
|
125
41
|
|
|
126
42
|
|
|
127
|
-
def
|
|
128
|
-
"""
|
|
129
|
-
value = os.getenv('INSTAGRAM_PLAYWRIGHT_HEADLESS', 'true').strip().casefold()
|
|
130
|
-
return value not in {'0', 'false', 'no', 'off'}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def _save_instagram_debug_html(username: str, document: str) -> None:
|
|
134
|
-
storage_dir = Path(__file__).resolve().parents[1] / 'storage'
|
|
135
|
-
path = Path(os.getenv('INSTAGRAM_DEBUG_PROFILE_HTML', str(storage_dir / 'debug_instagram.html')))
|
|
43
|
+
def _fetch_instagram_html_with_playwright(url: str, username: str) -> str:
|
|
44
|
+
"""Windows-only browser fallback; the existing Linux HTML parser remains authoritative."""
|
|
136
45
|
try:
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
46
|
+
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
|
|
47
|
+
with sync_playwright() as playwright:
|
|
48
|
+
browser = playwright.chromium.launch(headless=True)
|
|
49
|
+
try:
|
|
50
|
+
context = browser.new_context(
|
|
51
|
+
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0 Safari/537.36',
|
|
52
|
+
extra_http_headers={'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7'},
|
|
53
|
+
)
|
|
54
|
+
page = context.new_page()
|
|
55
|
+
response = page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
|
56
|
+
status = response.status if response else None
|
|
57
|
+
print(f'Playwright usado no Windows para @{username}; status HTML={status}')
|
|
58
|
+
page.wait_for_timeout(5000)
|
|
59
|
+
try:
|
|
60
|
+
page.wait_for_load_state('networkidle', timeout=10000)
|
|
61
|
+
except (PlaywrightTimeoutError, TimeoutError):
|
|
62
|
+
pass
|
|
63
|
+
html_content = page.content()
|
|
64
|
+
print(f'HTML Instagram recebido no Windows para @{username}: {len(html_content)} bytes')
|
|
65
|
+
return html_content
|
|
66
|
+
finally:
|
|
67
|
+
browser.close()
|
|
68
|
+
except Exception as exc:
|
|
69
|
+
print(f'Playwright falhou no Windows para @{username}: {exc}')
|
|
70
|
+
return ''
|
|
142
71
|
|
|
143
72
|
|
|
144
73
|
@dataclass
|
|
@@ -149,9 +78,8 @@ class IntegrationResult:
|
|
|
149
78
|
|
|
150
79
|
|
|
151
80
|
def normalize_instagram_bio(value: Any) -> str:
|
|
152
|
-
"""
|
|
153
|
-
|
|
154
|
-
return bio
|
|
81
|
+
"""Preserve the complete biography returned by Instagram."""
|
|
82
|
+
return str(value or '').strip()
|
|
155
83
|
|
|
156
84
|
|
|
157
85
|
def normalize_instagram_reference(source: str) -> dict[str, str]:
|
|
@@ -221,102 +149,14 @@ def _structured_metric_from_json(document: str, *keys: str) -> int | None:
|
|
|
221
149
|
return None
|
|
222
150
|
|
|
223
151
|
|
|
224
|
-
def
|
|
225
|
-
groups = (
|
|
226
|
-
('biography', 'bio', 'description'),
|
|
227
|
-
('edge_follow', 'following', 'following_count', 'followingCount', 'follows'),
|
|
228
|
-
('edge_followed_by', 'followers', 'follower_count', 'followers_count', 'followerCount'),
|
|
229
|
-
('edge_owner_to_timeline_media', 'posts', 'post_count', 'media_count'),
|
|
230
|
-
('username', 'user_name', 'handle'),
|
|
231
|
-
)
|
|
232
|
-
return sum(any(user.get(key) not in (None, '', {}, []) for key in group) for group in groups)
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
def _merge_profile(base: dict[str, Any] | None, candidate: Mapping[str, Any]) -> dict[str, Any]:
|
|
236
|
-
merged = dict(base or {})
|
|
237
|
-
for key, value in candidate.items():
|
|
238
|
-
if value not in (None, '', [], {}):
|
|
239
|
-
if isinstance(value, Mapping) and isinstance(merged.get(key), Mapping):
|
|
240
|
-
merged[key] = {**merged[key], **value}
|
|
241
|
-
else:
|
|
242
|
-
merged[key] = value
|
|
243
|
-
return merged
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
def _extract_profile_user(payload: Any) -> Mapping[str, Any] | None:
|
|
247
|
-
if not isinstance(payload, Mapping):
|
|
248
|
-
return None
|
|
249
|
-
candidates = [
|
|
250
|
-
payload.get('user'),
|
|
251
|
-
(payload.get('data') or {}).get('user') if isinstance(payload.get('data'), Mapping) else None,
|
|
252
|
-
(payload.get('graphql') or {}).get('user') if isinstance(payload.get('graphql'), Mapping) else None,
|
|
253
|
-
(payload.get('data') or {}).get('profile') if isinstance(payload.get('data'), Mapping) else None,
|
|
254
|
-
(payload.get('xdt_api__v1__users__web_profile_info') or {}).get('user') if isinstance(payload.get('xdt_api__v1__users__web_profile_info'), Mapping) else None,
|
|
255
|
-
]
|
|
256
|
-
direct = next((candidate for candidate in candidates if isinstance(candidate, Mapping)), None)
|
|
257
|
-
if direct is not None:
|
|
258
|
-
return direct
|
|
259
|
-
for node in _walk_json(payload):
|
|
260
|
-
if not isinstance(node, Mapping) or not node.get('username'):
|
|
261
|
-
continue
|
|
262
|
-
if any(key in node for key in ('biography', 'edge_follow', 'edge_followed_by', 'follower_count', 'following_count', 'full_name')):
|
|
263
|
-
return node
|
|
264
|
-
return None
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
def _shared_data_documents(document: str) -> list[Any]:
|
|
268
|
-
"""Extract JSON assigned to window._sharedData from the rendered profile HTML."""
|
|
269
|
-
decoder = json.JSONDecoder()
|
|
270
|
-
documents: list[Any] = []
|
|
271
|
-
patterns = (
|
|
272
|
-
r'window\s*\.\s*_sharedData\s*=\s*',
|
|
273
|
-
r'window\s*\[\s*["\']_sharedData["\']\s*\]\s*=\s*',
|
|
274
|
-
)
|
|
275
|
-
for pattern in patterns:
|
|
276
|
-
for match in re.finditer(pattern, document, flags=re.IGNORECASE):
|
|
277
|
-
start = document.find('{', match.end())
|
|
278
|
-
if start < 0:
|
|
279
|
-
continue
|
|
280
|
-
try:
|
|
281
|
-
parsed, _ = decoder.raw_decode(document[start:])
|
|
282
|
-
except json.JSONDecodeError:
|
|
283
|
-
continue
|
|
284
|
-
documents.append(parsed)
|
|
285
|
-
return documents
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
def _profile_users_from_document(document: str) -> list[Mapping[str, Any]]:
|
|
289
|
-
"""Return users found in legacy _sharedData and current embedded profile payloads."""
|
|
290
|
-
users: list[Mapping[str, Any]] = []
|
|
291
|
-
documents = _shared_data_documents(document) + _embedded_json_documents(document)
|
|
292
|
-
for payload in documents:
|
|
293
|
-
direct = _extract_profile_user(payload)
|
|
294
|
-
if isinstance(direct, Mapping):
|
|
295
|
-
users.append(direct)
|
|
296
|
-
for node in _walk_json(payload):
|
|
297
|
-
if not isinstance(node, Mapping):
|
|
298
|
-
continue
|
|
299
|
-
candidate = node.get('user')
|
|
300
|
-
if isinstance(candidate, Mapping) and any(key in candidate for key in ('biography', 'edge_follow', 'edge_followed_by', 'username')):
|
|
301
|
-
users.append(candidate)
|
|
302
|
-
unique: list[Mapping[str, Any]] = []
|
|
303
|
-
seen: set[str] = set()
|
|
304
|
-
for user in users:
|
|
305
|
-
marker = str(user.get('id') or user.get('pk') or user.get('username') or id(user))
|
|
306
|
-
if marker not in seen:
|
|
307
|
-
seen.add(marker)
|
|
308
|
-
unique.append(user)
|
|
309
|
-
return unique
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
def _extract_posts_from_documents(documents: list[Any], limit: int) -> list[dict[str, Any]]:
|
|
152
|
+
def _extract_posts_from_html(document: str, limit: int = 10) -> list[dict[str, Any]]:
|
|
313
153
|
posts: list[dict[str, Any]] = []
|
|
314
154
|
seen: set[str] = set()
|
|
315
|
-
for
|
|
316
|
-
for node in _walk_json(
|
|
317
|
-
if not isinstance(node,
|
|
155
|
+
for payload in _embedded_json_documents(document):
|
|
156
|
+
for node in _walk_json(payload):
|
|
157
|
+
if not isinstance(node, dict):
|
|
318
158
|
continue
|
|
319
|
-
post = _post_from_node(
|
|
159
|
+
post = _post_from_node(node)
|
|
320
160
|
if not post or post['id'] in seen:
|
|
321
161
|
continue
|
|
322
162
|
seen.add(post['id'])
|
|
@@ -326,111 +166,84 @@ def _extract_posts_from_documents(documents: list[Any], limit: int) -> list[dict
|
|
|
326
166
|
return posts
|
|
327
167
|
|
|
328
168
|
|
|
329
|
-
def
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
users = _profile_users_from_document(html)
|
|
334
|
-
user = max(users, key=_profile_quality, default={})
|
|
335
|
-
posts = _extract_posts_from_documents(documents, limit=10)
|
|
336
|
-
result = {'user': dict(user) if isinstance(user, Mapping) else {}, 'posts': posts}
|
|
337
|
-
LOGGER.info('Instagram HTML extraction profile=%s posts=%s', bool(result['user']), len(posts))
|
|
338
|
-
print(f"Playwright usado no Windows, dados extraídos com {'sucesso' if result['user'] or posts else 'falha'}")
|
|
339
|
-
return result
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
def _extract_posts_from_html(html_content: str, limit: int = 10) -> list[dict[str, Any]]:
|
|
343
|
-
"""Extract public posts from _sharedData and JSON scripts in rendered HTML."""
|
|
344
|
-
html = str(html_content or '')
|
|
345
|
-
documents = _shared_data_documents(html) + _embedded_json_documents(html)
|
|
346
|
-
posts = _extract_posts_from_documents(documents, limit=max(1, int(limit)))
|
|
347
|
-
LOGGER.info('Instagram HTML post extraction posts=%s limit=%s', len(posts), limit)
|
|
348
|
-
print(f"Playwright usado no Windows, posts extraídos: {len(posts)}")
|
|
349
|
-
return posts
|
|
169
|
+
def _extract_profile_user_from_html(document: str, username: str) -> dict[str, Any]:
|
|
170
|
+
description = _meta(document, 'og:description')
|
|
171
|
+
title = _meta(document, 'og:title') or username
|
|
172
|
+
avatar_url = _meta(document, 'og:image')
|
|
350
173
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
try:
|
|
356
|
-
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
|
|
357
|
-
with sync_playwright() as playwright:
|
|
358
|
-
browser = playwright.chromium.launch(headless=_instagram_playwright_headless())
|
|
359
|
-
try:
|
|
360
|
-
context = browser.new_context(
|
|
361
|
-
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0 Safari/537.36',
|
|
362
|
-
extra_http_headers={'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7'},
|
|
363
|
-
)
|
|
364
|
-
page = context.new_page()
|
|
365
|
-
response = page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
|
366
|
-
status = response.status if response else None
|
|
367
|
-
LOGGER.info('Instagram Playwright HTML profile URL=%s status=%s', url, status)
|
|
368
|
-
print(f'Playwright usado no Windows para o perfil @{username}')
|
|
369
|
-
page.wait_for_timeout(5000)
|
|
174
|
+
def metric(patterns: tuple[str, ...]) -> int | None:
|
|
175
|
+
for pattern in patterns:
|
|
176
|
+
found = re.search(pattern, description, flags=re.IGNORECASE)
|
|
177
|
+
if found:
|
|
370
178
|
try:
|
|
371
|
-
|
|
372
|
-
except
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
179
|
+
return int(re.sub(r'[^0-9]', '', found.group(1)))
|
|
180
|
+
except ValueError:
|
|
181
|
+
pass
|
|
182
|
+
return None
|
|
183
|
+
|
|
184
|
+
followers = metric((r'([\d,.]+)\s*(?:mil\s+)?(?:followers|seguidores)',)) or _structured_metric(document, 'edge_followed_by', 'followers', 'follower_count') or _structured_metric_from_json(document, 'edge_followed_by', 'followers', 'follower_count', 'followerCount')
|
|
185
|
+
following = metric((r'([\d,.]+)\s*(?:mil\s+)?(?:following|seguindo)',)) or _structured_metric(document, 'edge_follow', 'follows', 'following', 'following_count', 'followingCount') or _structured_metric_from_json(document, 'edge_follow', 'follows', 'following', 'following_count', 'followingCount')
|
|
186
|
+
post_count = metric((r'([\d,.]+)\s*(?:mil\s+)?(?:posts|publicações|publications)',)) or _structured_metric(document, 'edge_owner_to_timeline_media', 'posts', 'post_count')
|
|
187
|
+
country = ''
|
|
188
|
+
for payload in _embedded_json_documents(document):
|
|
189
|
+
country = extract_public_instagram_country(payload)
|
|
190
|
+
if country:
|
|
191
|
+
break
|
|
192
|
+
posts = _extract_posts_from_html(document, 10)
|
|
193
|
+
user: dict[str, Any] = {
|
|
194
|
+
'username': username,
|
|
195
|
+
'full_name': title.split('(')[0].strip() or username,
|
|
196
|
+
'biography': description,
|
|
197
|
+
'edge_followed_by': {'count': followers} if followers is not None else {},
|
|
198
|
+
'edge_follow': {'count': following} if following is not None else {},
|
|
199
|
+
'edge_owner_to_timeline_media': {'count': post_count, 'edges': [{'node': post} for post in posts]},
|
|
200
|
+
'country': country,
|
|
201
|
+
'profile_pic_url': avatar_url,
|
|
202
|
+
'_html_posts': posts,
|
|
203
|
+
}
|
|
204
|
+
return user if any((description, followers, following, post_count, country, posts)) else {}
|
|
389
205
|
|
|
390
206
|
|
|
391
207
|
def _fetch_web_profile_user(username: str) -> dict[str, Any] | None:
|
|
392
|
-
_ensure_windows_playwright()
|
|
393
208
|
headers = _instagram_headers()
|
|
394
209
|
headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0 Safari/537.36'
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
cookies = _instagram_cookies()
|
|
210
|
+
# The www host is the current public web endpoint. The legacy i host is
|
|
211
|
+
# retained as a fallback because Instagram rate-limits the two hosts
|
|
212
|
+
# independently and their availability can vary by region.
|
|
400
213
|
endpoints = (
|
|
401
214
|
f'https://www.instagram.com/api/v1/users/web_profile_info/?username={username}',
|
|
402
215
|
f'https://i.instagram.com/api/v1/users/web_profile_info/?username={username}',
|
|
403
216
|
)
|
|
217
|
+
def profile_quality(user: Mapping[str, Any]) -> int:
|
|
218
|
+
return sum(bool(user.get(key)) for key in ('biography', 'edge_follow', 'edge_followed_by', 'edge_owner_to_timeline_media', 'username'))
|
|
219
|
+
|
|
220
|
+
def merge_profile(base: dict[str, Any] | None, candidate: Mapping[str, Any]) -> dict[str, Any]:
|
|
221
|
+
merged = dict(base or {})
|
|
222
|
+
for key, value in candidate.items():
|
|
223
|
+
if value not in (None, '', [], {}):
|
|
224
|
+
if isinstance(value, Mapping) and isinstance(merged.get(key), Mapping):
|
|
225
|
+
merged[key] = {**merged[key], **value}
|
|
226
|
+
else:
|
|
227
|
+
merged[key] = value
|
|
228
|
+
return merged
|
|
404
229
|
|
|
405
|
-
request_headers = [
|
|
406
|
-
headers,
|
|
407
|
-
{**headers, 'User-Agent': 'Instagram 390.0.0.0.50 Android', 'Accept': 'application/json, text/plain, */*'},
|
|
408
|
-
{**headers, 'Referer': f'https://www.instagram.com/{username}/', 'Origin': 'https://www.instagram.com', 'X-Requested-With': 'XMLHttpRequest'},
|
|
409
|
-
]
|
|
410
230
|
best_user: dict[str, Any] | None = None
|
|
411
|
-
for
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
LOGGER.debug('Instagram profile URL=%s status=%s content_type=%s', endpoint, response.status_code, response.headers.get('content-type', ''))
|
|
416
|
-
if response.status_code >= 400:
|
|
417
|
-
continue
|
|
418
|
-
payload = response.json()
|
|
419
|
-
LOGGER.debug('Instagram profile URL=%s response=JSON', endpoint)
|
|
420
|
-
except (requests.RequestException, ValueError, AttributeError, StopIteration):
|
|
421
|
-
LOGGER.warning('Falha na requisição JSON do perfil Instagram URL=%s', endpoint, exc_info=True)
|
|
422
|
-
if best_user:
|
|
423
|
-
return best_user
|
|
231
|
+
for endpoint in endpoints:
|
|
232
|
+
try:
|
|
233
|
+
response = requests.get(endpoint, headers=headers, timeout=15)
|
|
234
|
+
if response.status_code >= 400:
|
|
424
235
|
continue
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
236
|
+
payload = response.json()
|
|
237
|
+
except (requests.RequestException, ValueError, AttributeError):
|
|
238
|
+
continue
|
|
239
|
+
user = ((payload.get('data') or {}).get('user') if isinstance(payload, dict) else None)
|
|
240
|
+
if isinstance(user, dict):
|
|
241
|
+
best_user = merge_profile(best_user, user)
|
|
242
|
+
if profile_quality(best_user) >= 4:
|
|
243
|
+
return best_user
|
|
430
244
|
|
|
431
245
|
curl = shutil.which('curl')
|
|
432
246
|
if curl:
|
|
433
|
-
LOGGER.debug('Fallback curl disponível em %s', curl)
|
|
434
247
|
for endpoint in endpoints:
|
|
435
248
|
try:
|
|
436
249
|
completed = subprocess.run(
|
|
@@ -443,20 +256,21 @@ def _fetch_web_profile_user(username: str) -> dict[str, Any] | None:
|
|
|
443
256
|
check=False,
|
|
444
257
|
)
|
|
445
258
|
payload = json.loads(completed.stdout) if completed.returncode == 0 and completed.stdout else None
|
|
446
|
-
|
|
447
|
-
user = _extract_profile_user(payload)
|
|
259
|
+
user = ((payload.get('data') or {}).get('user') if isinstance(payload, dict) else None)
|
|
448
260
|
if isinstance(user, dict):
|
|
449
|
-
best_user =
|
|
450
|
-
if
|
|
261
|
+
best_user = merge_profile(best_user, user)
|
|
262
|
+
if profile_quality(best_user) >= 4:
|
|
451
263
|
return best_user
|
|
452
264
|
except (OSError, subprocess.SubprocessError, ValueError):
|
|
453
|
-
LOGGER.warning('Falha no fallback curl do perfil Instagram URL=%s', endpoint, exc_info=True)
|
|
454
265
|
continue
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
266
|
+
if platform.system() == 'Windows':
|
|
267
|
+
html_content = _fetch_instagram_html_with_playwright(f'https://www.instagram.com/{username}/', username)
|
|
268
|
+
if html_content:
|
|
269
|
+
html_user = _extract_profile_user_from_html(html_content, username)
|
|
270
|
+
if html_user:
|
|
271
|
+
print(f'Perfil Instagram extraído do HTML no Windows para @{username}')
|
|
272
|
+
return html_user
|
|
273
|
+
return best_user
|
|
460
274
|
|
|
461
275
|
|
|
462
276
|
def _country_from_bloks(value: Any) -> str:
|
|
@@ -506,74 +320,25 @@ def _fetch_instagram_about_country(user_id: Any) -> str:
|
|
|
506
320
|
}
|
|
507
321
|
try:
|
|
508
322
|
response = requests.post(ABOUT_ACCOUNT_URL, data=payload, headers=_instagram_headers(), cookies=cookies, timeout=20)
|
|
509
|
-
LOGGER.debug('Instagram about-account URL=%s status=%s content_type=%s', ABOUT_ACCOUNT_URL, response.status_code, response.headers.get('content-type', ''))
|
|
510
323
|
if response.status_code >= 400:
|
|
511
324
|
return ''
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
return country
|
|
515
|
-
except (requests.RequestException, ValueError, TypeError) as exc:
|
|
516
|
-
LOGGER.warning('Falha ao consultar país público do Instagram: %s', exc, exc_info=True)
|
|
325
|
+
return _country_from_bloks(response.json())
|
|
326
|
+
except (requests.RequestException, ValueError, TypeError):
|
|
517
327
|
return ''
|
|
518
328
|
|
|
519
329
|
|
|
520
|
-
def _country_from_bio_fallback(bio: str) -> str:
|
|
521
|
-
patterns = (
|
|
522
|
-
r'\b(?:from|based in|located in|living in)\s+([A-Za-zÀ-ÿ][A-Za-zÀ-ÿ .-]{1,60})',
|
|
523
|
-
r'\b(?:de|baseado em|morando em|localizado em)\s+([A-Za-zÀ-ÿ][A-Za-zÀ-ÿ .-]{1,60})',
|
|
524
|
-
)
|
|
525
|
-
for pattern in patterns:
|
|
526
|
-
match = re.search(pattern, bio or '', flags=re.IGNORECASE)
|
|
527
|
-
if match:
|
|
528
|
-
return match.group(1).strip(' .,!;:')
|
|
529
|
-
return ''
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
def _profile_post_nodes(user: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
|
533
|
-
"""Return post nodes from both legacy GraphQL edges and current items payloads."""
|
|
534
|
-
containers: list[Any] = [
|
|
535
|
-
user.get('edge_owner_to_timeline_media'),
|
|
536
|
-
user.get('timeline_media'),
|
|
537
|
-
user.get('media'),
|
|
538
|
-
user,
|
|
539
|
-
]
|
|
540
|
-
nodes: list[Mapping[str, Any]] = []
|
|
541
|
-
for container in containers:
|
|
542
|
-
if not isinstance(container, Mapping):
|
|
543
|
-
continue
|
|
544
|
-
raw_items = container.get('edges') or container.get('items') or container.get('data') or []
|
|
545
|
-
if isinstance(raw_items, Mapping):
|
|
546
|
-
raw_items = raw_items.get('items') or raw_items.get('edges') or []
|
|
547
|
-
if not isinstance(raw_items, list):
|
|
548
|
-
continue
|
|
549
|
-
for item in raw_items:
|
|
550
|
-
node = item.get('node') if isinstance(item, Mapping) and isinstance(item.get('node'), Mapping) else item
|
|
551
|
-
if isinstance(node, Mapping):
|
|
552
|
-
nodes.append(node)
|
|
553
|
-
return nodes
|
|
554
|
-
|
|
555
|
-
|
|
556
330
|
def _profile_data_from_api(user: Mapping[str, Any], reference: Mapping[str, str]) -> dict[str, Any]:
|
|
557
331
|
followers = _profile_metric(user, 'edge_followed_by', 'followers', 'follower_count', 'followerCount')
|
|
558
|
-
|
|
559
|
-
following = normalize_instagram_metric(following_node) if isinstance(following_node, Mapping) else None
|
|
560
|
-
following = following if following is not None else _profile_metric(user, 'follows', 'following', 'following_count', 'followingCount')
|
|
332
|
+
following = _profile_metric(user, 'edge_follow', 'follows', 'following', 'following_count', 'followingCount')
|
|
561
333
|
followers = followers if followers is not None else _structured_metric_from_json(json.dumps(user), 'edge_followed_by', 'followers', 'follower_count', 'followerCount')
|
|
562
334
|
following = following if following is not None else _structured_metric_from_json(json.dumps(user), 'edge_follow', 'follows', 'following', 'following_count', 'followingCount')
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
biography = str(user.get('biography') or user.get('bio') or user.get('description') or '').strip()
|
|
567
|
-
direct_country = _country_value(user.get('country') or user.get('country_code'))
|
|
568
|
-
country = direct_country or extract_public_instagram_country(user)
|
|
569
|
-
if platform.system() != 'Windows':
|
|
570
|
-
country = country or _fetch_instagram_about_country(user.get('id') or user.get('pk'))
|
|
571
|
-
country = country or _country_from_bio_fallback(biography)
|
|
572
|
-
LOGGER.debug('Instagram profile extracted keys=%s edge_follow=%r bio=%s followers=%s following=%s posts=%s country=%s', sorted(str(key) for key in user.keys()), following_node, bool(biography), followers, following, media, country or '—')
|
|
335
|
+
media = ((user.get('edge_owner_to_timeline_media') or {}).get('count') if isinstance(user.get('edge_owner_to_timeline_media'), dict) else None)
|
|
336
|
+
biography = str(user.get('biography') or '').strip()
|
|
337
|
+
country = extract_public_instagram_country(user) or _fetch_instagram_about_country(user.get('id') or user.get('pk'))
|
|
573
338
|
return {
|
|
574
339
|
'id': f"instagram_{reference['username']}",
|
|
575
340
|
**reference,
|
|
576
|
-
'name': str(user.get('full_name') or user.get('
|
|
341
|
+
'name': str(user.get('full_name') or user.get('username') or reference['username']).strip(),
|
|
577
342
|
'bio': normalize_instagram_bio(biography),
|
|
578
343
|
'bio_raw': biography,
|
|
579
344
|
'country': country,
|
|
@@ -585,7 +350,7 @@ def _profile_data_from_api(user: Mapping[str, Any], reference: Mapping[str, str]
|
|
|
585
350
|
'public_lookup': True,
|
|
586
351
|
'metrics_source': 'instagram_web_profile_info',
|
|
587
352
|
'last_public_lookup_at': datetime.now(timezone.utc).isoformat(),
|
|
588
|
-
'_api_posts':
|
|
353
|
+
'_api_posts': (((user.get('edge_owner_to_timeline_media') or {}).get('edges') or []) if isinstance(user.get('edge_owner_to_timeline_media'), dict) else []),
|
|
589
354
|
}
|
|
590
355
|
|
|
591
356
|
|
|
@@ -600,14 +365,10 @@ def normalize_instagram_metric(value: Any) -> int | None:
|
|
|
600
365
|
|
|
601
366
|
|
|
602
367
|
PUBLIC_COUNTRY_KEYS = {
|
|
603
|
-
'about_this_account_country', 'account_based_in', 'account_country',
|
|
604
|
-
'
|
|
605
|
-
'transparency_country',
|
|
606
|
-
}
|
|
607
|
-
PUBLIC_TRANSPARENCY_KEYS = {
|
|
608
|
-
'about', 'about_account', 'about_this_account', 'about_this_account_data',
|
|
609
|
-
'account_transparency', 'transparency', 'account_info', 'profile_context',
|
|
368
|
+
'about_this_account_country', 'account_based_in', 'account_country', 'country_name',
|
|
369
|
+
'country_of_origin', 'country_of_registration', 'location_country', 'transparency_country',
|
|
610
370
|
}
|
|
371
|
+
PUBLIC_TRANSPARENCY_KEYS = {'about', 'about_account', 'account_transparency', 'transparency', 'account_info'}
|
|
611
372
|
|
|
612
373
|
|
|
613
374
|
def _country_value(value: Any) -> str:
|
|
@@ -651,23 +412,17 @@ def _profile_metric(user: Mapping[str, Any], *keys: str) -> int | None:
|
|
|
651
412
|
|
|
652
413
|
|
|
653
414
|
def fetch_public_instagram_profile(source: str) -> IntegrationResult:
|
|
654
|
-
LOGGER.debug('fetch_public_instagram_profile source=%s', source)
|
|
655
415
|
try:
|
|
656
416
|
reference = normalize_instagram_reference(source)
|
|
657
417
|
except ValueError as exc:
|
|
658
418
|
return IntegrationResult(False, str(exc), {})
|
|
659
419
|
api_user = _fetch_web_profile_user(reference['username'])
|
|
660
420
|
if api_user:
|
|
661
|
-
|
|
662
|
-
return IntegrationResult(True, 'Perfil Instagram encontrado publicamente.', data)
|
|
663
|
-
if platform.system() == 'Windows':
|
|
664
|
-
return IntegrationResult(False, 'Não foi possível obter os dados do Instagram. Verifique sua conexão ou tente novamente mais tarde.', reference)
|
|
421
|
+
return IntegrationResult(True, 'Perfil Instagram encontrado publicamente.', _profile_data_from_api(api_user, reference))
|
|
665
422
|
try:
|
|
666
423
|
response = requests.get(reference['url'], headers={'User-Agent': 'Mozilla/5.0', 'Accept-Language': 'pt-BR,pt;q=0.9,en;q=0.8'}, timeout=12)
|
|
667
424
|
except requests.RequestException as exc:
|
|
668
|
-
LOGGER.warning('Falha HTTP no perfil Instagram URL=%s: %s', reference['url'], exc, exc_info=True)
|
|
669
425
|
return IntegrationResult(False, f'Não foi possível consultar o perfil público do Instagram: {exc}', reference)
|
|
670
|
-
LOGGER.debug('Instagram HTML profile URL=%s status=%s content_type=%s response=HTML', reference['url'], response.status_code, response.headers.get('content-type', ''))
|
|
671
426
|
if response.status_code >= 400:
|
|
672
427
|
return IntegrationResult(False, f'O Instagram devolveu HTTP {response.status_code}. Confirme o @handle ou use o cadastro manual.', reference | {'status_code': response.status_code})
|
|
673
428
|
title = _meta(response.text, 'og:title') or reference['username']
|
|
@@ -692,8 +447,6 @@ def fetch_public_instagram_profile(source: str) -> IntegrationResult:
|
|
|
692
447
|
embedded_country = extract_public_instagram_country(document)
|
|
693
448
|
if embedded_country:
|
|
694
449
|
break
|
|
695
|
-
embedded_country = embedded_country or _country_from_bio_fallback(description)
|
|
696
|
-
LOGGER.debug('Instagram HTML profile extracted bio=%s followers=%s following=%s posts=%s country=%s', bool(description), followers, following, posts, embedded_country or '—')
|
|
697
450
|
data = {'id': f"instagram_{reference['username']}", **reference, 'name': title.split('(')[0].strip() or reference['username'], 'bio': normalize_instagram_bio(description), 'bio_raw': description, 'country': embedded_country, 'avatar_url': avatar_url, 'subscriber_count': followers, 'following_count': following, 'post_count': posts, 'public_lookup': True, 'metrics_source': 'instagram_public_page', 'last_public_lookup_at': datetime.now(timezone.utc).isoformat()}
|
|
698
451
|
return IntegrationResult(True, 'Perfil Instagram encontrado publicamente.', data)
|
|
699
452
|
|
|
@@ -755,73 +508,35 @@ def _post_from_node(node: dict[str, Any]) -> dict[str, Any] | None:
|
|
|
755
508
|
}
|
|
756
509
|
|
|
757
510
|
|
|
758
|
-
def _fetch_posts_with_playwright(reference: Mapping[str, str], limit: int) -> list[dict[str, Any]]:
|
|
759
|
-
"""Load a public profile page and extract posts only from its rendered HTML."""
|
|
760
|
-
url = str(reference.get('url') or '')
|
|
761
|
-
try:
|
|
762
|
-
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
|
|
763
|
-
with sync_playwright() as playwright:
|
|
764
|
-
browser = playwright.chromium.launch(headless=_instagram_playwright_headless())
|
|
765
|
-
try:
|
|
766
|
-
context = browser.new_context(
|
|
767
|
-
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0 Safari/537.36',
|
|
768
|
-
extra_http_headers={'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7'},
|
|
769
|
-
)
|
|
770
|
-
page = context.new_page()
|
|
771
|
-
response = page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
|
772
|
-
status = response.status if response else None
|
|
773
|
-
LOGGER.info('Instagram Playwright HTML posts URL=%s status=%s', url, status)
|
|
774
|
-
print(f'Playwright usado no Windows para os posts de @{reference.get("username", "")}'.strip())
|
|
775
|
-
page.wait_for_timeout(5000)
|
|
776
|
-
try:
|
|
777
|
-
page.wait_for_load_state('networkidle', timeout=10000)
|
|
778
|
-
except (PlaywrightTimeoutError, TimeoutError):
|
|
779
|
-
LOGGER.debug('Instagram Playwright networkidle excedido; usando HTML já carregado para posts.')
|
|
780
|
-
html_content = page.content()
|
|
781
|
-
_save_instagram_debug_html(reference.get('username', 'instagram'), html_content)
|
|
782
|
-
if status is not None and status >= 400:
|
|
783
|
-
return []
|
|
784
|
-
return _extract_posts_from_html(html_content, limit)
|
|
785
|
-
finally:
|
|
786
|
-
browser.close()
|
|
787
|
-
except Exception as exc:
|
|
788
|
-
LOGGER.warning('Falha no Playwright HTML dos posts Instagram URL=%s: %s', url, exc, exc_info=True)
|
|
789
|
-
print(f'Playwright falhou no Windows para os posts: {exc}')
|
|
790
|
-
_streamlit_message('error', f'Playwright não conseguiu consultar os posts Instagram: {exc}', 'posts_playwright_error')
|
|
791
|
-
return []
|
|
792
|
-
|
|
793
|
-
|
|
794
511
|
def fetch_public_instagram_posts(source: str, limit: int = 10) -> IntegrationResult:
|
|
795
|
-
LOGGER.debug('fetch_public_instagram_posts source=%s limit=%s', source, limit)
|
|
796
|
-
_ensure_windows_playwright()
|
|
797
512
|
try:
|
|
798
513
|
reference = normalize_instagram_reference(source)
|
|
799
514
|
except ValueError as exc:
|
|
800
515
|
return IntegrationResult(False, str(exc), {})
|
|
801
|
-
if platform.system() == 'Windows':
|
|
802
|
-
LOGGER.debug('Windows detectado: posts Instagram exclusivamente via HTML Playwright.')
|
|
803
|
-
posts = _fetch_posts_with_playwright(reference, limit)
|
|
804
|
-
if not posts:
|
|
805
|
-
_streamlit_message('error', 'Não foi possível obter os dados do Instagram. Verifique sua conexão ou tente novamente mais tarde.', 'posts_instagram_failed')
|
|
806
|
-
return IntegrationResult(bool(posts), 'Posts públicos encontrados.' if posts else 'Não foi possível obter os dados do Instagram. Verifique sua conexão ou tente novamente mais tarde.', reference | {'posts': posts[:max(1, int(limit))]})
|
|
807
516
|
api_user = _fetch_web_profile_user(reference['username'])
|
|
808
517
|
if api_user:
|
|
809
518
|
api_posts: list[dict[str, Any]] = []
|
|
810
|
-
for
|
|
811
|
-
|
|
812
|
-
if
|
|
813
|
-
|
|
519
|
+
for edge in ((api_user.get('edge_owner_to_timeline_media') or {}).get('edges') or []):
|
|
520
|
+
node = edge.get('node') if isinstance(edge, dict) else None
|
|
521
|
+
if isinstance(node, dict):
|
|
522
|
+
post = _post_from_node(node)
|
|
523
|
+
if post:
|
|
524
|
+
api_posts.append(post)
|
|
814
525
|
if api_posts:
|
|
815
|
-
LOGGER.debug('Instagram API posts extracted=%s response=JSON', len(api_posts))
|
|
816
526
|
return IntegrationResult(True, 'Posts públicos encontrados.', reference | {'posts': api_posts[:max(1, int(limit))]})
|
|
817
527
|
if api_user.get('is_private'):
|
|
818
528
|
return IntegrationResult(False, 'Esta conta é privada. O Instagram não disponibiliza posts sem uma sessão autenticada.', reference | {'posts': [], 'is_private': True})
|
|
529
|
+
if platform.system() == 'Windows':
|
|
530
|
+
html_posts = api_user.get('_html_posts') or []
|
|
531
|
+
return IntegrationResult(bool(html_posts), 'Posts públicos encontrados.' if html_posts else 'Não foi possível encontrar posts públicos nesta página do Instagram.', reference | {'posts': html_posts[:max(1, int(limit))]})
|
|
532
|
+
if platform.system() == 'Windows':
|
|
533
|
+
html_content = _fetch_instagram_html_with_playwright(reference['url'], reference['username'])
|
|
534
|
+
posts = _extract_posts_from_html(html_content, limit) if html_content else []
|
|
535
|
+
return IntegrationResult(bool(posts), 'Posts públicos encontrados.' if posts else 'Não foi possível encontrar posts públicos nesta página do Instagram.', reference | {'posts': posts[:max(1, int(limit))]})
|
|
819
536
|
try:
|
|
820
|
-
response = requests.get(reference['url'], headers={'User-Agent': 'Mozilla/5.0', 'Accept-Language': 'pt-BR,pt;q=0.9,en;q=0.8'},
|
|
537
|
+
response = requests.get(reference['url'], headers={'User-Agent': 'Mozilla/5.0', 'Accept-Language': 'pt-BR,pt;q=0.9,en;q=0.8'}, timeout=15)
|
|
821
538
|
except requests.RequestException as exc:
|
|
822
|
-
LOGGER.warning('Falha HTTP nos posts Instagram URL=%s: %s', reference['url'], exc, exc_info=True)
|
|
823
539
|
return IntegrationResult(False, f'Não foi possível consultar os posts públicos do Instagram: {exc}', reference)
|
|
824
|
-
LOGGER.debug('Instagram HTML posts URL=%s status=%s content_type=%s response=HTML', reference['url'], response.status_code, response.headers.get('content-type', ''))
|
|
825
540
|
if response.status_code >= 400:
|
|
826
541
|
return IntegrationResult(False, f'O Instagram devolveu HTTP {response.status_code} ao consultar os posts.', reference | {'status_code': response.status_code})
|
|
827
542
|
posts: list[dict[str, Any]] = []
|
|
@@ -837,10 +552,7 @@ def fetch_public_instagram_posts(source: str, limit: int = 10) -> IntegrationRes
|
|
|
837
552
|
break
|
|
838
553
|
if len(posts) >= max(1, int(limit)):
|
|
839
554
|
break
|
|
840
|
-
if
|
|
841
|
-
posts = _fetch_posts_with_playwright(reference, limit)
|
|
842
|
-
LOGGER.debug('Instagram posts extracted=%s limit=%s', len(posts), limit)
|
|
843
|
-
return IntegrationResult(bool(posts), 'Posts públicos encontrados.' if posts else 'Não foi possível obter os dados do Instagram. Verifique sua conexão ou tente novamente mais tarde.', reference | {'posts': posts[:max(1, int(limit))]})
|
|
555
|
+
return IntegrationResult(bool(posts), 'Posts públicos encontrados.' if posts else 'Não foi possível encontrar posts públicos nesta página do Instagram.', reference | {'posts': posts[:max(1, int(limit))]})
|
|
844
556
|
|
|
845
557
|
|
|
846
558
|
__all__ = ['IntegrationResult', 'extract_public_instagram_country', 'fetch_public_instagram_posts', 'fetch_public_instagram_profile', 'normalize_instagram_bio', 'normalize_instagram_metric', 'normalize_instagram_reference']
|
package/package.json
CHANGED