@danhachuel/thunderbolt 0.6.57 → 0.6.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/social_networks_ui.py +23 -123
- package/integrations/instagram_public.py +167 -421
- package/package.json +1 -1
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import io
|
|
4
|
-
import logging
|
|
5
4
|
import re
|
|
6
5
|
import uuid
|
|
7
6
|
import zipfile
|
|
8
7
|
import unicodedata
|
|
9
|
-
import hashlib
|
|
10
8
|
from typing import Any, Callable, Mapping
|
|
11
9
|
|
|
12
10
|
import requests
|
|
@@ -17,19 +15,10 @@ from hermes_ui.influencers import InfluencerBackendError, STANDALONE_CONTENT_INF
|
|
|
17
15
|
from hermes_ui.storage import read_json, write_json
|
|
18
16
|
from hermes_ui.countries import COUNTRY_OPTIONS
|
|
19
17
|
from hermes_ui.languages import LANGUAGE_CODES, language_code, language_label
|
|
20
|
-
from integrations.instagram_public import
|
|
21
|
-
extract_public_instagram_country,
|
|
22
|
-
fetch_public_instagram_posts,
|
|
23
|
-
fetch_public_instagram_profile,
|
|
24
|
-
normalize_instagram_bio,
|
|
25
|
-
normalize_instagram_metric,
|
|
26
|
-
)
|
|
18
|
+
from integrations.instagram_public import fetch_public_instagram_posts, fetch_public_instagram_profile, normalize_instagram_bio, normalize_instagram_metric
|
|
27
19
|
from integrations.meta_social import test_facebook_pages_api_card, test_instagram_api_card
|
|
28
20
|
|
|
29
21
|
|
|
30
|
-
LOGGER = logging.getLogger(__name__)
|
|
31
|
-
|
|
32
|
-
|
|
33
22
|
def _clean(value: Any) -> str:
|
|
34
23
|
return str(value or "").strip()
|
|
35
24
|
|
|
@@ -78,35 +67,11 @@ def _country_key(value: Any) -> str:
|
|
|
78
67
|
COUNTRY_ALIASES = {
|
|
79
68
|
"brazil": "Brasil",
|
|
80
69
|
"brasil": "Brasil",
|
|
81
|
-
"br": "Brasil",
|
|
82
70
|
"usa": "Estados Unidos",
|
|
83
71
|
"us": "Estados Unidos",
|
|
84
72
|
"united states": "Estados Unidos",
|
|
85
73
|
"united states of america": "Estados Unidos",
|
|
86
|
-
"pt": "Portugal",
|
|
87
74
|
"portugal": "Portugal",
|
|
88
|
-
"uk": "Reino Unido",
|
|
89
|
-
"gb": "Reino Unido",
|
|
90
|
-
"united kingdom": "Reino Unido",
|
|
91
|
-
"england": "Reino Unido",
|
|
92
|
-
"germany": "Alemanha",
|
|
93
|
-
"de": "Alemanha",
|
|
94
|
-
"france": "França",
|
|
95
|
-
"fr": "França",
|
|
96
|
-
"italy": "Itália",
|
|
97
|
-
"it": "Itália",
|
|
98
|
-
"spain": "Espanha",
|
|
99
|
-
"es": "Espanha",
|
|
100
|
-
"canada": "Canadá",
|
|
101
|
-
"ca": "Canadá",
|
|
102
|
-
"australia": "Austrália",
|
|
103
|
-
"au": "Austrália",
|
|
104
|
-
"japan": "Japão",
|
|
105
|
-
"jp": "Japão",
|
|
106
|
-
"mexico": "México",
|
|
107
|
-
"mx": "México",
|
|
108
|
-
"argentina": "Argentina",
|
|
109
|
-
"ar": "Argentina",
|
|
110
75
|
}
|
|
111
76
|
|
|
112
77
|
|
|
@@ -124,17 +89,11 @@ def _normalise_country(value: Any) -> str:
|
|
|
124
89
|
|
|
125
90
|
|
|
126
91
|
def _profile_country(profile: Mapping[str, Any]) -> str:
|
|
127
|
-
for key in ("country", "country_name", "country_of_registration", "account_country", "account_based_in"
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
for candidate in candidates:
|
|
133
|
-
country = _normalise_country(candidate)
|
|
134
|
-
if country:
|
|
135
|
-
return country
|
|
136
|
-
extracted = _normalise_country(extract_public_instagram_country(profile))
|
|
137
|
-
return extracted
|
|
92
|
+
for key in ("country", "country_name", "country_of_registration", "account_country", "account_based_in"):
|
|
93
|
+
country = _normalise_country(profile.get(key))
|
|
94
|
+
if country:
|
|
95
|
+
return country
|
|
96
|
+
return ""
|
|
138
97
|
|
|
139
98
|
|
|
140
99
|
def _language_index(value: Any) -> int:
|
|
@@ -165,41 +124,10 @@ def _instagram_profile_storage_id(profile: Mapping[str, Any]) -> str:
|
|
|
165
124
|
return re.sub(r"[^A-Za-z0-9_.-]+", "_", value) or "unknown"
|
|
166
125
|
|
|
167
126
|
|
|
168
|
-
def _clear_instagram_result_widget_state(profile: Mapping[str, Any]) -> None:
|
|
169
|
-
"""Clear every previous widget key for this profile before rebuilding the form."""
|
|
170
|
-
widget_id = _instagram_profile_storage_id(profile)
|
|
171
|
-
for field in ("name", "bio", "country", "language", "posts", "followers", "following", "character"):
|
|
172
|
-
prefix = f"social_instagram_result_{field}_{widget_id}"
|
|
173
|
-
for key in list(st.session_state.keys()):
|
|
174
|
-
if str(key).startswith(prefix):
|
|
175
|
-
st.session_state.pop(key, None)
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
def _instagram_result_widget_id(profile: Mapping[str, Any]) -> str:
|
|
179
|
-
base_id = _instagram_profile_storage_id(profile)
|
|
180
|
-
signature = "|".join(_clean(profile.get(key)) for key in ("bio", "bio_raw", "following_count", "subscriber_count", "post_count", "country"))
|
|
181
|
-
digest = hashlib.sha1(signature.encode("utf-8")).hexdigest()[:10]
|
|
182
|
-
return f"{base_id}_{digest}"
|
|
183
|
-
|
|
184
|
-
|
|
185
127
|
def _normalise_instagram_posts(value: Any) -> list[dict[str, Any]]:
|
|
186
|
-
if isinstance(value, Mapping):
|
|
187
|
-
value = value.get("posts") or value.get("items") or value.get("edges") or []
|
|
188
128
|
if not isinstance(value, list):
|
|
189
129
|
return []
|
|
190
|
-
|
|
191
|
-
seen: set[str] = set()
|
|
192
|
-
for item in value:
|
|
193
|
-
post = item.get("node") if isinstance(item, Mapping) and isinstance(item.get("node"), Mapping) else item
|
|
194
|
-
if not isinstance(post, Mapping):
|
|
195
|
-
continue
|
|
196
|
-
data = dict(post)
|
|
197
|
-
identifier = _clean(data.get("id") or data.get("shortcode") or data.get("url"))
|
|
198
|
-
if not identifier or identifier in seen:
|
|
199
|
-
continue
|
|
200
|
-
seen.add(identifier)
|
|
201
|
-
normalised.append(data)
|
|
202
|
-
return normalised
|
|
130
|
+
return [dict(post) for post in value if isinstance(post, Mapping) and _clean(post.get("id") or post.get("url"))]
|
|
203
131
|
|
|
204
132
|
|
|
205
133
|
def _stored_instagram_posts(profile: Mapping[str, Any]) -> list[dict[str, Any]]:
|
|
@@ -248,8 +176,8 @@ def _instagram_posts_archive(posts: list[dict[str, Any]]) -> bytes:
|
|
|
248
176
|
elif "webp" in content_type:
|
|
249
177
|
extension = ".webp"
|
|
250
178
|
bundle.writestr(f"post-{index:02d}{extension}", response.content)
|
|
251
|
-
except requests.RequestException
|
|
252
|
-
|
|
179
|
+
except requests.RequestException:
|
|
180
|
+
pass
|
|
253
181
|
manifest.append(f"{index}. {_clean(post.get('url'))}\n{_clean(post.get('caption'))}\n")
|
|
254
182
|
bundle.writestr("posts.txt", "\n".join(manifest))
|
|
255
183
|
return archive.getvalue()
|
|
@@ -270,14 +198,13 @@ def _render_instagram_posts(profile: Mapping[str, Any]) -> None:
|
|
|
270
198
|
if load_clicked or refresh_clicked:
|
|
271
199
|
target = max(10, len(posts)) if refresh_clicked else 10
|
|
272
200
|
posts_ok, posts_message, loaded_posts = _load_instagram_posts(profile, limit=target)
|
|
273
|
-
# DEBUG WINDOWS INSTAGRAM: manter até a validação final do carregamento público.
|
|
274
|
-
st.write({"instagram_posts_ok": posts_ok, "instagram_posts_message": posts_message, "instagram_posts_count": len(loaded_posts)})
|
|
275
201
|
if posts_ok:
|
|
276
|
-
posts =
|
|
202
|
+
posts = loaded_posts
|
|
277
203
|
_persist_instagram_posts(profile, posts)
|
|
278
204
|
st.success(f"{len(posts)} posts carregados.")
|
|
205
|
+
st.rerun()
|
|
279
206
|
else:
|
|
280
|
-
st.
|
|
207
|
+
st.warning(posts_message)
|
|
281
208
|
posts = st.session_state.get(state_key, posts)
|
|
282
209
|
if not posts:
|
|
283
210
|
st.info("Clique em Carregar últimos 10 para consultar os posts públicos desta conta.")
|
|
@@ -296,10 +223,9 @@ def _render_instagram_posts(profile: Mapping[str, Any]) -> None:
|
|
|
296
223
|
if st.button("Mostrar + 10", key=f"more_instagram_posts_{profile_id}", use_container_width=True):
|
|
297
224
|
posts_ok, posts_message, loaded_posts = _load_instagram_posts(profile, limit=len(posts) + 10)
|
|
298
225
|
if posts_ok:
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
st.error(posts_message)
|
|
226
|
+
_persist_instagram_posts(profile, loaded_posts)
|
|
227
|
+
st.rerun()
|
|
228
|
+
st.warning(posts_message)
|
|
303
229
|
|
|
304
230
|
|
|
305
231
|
def _api_card_defaults(kind: str) -> tuple[str, list[str], Callable[[Mapping[str, Any]], dict[str, Any]]]:
|
|
@@ -446,8 +372,7 @@ def _characters(settings: Mapping[str, Any]) -> tuple[list[dict[str, Any]], Any
|
|
|
446
372
|
repository = get_repository(settings)
|
|
447
373
|
items = [item for item in repository.list_influencers() if _clean(item.get("id")) != STANDALONE_CONTENT_INFLUENCER_ID]
|
|
448
374
|
return items, repository
|
|
449
|
-
except Exception
|
|
450
|
-
LOGGER.warning("Não foi possível carregar as personagens: %s", exc, exc_info=True)
|
|
375
|
+
except (InfluencerBackendError, Exception):
|
|
451
376
|
return [], None
|
|
452
377
|
|
|
453
378
|
|
|
@@ -559,39 +484,19 @@ def _render_instagram_card(profile: dict[str, Any], characters: list[dict[str, A
|
|
|
559
484
|
refresh_col, edit_col = st.columns(2)
|
|
560
485
|
with refresh_col:
|
|
561
486
|
if st.button("↻", help="Actualizar posts, seguidores e seguindo", key=f"refresh_instagram_{profile_id}"):
|
|
562
|
-
|
|
563
|
-
refreshed_ok, refreshed_message, refreshed = _refresh_instagram_profile(profile)
|
|
564
|
-
except Exception as exc:
|
|
565
|
-
st.error(f"Falha ao actualizar o perfil Instagram: {exc}")
|
|
566
|
-
refreshed_ok, refreshed_message, refreshed = False, str(exc), {}
|
|
487
|
+
refreshed_ok, refreshed_message, refreshed = _refresh_instagram_profile(profile)
|
|
567
488
|
if refreshed_ok:
|
|
568
489
|
current_posts = _stored_instagram_posts(profile)
|
|
569
|
-
|
|
570
|
-
posts_ok, posts_message, refreshed_posts = _load_instagram_posts(refreshed, limit=max(10, len(current_posts)))
|
|
571
|
-
except Exception as exc:
|
|
572
|
-
posts_ok, posts_message, refreshed_posts = False, str(exc), []
|
|
490
|
+
posts_ok, _, refreshed_posts = _load_instagram_posts(refreshed, limit=max(10, len(current_posts)))
|
|
573
491
|
if posts_ok:
|
|
574
492
|
_persist_instagram_posts(profile, refreshed_posts)
|
|
575
|
-
|
|
576
|
-
st.warning(f"Perfil actualizado, mas os posts não foram actualizados: {posts_message}")
|
|
577
|
-
# Debug temporário removido após validação; manter para reactivar se necessário:
|
|
578
|
-
# st.write({"profile_ok": refreshed_ok, "profile_message": refreshed_message, "profile_data": refreshed})
|
|
579
|
-
# st.write({"posts_ok": posts_ok, "posts_message": posts_message, "posts_count": len(refreshed_posts)})
|
|
580
|
-
try:
|
|
581
|
-
updated = update_channel(profile_id, {**refreshed, "id": profile_id, "country": _profile_country(refreshed) or _profile_country(profile), "language": profile.get("language", ""), "character_id": profile.get("character_id", "")})
|
|
582
|
-
except (OSError, PermissionError) as exc:
|
|
583
|
-
st.error(f"Não foi possível guardar os dados no Windows: {exc}")
|
|
584
|
-
return
|
|
585
|
-
except Exception as exc:
|
|
586
|
-
st.error(f"Falha ao guardar os dados da conta Instagram: {exc}")
|
|
587
|
-
return
|
|
493
|
+
updated = update_channel(profile_id, {**refreshed, "id": profile_id, "country": _profile_country(refreshed) or _profile_country(profile), "language": profile.get("language", ""), "character_id": profile.get("character_id", "")})
|
|
588
494
|
if updated is None:
|
|
589
|
-
st.error("A conta Instagram já não existe no armazenamento local
|
|
495
|
+
st.error("A conta Instagram já não existe no armazenamento local.")
|
|
590
496
|
return
|
|
591
497
|
st.success("Métricas Instagram actualizadas.")
|
|
592
498
|
st.rerun()
|
|
593
|
-
|
|
594
|
-
st.error(f"Não foi possível actualizar o perfil Instagram: {refreshed_message}")
|
|
499
|
+
st.warning(refreshed_message)
|
|
595
500
|
with edit_col:
|
|
596
501
|
if st.button("Editar", key=f"edit_instagram_button_{profile_id}", use_container_width=True):
|
|
597
502
|
for field in ("name", "handle", "bio", "country", "language", "posts", "followers", "following"):
|
|
@@ -678,20 +583,15 @@ def render_social_networks(settings: dict[str, Any]) -> None:
|
|
|
678
583
|
if search_clicked:
|
|
679
584
|
result = fetch_public_instagram_profile(source)
|
|
680
585
|
result_data = dict(result.data) if isinstance(result.data, dict) else {}
|
|
681
|
-
if result_data:
|
|
682
|
-
_clear_instagram_result_widget_state(result_data)
|
|
683
586
|
st.session_state["social_instagram_result"] = result_data
|
|
684
587
|
st.session_state["social_instagram_ok"] = result.ok
|
|
685
588
|
st.session_state["social_instagram_message"] = result.message
|
|
686
589
|
if st.session_state.get("social_instagram_message"):
|
|
687
|
-
if st.session_state.get("social_instagram_ok")
|
|
688
|
-
st.success(st.session_state["social_instagram_message"])
|
|
689
|
-
else:
|
|
690
|
-
st.error(st.session_state["social_instagram_message"])
|
|
590
|
+
(st.success if st.session_state.get("social_instagram_ok") else st.warning)(st.session_state["social_instagram_message"])
|
|
691
591
|
data = st.session_state.get("social_instagram_result", {}) if st.session_state.get("social_instagram_ok") else {}
|
|
692
592
|
if data:
|
|
693
593
|
characters, _ = _characters(settings)
|
|
694
|
-
result_widget_id =
|
|
594
|
+
result_widget_id = _instagram_profile_storage_id(data)
|
|
695
595
|
with st.container(border=True):
|
|
696
596
|
st.subheader("Conta Instagram encontrada")
|
|
697
597
|
preview_cols = st.columns([0.8, 2.2, 1.2, 1.2, 1.2])
|
|
@@ -2,107 +2,23 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import re
|
|
4
4
|
import json
|
|
5
|
-
import logging
|
|
6
5
|
import os
|
|
7
6
|
import platform
|
|
8
|
-
import sys
|
|
9
7
|
import shutil
|
|
10
8
|
import subprocess
|
|
11
9
|
from dataclasses import dataclass
|
|
12
10
|
from datetime import datetime, timezone
|
|
13
11
|
from html import unescape
|
|
14
|
-
from pathlib import Path
|
|
15
12
|
from typing import Any, Mapping
|
|
16
13
|
from urllib.parse import urlparse
|
|
17
14
|
|
|
18
15
|
import requests
|
|
19
16
|
|
|
20
|
-
try:
|
|
21
|
-
import streamlit as st
|
|
22
|
-
except ImportError: # pragma: no cover - Streamlit is a runtime dependency
|
|
23
|
-
st = None
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
LOGGER = logging.getLogger(__name__)
|
|
27
|
-
_WINDOWS_PLAYWRIGHT_CHECKED = False
|
|
28
|
-
|
|
29
17
|
|
|
30
18
|
ABOUT_ACCOUNT_URL = 'https://i.instagram.com/api/v1/bloks/apps/com.instagram.interactions.about_this_account/'
|
|
31
19
|
ABOUT_ACCOUNT_BLOKS_VERSION = '8ca96ca267e30c02cf90888d91eeff09627f0e3fd2bd9df472278c9a6c022cbb'
|
|
32
20
|
|
|
33
21
|
|
|
34
|
-
def _load_dotenv_compat() -> None:
|
|
35
|
-
"""Load .env when python-dotenv exists, otherwise parse simple KEY=VALUE lines."""
|
|
36
|
-
try:
|
|
37
|
-
from dotenv import load_dotenv
|
|
38
|
-
load_dotenv()
|
|
39
|
-
return
|
|
40
|
-
except ImportError:
|
|
41
|
-
LOGGER.debug('python-dotenv não está instalado; será usado o parser .env compatível.')
|
|
42
|
-
env_path = Path(__file__).resolve().parents[1] / '.env'
|
|
43
|
-
if not env_path.is_file():
|
|
44
|
-
return
|
|
45
|
-
try:
|
|
46
|
-
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
|
|
47
|
-
line = raw_line.strip()
|
|
48
|
-
if not line or line.startswith('#') or '=' not in line:
|
|
49
|
-
continue
|
|
50
|
-
key, value = line.split('=', 1)
|
|
51
|
-
value = value.strip().strip('"').strip("'")
|
|
52
|
-
os.environ.setdefault(key.strip(), value)
|
|
53
|
-
except OSError as exc:
|
|
54
|
-
LOGGER.warning('Não foi possível ler %s: %s', env_path, exc)
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def _streamlit_message(level: str, message: str, key: str) -> None:
|
|
58
|
-
"""Show a runtime diagnostic once when Streamlit is available."""
|
|
59
|
-
if st is None:
|
|
60
|
-
return
|
|
61
|
-
try:
|
|
62
|
-
shown_key = f'_instagram_debug_{key}'
|
|
63
|
-
if st.session_state.get(shown_key):
|
|
64
|
-
return
|
|
65
|
-
st.session_state[shown_key] = True
|
|
66
|
-
getattr(st, level)(message)
|
|
67
|
-
except Exception as exc: # Streamlit may be imported outside an active app context.
|
|
68
|
-
LOGGER.debug('Não foi possível mostrar diagnóstico Streamlit: %s', exc)
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
def _ensure_windows_playwright() -> None:
|
|
72
|
-
"""Ensure Chromium exists on Windows before using the Playwright fallback."""
|
|
73
|
-
global _WINDOWS_PLAYWRIGHT_CHECKED
|
|
74
|
-
if os.name != 'nt' or _WINDOWS_PLAYWRIGHT_CHECKED:
|
|
75
|
-
return
|
|
76
|
-
_WINDOWS_PLAYWRIGHT_CHECKED = True
|
|
77
|
-
_load_dotenv_compat()
|
|
78
|
-
try:
|
|
79
|
-
completed = subprocess.run(
|
|
80
|
-
[sys.executable, '-m', 'playwright', 'install', 'chromium'],
|
|
81
|
-
capture_output=True,
|
|
82
|
-
text=True,
|
|
83
|
-
encoding='utf-8',
|
|
84
|
-
errors='replace',
|
|
85
|
-
timeout=180,
|
|
86
|
-
check=False,
|
|
87
|
-
)
|
|
88
|
-
if completed.returncode != 0:
|
|
89
|
-
detail = (completed.stderr or completed.stdout or 'sem detalhes').strip()[-1000:]
|
|
90
|
-
message = f'Não foi possível instalar o Chromium do Playwright no Windows: {detail}'
|
|
91
|
-
LOGGER.warning(message)
|
|
92
|
-
_streamlit_message('error', message + ' Execute: python -m playwright install chromium', 'playwright_install_error')
|
|
93
|
-
else:
|
|
94
|
-
LOGGER.info('Chromium do Playwright verificado/instalado no Windows.')
|
|
95
|
-
except (OSError, subprocess.SubprocessError) as exc:
|
|
96
|
-
message = f'Falha ao executar a instalação do Chromium no Windows: {exc}'
|
|
97
|
-
LOGGER.warning(message)
|
|
98
|
-
_streamlit_message('error', message + ' Execute: python -m playwright install chromium', 'playwright_install_exception')
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
_load_dotenv_compat()
|
|
102
|
-
if os.name == 'nt':
|
|
103
|
-
_ensure_windows_playwright()
|
|
104
|
-
|
|
105
|
-
|
|
106
22
|
def _instagram_cookies() -> dict[str, str]:
|
|
107
23
|
cookies = {}
|
|
108
24
|
sessionid = os.getenv('INSTAGRAM_SESSIONID') or os.getenv('IG_SESSIONID')
|
|
@@ -124,21 +40,34 @@ def _instagram_headers() -> dict[str, str]:
|
|
|
124
40
|
}
|
|
125
41
|
|
|
126
42
|
|
|
127
|
-
def
|
|
128
|
-
"""
|
|
129
|
-
value = os.getenv('INSTAGRAM_PLAYWRIGHT_HEADLESS', 'true').strip().casefold()
|
|
130
|
-
return value not in {'0', 'false', 'no', 'off'}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def _save_instagram_debug_html(username: str, document: str) -> None:
|
|
134
|
-
storage_dir = Path(__file__).resolve().parents[1] / 'storage'
|
|
135
|
-
path = Path(os.getenv('INSTAGRAM_DEBUG_PROFILE_HTML', str(storage_dir / 'debug_instagram.html')))
|
|
43
|
+
def _fetch_instagram_html_with_playwright(url: str, username: str) -> str:
|
|
44
|
+
"""Windows-only browser fallback; the existing Linux HTML parser remains authoritative."""
|
|
136
45
|
try:
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
46
|
+
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
|
|
47
|
+
with sync_playwright() as playwright:
|
|
48
|
+
browser = playwright.chromium.launch(headless=True)
|
|
49
|
+
try:
|
|
50
|
+
context = browser.new_context(
|
|
51
|
+
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0 Safari/537.36',
|
|
52
|
+
extra_http_headers={'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7'},
|
|
53
|
+
)
|
|
54
|
+
page = context.new_page()
|
|
55
|
+
response = page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
|
56
|
+
status = response.status if response else None
|
|
57
|
+
print(f'Playwright usado no Windows para @{username}; status HTML={status}')
|
|
58
|
+
page.wait_for_timeout(5000)
|
|
59
|
+
try:
|
|
60
|
+
page.wait_for_load_state('networkidle', timeout=10000)
|
|
61
|
+
except (PlaywrightTimeoutError, TimeoutError):
|
|
62
|
+
pass
|
|
63
|
+
html_content = page.content()
|
|
64
|
+
print(f'HTML Instagram recebido no Windows para @{username}: {len(html_content)} bytes')
|
|
65
|
+
return html_content
|
|
66
|
+
finally:
|
|
67
|
+
browser.close()
|
|
68
|
+
except Exception as exc:
|
|
69
|
+
print(f'Playwright falhou no Windows para @{username}: {exc}')
|
|
70
|
+
return ''
|
|
142
71
|
|
|
143
72
|
|
|
144
73
|
@dataclass
|
|
@@ -149,9 +78,8 @@ class IntegrationResult:
|
|
|
149
78
|
|
|
150
79
|
|
|
151
80
|
def normalize_instagram_bio(value: Any) -> str:
|
|
152
|
-
"""
|
|
153
|
-
|
|
154
|
-
return bio
|
|
81
|
+
"""Preserve the complete biography returned by Instagram."""
|
|
82
|
+
return str(value or '').strip()
|
|
155
83
|
|
|
156
84
|
|
|
157
85
|
def normalize_instagram_reference(source: str) -> dict[str, str]:
|
|
@@ -221,102 +149,14 @@ def _structured_metric_from_json(document: str, *keys: str) -> int | None:
|
|
|
221
149
|
return None
|
|
222
150
|
|
|
223
151
|
|
|
224
|
-
def
|
|
225
|
-
groups = (
|
|
226
|
-
('biography', 'bio', 'description'),
|
|
227
|
-
('edge_follow', 'following', 'following_count', 'followingCount', 'follows'),
|
|
228
|
-
('edge_followed_by', 'followers', 'follower_count', 'followers_count', 'followerCount'),
|
|
229
|
-
('edge_owner_to_timeline_media', 'posts', 'post_count', 'media_count'),
|
|
230
|
-
('username', 'user_name', 'handle'),
|
|
231
|
-
)
|
|
232
|
-
return sum(any(user.get(key) not in (None, '', {}, []) for key in group) for group in groups)
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
def _merge_profile(base: dict[str, Any] | None, candidate: Mapping[str, Any]) -> dict[str, Any]:
|
|
236
|
-
merged = dict(base or {})
|
|
237
|
-
for key, value in candidate.items():
|
|
238
|
-
if value not in (None, '', [], {}):
|
|
239
|
-
if isinstance(value, Mapping) and isinstance(merged.get(key), Mapping):
|
|
240
|
-
merged[key] = {**merged[key], **value}
|
|
241
|
-
else:
|
|
242
|
-
merged[key] = value
|
|
243
|
-
return merged
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
def _extract_profile_user(payload: Any) -> Mapping[str, Any] | None:
|
|
247
|
-
if not isinstance(payload, Mapping):
|
|
248
|
-
return None
|
|
249
|
-
candidates = [
|
|
250
|
-
payload.get('user'),
|
|
251
|
-
(payload.get('data') or {}).get('user') if isinstance(payload.get('data'), Mapping) else None,
|
|
252
|
-
(payload.get('graphql') or {}).get('user') if isinstance(payload.get('graphql'), Mapping) else None,
|
|
253
|
-
(payload.get('data') or {}).get('profile') if isinstance(payload.get('data'), Mapping) else None,
|
|
254
|
-
(payload.get('xdt_api__v1__users__web_profile_info') or {}).get('user') if isinstance(payload.get('xdt_api__v1__users__web_profile_info'), Mapping) else None,
|
|
255
|
-
]
|
|
256
|
-
direct = next((candidate for candidate in candidates if isinstance(candidate, Mapping)), None)
|
|
257
|
-
if direct is not None:
|
|
258
|
-
return direct
|
|
259
|
-
for node in _walk_json(payload):
|
|
260
|
-
if not isinstance(node, Mapping) or not node.get('username'):
|
|
261
|
-
continue
|
|
262
|
-
if any(key in node for key in ('biography', 'edge_follow', 'edge_followed_by', 'follower_count', 'following_count', 'full_name')):
|
|
263
|
-
return node
|
|
264
|
-
return None
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
def _shared_data_documents(document: str) -> list[Any]:
|
|
268
|
-
"""Extract JSON assigned to window._sharedData from the rendered profile HTML."""
|
|
269
|
-
decoder = json.JSONDecoder()
|
|
270
|
-
documents: list[Any] = []
|
|
271
|
-
patterns = (
|
|
272
|
-
r'window\s*\.\s*_sharedData\s*=\s*',
|
|
273
|
-
r'window\s*\[\s*["\']_sharedData["\']\s*\]\s*=\s*',
|
|
274
|
-
)
|
|
275
|
-
for pattern in patterns:
|
|
276
|
-
for match in re.finditer(pattern, document, flags=re.IGNORECASE):
|
|
277
|
-
start = document.find('{', match.end())
|
|
278
|
-
if start < 0:
|
|
279
|
-
continue
|
|
280
|
-
try:
|
|
281
|
-
parsed, _ = decoder.raw_decode(document[start:])
|
|
282
|
-
except json.JSONDecodeError:
|
|
283
|
-
continue
|
|
284
|
-
documents.append(parsed)
|
|
285
|
-
return documents
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
def _profile_users_from_document(document: str) -> list[Mapping[str, Any]]:
|
|
289
|
-
"""Return users found in legacy _sharedData and current embedded profile payloads."""
|
|
290
|
-
users: list[Mapping[str, Any]] = []
|
|
291
|
-
documents = _shared_data_documents(document) + _embedded_json_documents(document)
|
|
292
|
-
for payload in documents:
|
|
293
|
-
direct = _extract_profile_user(payload)
|
|
294
|
-
if isinstance(direct, Mapping):
|
|
295
|
-
users.append(direct)
|
|
296
|
-
for node in _walk_json(payload):
|
|
297
|
-
if not isinstance(node, Mapping):
|
|
298
|
-
continue
|
|
299
|
-
candidate = node.get('user')
|
|
300
|
-
if isinstance(candidate, Mapping) and any(key in candidate for key in ('biography', 'edge_follow', 'edge_followed_by', 'username')):
|
|
301
|
-
users.append(candidate)
|
|
302
|
-
unique: list[Mapping[str, Any]] = []
|
|
303
|
-
seen: set[str] = set()
|
|
304
|
-
for user in users:
|
|
305
|
-
marker = str(user.get('id') or user.get('pk') or user.get('username') or id(user))
|
|
306
|
-
if marker not in seen:
|
|
307
|
-
seen.add(marker)
|
|
308
|
-
unique.append(user)
|
|
309
|
-
return unique
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
def _extract_posts_from_documents(documents: list[Any], limit: int) -> list[dict[str, Any]]:
|
|
152
|
+
def _extract_posts_from_html(document: str, limit: int = 10) -> list[dict[str, Any]]:
|
|
313
153
|
posts: list[dict[str, Any]] = []
|
|
314
154
|
seen: set[str] = set()
|
|
315
|
-
for
|
|
316
|
-
for node in _walk_json(
|
|
317
|
-
if not isinstance(node,
|
|
155
|
+
for payload in _embedded_json_documents(document):
|
|
156
|
+
for node in _walk_json(payload):
|
|
157
|
+
if not isinstance(node, dict):
|
|
318
158
|
continue
|
|
319
|
-
post = _post_from_node(
|
|
159
|
+
post = _post_from_node(node)
|
|
320
160
|
if not post or post['id'] in seen:
|
|
321
161
|
continue
|
|
322
162
|
seen.add(post['id'])
|
|
@@ -326,111 +166,118 @@ def _extract_posts_from_documents(documents: list[Any], limit: int) -> list[dict
|
|
|
326
166
|
return posts
|
|
327
167
|
|
|
328
168
|
|
|
329
|
-
def
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
posts = _extract_posts_from_documents(documents, limit=max(1, int(limit)))
|
|
347
|
-
LOGGER.info('Instagram HTML post extraction posts=%s limit=%s', len(posts), limit)
|
|
348
|
-
print(f"Playwright usado no Windows, posts extraídos: {len(posts)}")
|
|
349
|
-
return posts
|
|
169
|
+
def _metric_text_value(value: Any) -> int | None:
|
|
170
|
+
if isinstance(value, Mapping):
|
|
171
|
+
value = value.get('count') or value.get('value')
|
|
172
|
+
text = str(value or '').strip().casefold().replace(' ', '')
|
|
173
|
+
if not text:
|
|
174
|
+
return None
|
|
175
|
+
multiplier = 1
|
|
176
|
+
if text.endswith(('k', 'mil')):
|
|
177
|
+
multiplier = 1000
|
|
178
|
+
text = text[:-1] if text.endswith('k') else text[:-3]
|
|
179
|
+
elif text.endswith(('m', 'mi', 'milhões')):
|
|
180
|
+
multiplier = 1_000_000
|
|
181
|
+
text = text[:-1] if text.endswith('m') else text[:-2] if text.endswith('mi') else text[:-7]
|
|
182
|
+
digits = re.sub(r'[^0-9]', '', text)
|
|
183
|
+
if not digits:
|
|
184
|
+
return None
|
|
185
|
+
return int(digits) * multiplier
|
|
350
186
|
|
|
351
187
|
|
|
352
|
-
def
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
188
|
+
def _embedded_profile_user(document: str, username: str) -> Mapping[str, Any]:
|
|
189
|
+
wanted = username.casefold()
|
|
190
|
+
for payload in _embedded_json_documents(document):
|
|
191
|
+
for node in _walk_json(payload):
|
|
192
|
+
if not isinstance(node, Mapping):
|
|
193
|
+
continue
|
|
194
|
+
node_username = str(node.get('username') or node.get('handle') or '').lstrip('@').casefold()
|
|
195
|
+
if node_username == wanted and any(key in node for key in ('biography', 'bio', 'edge_followed_by', 'edge_follow', 'followers', 'following')):
|
|
196
|
+
return node
|
|
197
|
+
return {}
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _extract_profile_user_from_html(document: str, username: str) -> dict[str, Any]:
|
|
201
|
+
embedded_user = _embedded_profile_user(document, username)
|
|
202
|
+
seo_description = _meta(document, 'og:description')
|
|
203
|
+
biography = str(embedded_user.get('biography') or embedded_user.get('bio') or '').strip()
|
|
204
|
+
description = biography or seo_description
|
|
205
|
+
title = str(embedded_user.get('full_name') or embedded_user.get('name') or _meta(document, 'og:title') or username).split('(')[0].strip() or username
|
|
206
|
+
avatar_url = str(embedded_user.get('profile_pic_url_hd') or embedded_user.get('profile_pic_url') or _meta(document, 'og:image')).strip()
|
|
207
|
+
|
|
208
|
+
def metric(patterns: tuple[str, ...]) -> int | None:
|
|
209
|
+
for pattern in patterns:
|
|
210
|
+
found = re.search(pattern, seo_description, flags=re.IGNORECASE)
|
|
211
|
+
if found:
|
|
212
|
+
parsed = _metric_text_value(found.group(1) + (found.group(2) or ''))
|
|
213
|
+
if parsed is not None:
|
|
214
|
+
return parsed
|
|
215
|
+
return None
|
|
216
|
+
|
|
217
|
+
followers = _metric_text_value(embedded_user.get('edge_followed_by')) or _metric_text_value(embedded_user.get('followers')) or metric((r'([\d.,]+)\s*(k|mil)?\s*(?:followers|seguidores)',)) or _structured_metric(document, 'edge_followed_by', 'followers', 'follower_count') or _structured_metric_from_json(document, 'edge_followed_by', 'followers', 'follower_count', 'followerCount')
|
|
218
|
+
following = _metric_text_value(embedded_user.get('edge_follow')) or _metric_text_value(embedded_user.get('following')) or metric((r'([\d.,]+)\s*(k|mil)?\s*(?:following|seguindo)',)) or _structured_metric(document, 'edge_follow', 'follows', 'following', 'following_count', 'followingCount') or _structured_metric_from_json(document, 'edge_follow', 'follows', 'following', 'following_count', 'followingCount')
|
|
219
|
+
post_count = _metric_text_value(embedded_user.get('edge_owner_to_timeline_media')) or _metric_text_value(embedded_user.get('posts')) or metric((r'([\d.,]+)\s*(k|mil)?\s*(?:posts|publicações|publications)',)) or _structured_metric(document, 'edge_owner_to_timeline_media', 'posts', 'post_count')
|
|
220
|
+
country = _country_value(embedded_user.get('country') or embedded_user.get('country_code')) or extract_public_instagram_country(embedded_user)
|
|
221
|
+
if not country:
|
|
222
|
+
for payload in _embedded_json_documents(document):
|
|
223
|
+
country = extract_public_instagram_country(payload)
|
|
224
|
+
if country:
|
|
225
|
+
break
|
|
226
|
+
posts = _extract_posts_from_html(document, 10)
|
|
227
|
+
user: dict[str, Any] = {
|
|
228
|
+
'username': username,
|
|
229
|
+
'full_name': title.split('(')[0].strip() or username,
|
|
230
|
+
'biography': description,
|
|
231
|
+
'edge_followed_by': {'count': followers} if followers is not None else {},
|
|
232
|
+
'edge_follow': {'count': following} if following is not None else {},
|
|
233
|
+
'edge_owner_to_timeline_media': {'count': post_count, 'edges': [{'node': post} for post in posts]},
|
|
234
|
+
'country': country,
|
|
235
|
+
'profile_pic_url': avatar_url,
|
|
236
|
+
'_html_posts': posts,
|
|
237
|
+
}
|
|
238
|
+
return user if any((description, followers, following, post_count, country, posts)) else {}
|
|
389
239
|
|
|
390
240
|
|
|
391
241
|
def _fetch_web_profile_user(username: str) -> dict[str, Any] | None:
|
|
392
|
-
_ensure_windows_playwright()
|
|
393
242
|
headers = _instagram_headers()
|
|
394
243
|
headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0 Safari/537.36'
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
cookies = _instagram_cookies()
|
|
244
|
+
# The www host is the current public web endpoint. The legacy i host is
|
|
245
|
+
# retained as a fallback because Instagram rate-limits the two hosts
|
|
246
|
+
# independently and their availability can vary by region.
|
|
400
247
|
endpoints = (
|
|
401
248
|
f'https://www.instagram.com/api/v1/users/web_profile_info/?username={username}',
|
|
402
249
|
f'https://i.instagram.com/api/v1/users/web_profile_info/?username={username}',
|
|
403
250
|
)
|
|
251
|
+
def profile_quality(user: Mapping[str, Any]) -> int:
|
|
252
|
+
return sum(bool(user.get(key)) for key in ('biography', 'edge_follow', 'edge_followed_by', 'edge_owner_to_timeline_media', 'username'))
|
|
253
|
+
|
|
254
|
+
def merge_profile(base: dict[str, Any] | None, candidate: Mapping[str, Any]) -> dict[str, Any]:
|
|
255
|
+
merged = dict(base or {})
|
|
256
|
+
for key, value in candidate.items():
|
|
257
|
+
if value not in (None, '', [], {}):
|
|
258
|
+
if isinstance(value, Mapping) and isinstance(merged.get(key), Mapping):
|
|
259
|
+
merged[key] = {**merged[key], **value}
|
|
260
|
+
else:
|
|
261
|
+
merged[key] = value
|
|
262
|
+
return merged
|
|
404
263
|
|
|
405
|
-
request_headers = [
|
|
406
|
-
headers,
|
|
407
|
-
{**headers, 'User-Agent': 'Instagram 390.0.0.0.50 Android', 'Accept': 'application/json, text/plain, */*'},
|
|
408
|
-
{**headers, 'Referer': f'https://www.instagram.com/{username}/', 'Origin': 'https://www.instagram.com', 'X-Requested-With': 'XMLHttpRequest'},
|
|
409
|
-
]
|
|
410
264
|
best_user: dict[str, Any] | None = None
|
|
411
|
-
for
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
LOGGER.debug('Instagram profile URL=%s status=%s content_type=%s', endpoint, response.status_code, response.headers.get('content-type', ''))
|
|
416
|
-
if response.status_code >= 400:
|
|
417
|
-
continue
|
|
418
|
-
payload = response.json()
|
|
419
|
-
LOGGER.debug('Instagram profile URL=%s response=JSON', endpoint)
|
|
420
|
-
except (requests.RequestException, ValueError, AttributeError, StopIteration):
|
|
421
|
-
LOGGER.warning('Falha na requisição JSON do perfil Instagram URL=%s', endpoint, exc_info=True)
|
|
422
|
-
if best_user:
|
|
423
|
-
return best_user
|
|
265
|
+
for endpoint in endpoints:
|
|
266
|
+
try:
|
|
267
|
+
response = requests.get(endpoint, headers=headers, timeout=15)
|
|
268
|
+
if response.status_code >= 400:
|
|
424
269
|
continue
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
270
|
+
payload = response.json()
|
|
271
|
+
except (requests.RequestException, ValueError, AttributeError):
|
|
272
|
+
continue
|
|
273
|
+
user = ((payload.get('data') or {}).get('user') if isinstance(payload, dict) else None)
|
|
274
|
+
if isinstance(user, dict):
|
|
275
|
+
best_user = merge_profile(best_user, user)
|
|
276
|
+
if profile_quality(best_user) >= 4:
|
|
277
|
+
return best_user
|
|
430
278
|
|
|
431
279
|
curl = shutil.which('curl')
|
|
432
280
|
if curl:
|
|
433
|
-
LOGGER.debug('Fallback curl disponível em %s', curl)
|
|
434
281
|
for endpoint in endpoints:
|
|
435
282
|
try:
|
|
436
283
|
completed = subprocess.run(
|
|
@@ -443,20 +290,21 @@ def _fetch_web_profile_user(username: str) -> dict[str, Any] | None:
|
|
|
443
290
|
check=False,
|
|
444
291
|
)
|
|
445
292
|
payload = json.loads(completed.stdout) if completed.returncode == 0 and completed.stdout else None
|
|
446
|
-
|
|
447
|
-
user = _extract_profile_user(payload)
|
|
293
|
+
user = ((payload.get('data') or {}).get('user') if isinstance(payload, dict) else None)
|
|
448
294
|
if isinstance(user, dict):
|
|
449
|
-
best_user =
|
|
450
|
-
if
|
|
295
|
+
best_user = merge_profile(best_user, user)
|
|
296
|
+
if profile_quality(best_user) >= 4:
|
|
451
297
|
return best_user
|
|
452
298
|
except (OSError, subprocess.SubprocessError, ValueError):
|
|
453
|
-
LOGGER.warning('Falha no fallback curl do perfil Instagram URL=%s', endpoint, exc_info=True)
|
|
454
299
|
continue
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
300
|
+
if platform.system() == 'Windows':
|
|
301
|
+
html_content = _fetch_instagram_html_with_playwright(f'https://www.instagram.com/{username}/', username)
|
|
302
|
+
if html_content:
|
|
303
|
+
html_user = _extract_profile_user_from_html(html_content, username)
|
|
304
|
+
if html_user:
|
|
305
|
+
print(f'Perfil Instagram extraído do HTML no Windows para @{username}')
|
|
306
|
+
return html_user
|
|
307
|
+
return best_user
|
|
460
308
|
|
|
461
309
|
|
|
462
310
|
def _country_from_bloks(value: Any) -> str:
|
|
@@ -506,74 +354,25 @@ def _fetch_instagram_about_country(user_id: Any) -> str:
|
|
|
506
354
|
}
|
|
507
355
|
try:
|
|
508
356
|
response = requests.post(ABOUT_ACCOUNT_URL, data=payload, headers=_instagram_headers(), cookies=cookies, timeout=20)
|
|
509
|
-
LOGGER.debug('Instagram about-account URL=%s status=%s content_type=%s', ABOUT_ACCOUNT_URL, response.status_code, response.headers.get('content-type', ''))
|
|
510
357
|
if response.status_code >= 400:
|
|
511
358
|
return ''
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
return country
|
|
515
|
-
except (requests.RequestException, ValueError, TypeError) as exc:
|
|
516
|
-
LOGGER.warning('Falha ao consultar país público do Instagram: %s', exc, exc_info=True)
|
|
359
|
+
return _country_from_bloks(response.json())
|
|
360
|
+
except (requests.RequestException, ValueError, TypeError):
|
|
517
361
|
return ''
|
|
518
362
|
|
|
519
363
|
|
|
520
|
-
def _country_from_bio_fallback(bio: str) -> str:
|
|
521
|
-
patterns = (
|
|
522
|
-
r'\b(?:from|based in|located in|living in)\s+([A-Za-zÀ-ÿ][A-Za-zÀ-ÿ .-]{1,60})',
|
|
523
|
-
r'\b(?:de|baseado em|morando em|localizado em)\s+([A-Za-zÀ-ÿ][A-Za-zÀ-ÿ .-]{1,60})',
|
|
524
|
-
)
|
|
525
|
-
for pattern in patterns:
|
|
526
|
-
match = re.search(pattern, bio or '', flags=re.IGNORECASE)
|
|
527
|
-
if match:
|
|
528
|
-
return match.group(1).strip(' .,!;:')
|
|
529
|
-
return ''
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
def _profile_post_nodes(user: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
|
533
|
-
"""Return post nodes from both legacy GraphQL edges and current items payloads."""
|
|
534
|
-
containers: list[Any] = [
|
|
535
|
-
user.get('edge_owner_to_timeline_media'),
|
|
536
|
-
user.get('timeline_media'),
|
|
537
|
-
user.get('media'),
|
|
538
|
-
user,
|
|
539
|
-
]
|
|
540
|
-
nodes: list[Mapping[str, Any]] = []
|
|
541
|
-
for container in containers:
|
|
542
|
-
if not isinstance(container, Mapping):
|
|
543
|
-
continue
|
|
544
|
-
raw_items = container.get('edges') or container.get('items') or container.get('data') or []
|
|
545
|
-
if isinstance(raw_items, Mapping):
|
|
546
|
-
raw_items = raw_items.get('items') or raw_items.get('edges') or []
|
|
547
|
-
if not isinstance(raw_items, list):
|
|
548
|
-
continue
|
|
549
|
-
for item in raw_items:
|
|
550
|
-
node = item.get('node') if isinstance(item, Mapping) and isinstance(item.get('node'), Mapping) else item
|
|
551
|
-
if isinstance(node, Mapping):
|
|
552
|
-
nodes.append(node)
|
|
553
|
-
return nodes
|
|
554
|
-
|
|
555
|
-
|
|
556
364
|
def _profile_data_from_api(user: Mapping[str, Any], reference: Mapping[str, str]) -> dict[str, Any]:
|
|
557
365
|
followers = _profile_metric(user, 'edge_followed_by', 'followers', 'follower_count', 'followerCount')
|
|
558
|
-
|
|
559
|
-
following = normalize_instagram_metric(following_node) if isinstance(following_node, Mapping) else None
|
|
560
|
-
following = following if following is not None else _profile_metric(user, 'follows', 'following', 'following_count', 'followingCount')
|
|
366
|
+
following = _profile_metric(user, 'edge_follow', 'follows', 'following', 'following_count', 'followingCount')
|
|
561
367
|
followers = followers if followers is not None else _structured_metric_from_json(json.dumps(user), 'edge_followed_by', 'followers', 'follower_count', 'followerCount')
|
|
562
368
|
following = following if following is not None else _structured_metric_from_json(json.dumps(user), 'edge_follow', 'follows', 'following', 'following_count', 'followingCount')
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
biography = str(user.get('biography') or user.get('bio') or user.get('description') or '').strip()
|
|
567
|
-
direct_country = _country_value(user.get('country') or user.get('country_code'))
|
|
568
|
-
country = direct_country or extract_public_instagram_country(user)
|
|
569
|
-
if platform.system() != 'Windows':
|
|
570
|
-
country = country or _fetch_instagram_about_country(user.get('id') or user.get('pk'))
|
|
571
|
-
country = country or _country_from_bio_fallback(biography)
|
|
572
|
-
LOGGER.debug('Instagram profile extracted keys=%s edge_follow=%r bio=%s followers=%s following=%s posts=%s country=%s', sorted(str(key) for key in user.keys()), following_node, bool(biography), followers, following, media, country or '—')
|
|
369
|
+
media = ((user.get('edge_owner_to_timeline_media') or {}).get('count') if isinstance(user.get('edge_owner_to_timeline_media'), dict) else None)
|
|
370
|
+
biography = str(user.get('biography') or '').strip()
|
|
371
|
+
country = extract_public_instagram_country(user) or _fetch_instagram_about_country(user.get('id') or user.get('pk'))
|
|
573
372
|
return {
|
|
574
373
|
'id': f"instagram_{reference['username']}",
|
|
575
374
|
**reference,
|
|
576
|
-
'name': str(user.get('full_name') or user.get('
|
|
375
|
+
'name': str(user.get('full_name') or user.get('username') or reference['username']).strip(),
|
|
577
376
|
'bio': normalize_instagram_bio(biography),
|
|
578
377
|
'bio_raw': biography,
|
|
579
378
|
'country': country,
|
|
@@ -585,7 +384,7 @@ def _profile_data_from_api(user: Mapping[str, Any], reference: Mapping[str, str]
|
|
|
585
384
|
'public_lookup': True,
|
|
586
385
|
'metrics_source': 'instagram_web_profile_info',
|
|
587
386
|
'last_public_lookup_at': datetime.now(timezone.utc).isoformat(),
|
|
588
|
-
'_api_posts':
|
|
387
|
+
'_api_posts': (((user.get('edge_owner_to_timeline_media') or {}).get('edges') or []) if isinstance(user.get('edge_owner_to_timeline_media'), dict) else []),
|
|
589
388
|
}
|
|
590
389
|
|
|
591
390
|
|
|
@@ -600,14 +399,10 @@ def normalize_instagram_metric(value: Any) -> int | None:
|
|
|
600
399
|
|
|
601
400
|
|
|
602
401
|
PUBLIC_COUNTRY_KEYS = {
|
|
603
|
-
'about_this_account_country', 'account_based_in', 'account_country',
|
|
604
|
-
'
|
|
605
|
-
'transparency_country',
|
|
606
|
-
}
|
|
607
|
-
PUBLIC_TRANSPARENCY_KEYS = {
|
|
608
|
-
'about', 'about_account', 'about_this_account', 'about_this_account_data',
|
|
609
|
-
'account_transparency', 'transparency', 'account_info', 'profile_context',
|
|
402
|
+
'about_this_account_country', 'account_based_in', 'account_country', 'country_name',
|
|
403
|
+
'country_of_origin', 'country_of_registration', 'location_country', 'transparency_country',
|
|
610
404
|
}
|
|
405
|
+
PUBLIC_TRANSPARENCY_KEYS = {'about', 'about_account', 'account_transparency', 'transparency', 'account_info'}
|
|
611
406
|
|
|
612
407
|
|
|
613
408
|
def _country_value(value: Any) -> str:
|
|
@@ -651,23 +446,17 @@ def _profile_metric(user: Mapping[str, Any], *keys: str) -> int | None:
|
|
|
651
446
|
|
|
652
447
|
|
|
653
448
|
def fetch_public_instagram_profile(source: str) -> IntegrationResult:
|
|
654
|
-
LOGGER.debug('fetch_public_instagram_profile source=%s', source)
|
|
655
449
|
try:
|
|
656
450
|
reference = normalize_instagram_reference(source)
|
|
657
451
|
except ValueError as exc:
|
|
658
452
|
return IntegrationResult(False, str(exc), {})
|
|
659
453
|
api_user = _fetch_web_profile_user(reference['username'])
|
|
660
454
|
if api_user:
|
|
661
|
-
|
|
662
|
-
return IntegrationResult(True, 'Perfil Instagram encontrado publicamente.', data)
|
|
663
|
-
if platform.system() == 'Windows':
|
|
664
|
-
return IntegrationResult(False, 'Não foi possível obter os dados do Instagram. Verifique sua conexão ou tente novamente mais tarde.', reference)
|
|
455
|
+
return IntegrationResult(True, 'Perfil Instagram encontrado publicamente.', _profile_data_from_api(api_user, reference))
|
|
665
456
|
try:
|
|
666
457
|
response = requests.get(reference['url'], headers={'User-Agent': 'Mozilla/5.0', 'Accept-Language': 'pt-BR,pt;q=0.9,en;q=0.8'}, timeout=12)
|
|
667
458
|
except requests.RequestException as exc:
|
|
668
|
-
LOGGER.warning('Falha HTTP no perfil Instagram URL=%s: %s', reference['url'], exc, exc_info=True)
|
|
669
459
|
return IntegrationResult(False, f'Não foi possível consultar o perfil público do Instagram: {exc}', reference)
|
|
670
|
-
LOGGER.debug('Instagram HTML profile URL=%s status=%s content_type=%s response=HTML', reference['url'], response.status_code, response.headers.get('content-type', ''))
|
|
671
460
|
if response.status_code >= 400:
|
|
672
461
|
return IntegrationResult(False, f'O Instagram devolveu HTTP {response.status_code}. Confirme o @handle ou use o cadastro manual.', reference | {'status_code': response.status_code})
|
|
673
462
|
title = _meta(response.text, 'og:title') or reference['username']
|
|
@@ -692,8 +481,6 @@ def fetch_public_instagram_profile(source: str) -> IntegrationResult:
|
|
|
692
481
|
embedded_country = extract_public_instagram_country(document)
|
|
693
482
|
if embedded_country:
|
|
694
483
|
break
|
|
695
|
-
embedded_country = embedded_country or _country_from_bio_fallback(description)
|
|
696
|
-
LOGGER.debug('Instagram HTML profile extracted bio=%s followers=%s following=%s posts=%s country=%s', bool(description), followers, following, posts, embedded_country or '—')
|
|
697
484
|
data = {'id': f"instagram_{reference['username']}", **reference, 'name': title.split('(')[0].strip() or reference['username'], 'bio': normalize_instagram_bio(description), 'bio_raw': description, 'country': embedded_country, 'avatar_url': avatar_url, 'subscriber_count': followers, 'following_count': following, 'post_count': posts, 'public_lookup': True, 'metrics_source': 'instagram_public_page', 'last_public_lookup_at': datetime.now(timezone.utc).isoformat()}
|
|
698
485
|
return IntegrationResult(True, 'Perfil Instagram encontrado publicamente.', data)
|
|
699
486
|
|
|
@@ -755,73 +542,35 @@ def _post_from_node(node: dict[str, Any]) -> dict[str, Any] | None:
|
|
|
755
542
|
}
|
|
756
543
|
|
|
757
544
|
|
|
758
|
-
def _fetch_posts_with_playwright(reference: Mapping[str, str], limit: int) -> list[dict[str, Any]]:
|
|
759
|
-
"""Load a public profile page and extract posts only from its rendered HTML."""
|
|
760
|
-
url = str(reference.get('url') or '')
|
|
761
|
-
try:
|
|
762
|
-
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError, sync_playwright
|
|
763
|
-
with sync_playwright() as playwright:
|
|
764
|
-
browser = playwright.chromium.launch(headless=_instagram_playwright_headless())
|
|
765
|
-
try:
|
|
766
|
-
context = browser.new_context(
|
|
767
|
-
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0 Safari/537.36',
|
|
768
|
-
extra_http_headers={'Accept-Language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7'},
|
|
769
|
-
)
|
|
770
|
-
page = context.new_page()
|
|
771
|
-
response = page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
|
772
|
-
status = response.status if response else None
|
|
773
|
-
LOGGER.info('Instagram Playwright HTML posts URL=%s status=%s', url, status)
|
|
774
|
-
print(f'Playwright usado no Windows para os posts de @{reference.get("username", "")}'.strip())
|
|
775
|
-
page.wait_for_timeout(5000)
|
|
776
|
-
try:
|
|
777
|
-
page.wait_for_load_state('networkidle', timeout=10000)
|
|
778
|
-
except (PlaywrightTimeoutError, TimeoutError):
|
|
779
|
-
LOGGER.debug('Instagram Playwright networkidle excedido; usando HTML já carregado para posts.')
|
|
780
|
-
html_content = page.content()
|
|
781
|
-
_save_instagram_debug_html(reference.get('username', 'instagram'), html_content)
|
|
782
|
-
if status is not None and status >= 400:
|
|
783
|
-
return []
|
|
784
|
-
return _extract_posts_from_html(html_content, limit)
|
|
785
|
-
finally:
|
|
786
|
-
browser.close()
|
|
787
|
-
except Exception as exc:
|
|
788
|
-
LOGGER.warning('Falha no Playwright HTML dos posts Instagram URL=%s: %s', url, exc, exc_info=True)
|
|
789
|
-
print(f'Playwright falhou no Windows para os posts: {exc}')
|
|
790
|
-
_streamlit_message('error', f'Playwright não conseguiu consultar os posts Instagram: {exc}', 'posts_playwright_error')
|
|
791
|
-
return []
|
|
792
|
-
|
|
793
|
-
|
|
794
545
|
def fetch_public_instagram_posts(source: str, limit: int = 10) -> IntegrationResult:
|
|
795
|
-
LOGGER.debug('fetch_public_instagram_posts source=%s limit=%s', source, limit)
|
|
796
|
-
_ensure_windows_playwright()
|
|
797
546
|
try:
|
|
798
547
|
reference = normalize_instagram_reference(source)
|
|
799
548
|
except ValueError as exc:
|
|
800
549
|
return IntegrationResult(False, str(exc), {})
|
|
801
|
-
if platform.system() == 'Windows':
|
|
802
|
-
LOGGER.debug('Windows detectado: posts Instagram exclusivamente via HTML Playwright.')
|
|
803
|
-
posts = _fetch_posts_with_playwright(reference, limit)
|
|
804
|
-
if not posts:
|
|
805
|
-
_streamlit_message('error', 'Não foi possível obter os dados do Instagram. Verifique sua conexão ou tente novamente mais tarde.', 'posts_instagram_failed')
|
|
806
|
-
return IntegrationResult(bool(posts), 'Posts públicos encontrados.' if posts else 'Não foi possível obter os dados do Instagram. Verifique sua conexão ou tente novamente mais tarde.', reference | {'posts': posts[:max(1, int(limit))]})
|
|
807
550
|
api_user = _fetch_web_profile_user(reference['username'])
|
|
808
551
|
if api_user:
|
|
809
552
|
api_posts: list[dict[str, Any]] = []
|
|
810
|
-
for
|
|
811
|
-
|
|
812
|
-
if
|
|
813
|
-
|
|
553
|
+
for edge in ((api_user.get('edge_owner_to_timeline_media') or {}).get('edges') or []):
|
|
554
|
+
node = edge.get('node') if isinstance(edge, dict) else None
|
|
555
|
+
if isinstance(node, dict):
|
|
556
|
+
post = _post_from_node(node)
|
|
557
|
+
if post:
|
|
558
|
+
api_posts.append(post)
|
|
814
559
|
if api_posts:
|
|
815
|
-
LOGGER.debug('Instagram API posts extracted=%s response=JSON', len(api_posts))
|
|
816
560
|
return IntegrationResult(True, 'Posts públicos encontrados.', reference | {'posts': api_posts[:max(1, int(limit))]})
|
|
817
561
|
if api_user.get('is_private'):
|
|
818
562
|
return IntegrationResult(False, 'Esta conta é privada. O Instagram não disponibiliza posts sem uma sessão autenticada.', reference | {'posts': [], 'is_private': True})
|
|
563
|
+
if platform.system() == 'Windows':
|
|
564
|
+
html_posts = api_user.get('_html_posts') or []
|
|
565
|
+
return IntegrationResult(bool(html_posts), 'Posts públicos encontrados.' if html_posts else 'Não foi possível encontrar posts públicos nesta página do Instagram.', reference | {'posts': html_posts[:max(1, int(limit))]})
|
|
566
|
+
if platform.system() == 'Windows':
|
|
567
|
+
html_content = _fetch_instagram_html_with_playwright(reference['url'], reference['username'])
|
|
568
|
+
posts = _extract_posts_from_html(html_content, limit) if html_content else []
|
|
569
|
+
return IntegrationResult(bool(posts), 'Posts públicos encontrados.' if posts else 'Não foi possível encontrar posts públicos nesta página do Instagram.', reference | {'posts': posts[:max(1, int(limit))]})
|
|
819
570
|
try:
|
|
820
|
-
response = requests.get(reference['url'], headers={'User-Agent': 'Mozilla/5.0', 'Accept-Language': 'pt-BR,pt;q=0.9,en;q=0.8'},
|
|
571
|
+
response = requests.get(reference['url'], headers={'User-Agent': 'Mozilla/5.0', 'Accept-Language': 'pt-BR,pt;q=0.9,en;q=0.8'}, timeout=15)
|
|
821
572
|
except requests.RequestException as exc:
|
|
822
|
-
LOGGER.warning('Falha HTTP nos posts Instagram URL=%s: %s', reference['url'], exc, exc_info=True)
|
|
823
573
|
return IntegrationResult(False, f'Não foi possível consultar os posts públicos do Instagram: {exc}', reference)
|
|
824
|
-
LOGGER.debug('Instagram HTML posts URL=%s status=%s content_type=%s response=HTML', reference['url'], response.status_code, response.headers.get('content-type', ''))
|
|
825
574
|
if response.status_code >= 400:
|
|
826
575
|
return IntegrationResult(False, f'O Instagram devolveu HTTP {response.status_code} ao consultar os posts.', reference | {'status_code': response.status_code})
|
|
827
576
|
posts: list[dict[str, Any]] = []
|
|
@@ -837,10 +586,7 @@ def fetch_public_instagram_posts(source: str, limit: int = 10) -> IntegrationRes
|
|
|
837
586
|
break
|
|
838
587
|
if len(posts) >= max(1, int(limit)):
|
|
839
588
|
break
|
|
840
|
-
if
|
|
841
|
-
posts = _fetch_posts_with_playwright(reference, limit)
|
|
842
|
-
LOGGER.debug('Instagram posts extracted=%s limit=%s', len(posts), limit)
|
|
843
|
-
return IntegrationResult(bool(posts), 'Posts públicos encontrados.' if posts else 'Não foi possível obter os dados do Instagram. Verifique sua conexão ou tente novamente mais tarde.', reference | {'posts': posts[:max(1, int(limit))]})
|
|
589
|
+
return IntegrationResult(bool(posts), 'Posts públicos encontrados.' if posts else 'Não foi possível encontrar posts públicos nesta página do Instagram.', reference | {'posts': posts[:max(1, int(limit))]})
|
|
844
590
|
|
|
845
591
|
|
|
846
592
|
__all__ = ['IntegrationResult', 'extract_public_instagram_country', 'fetch_public_instagram_posts', 'fetch_public_instagram_profile', 'normalize_instagram_bio', 'normalize_instagram_metric', 'normalize_instagram_reference']
|
package/package.json
CHANGED