devez-vibe 1.10.19 → 1.10.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +73 -73
- package/bin/dvz.exe +0 -0
- package/bin/dvz.js +35 -35
- package/bridge/claude-agent-sdk-bridge.mjs +32 -0
- package/install-skills.mjs +11 -11
- package/package.json +1 -1
- package/prepare-search-deps.mjs +42 -42
- package/skills/insane-search/SKILL.md +36 -36
- package/skills/insane-search/engine/__main__.py +44 -44
- package/skills/insane-search/engine/executor.py +246 -246
- package/skills/insane-search/engine/fetch_chain.py +443 -443
- package/skills/insane-search/engine/network.py +108 -108
- package/skills/insane-search/engine/phase0.py +380 -380
- package/skills/insane-search/engine/session_input.py +120 -120
- package/skills/insane-search/engine/templates/invisible_playwright_fetch.py +57 -57
- package/skills/insane-search/engine/templates/nodriver_fetch.py +9 -9
- package/skills/insane-search/engine/templates/patchright_fetch.py +9 -9
- package/skills/insane-search/engine/templates/playwright_mobile_chrome.js +9 -9
- package/skills/insane-search/engine/templates/playwright_real_chrome.js +7 -7
- package/skills/insane-search/engine/templates/scrapling_fetch.py +46 -46
- package/skills/insane-search/engine/tests/test_smoke.py +8 -8
- package/skills/insane-search/engine/tests/test_t10_artifact_routes.py +126 -126
- package/skills/insane-search/engine/tests/test_t11_session_handoff.py +157 -157
- package/skills/insane-search/engine/tests/test_t5_pdfplumber.py +11 -11
- package/skills/insane-search/engine/tests/test_t7_browser_gate.py +3 -3
- package/skills/insane-search/engine/tests/test_t8_network_routes.py +207 -207
- package/skills/insane-search/engine/tests/test_t9_evidence_bundle.py +89 -89
- package/skills/insane-search/engine/tests/test_u5.py +9 -9
- package/skills/insane-search/engine/transport.py +94 -94
- package/skills/insane-search/engine/waf_profiles.yaml +38 -38
- package/skills/insane-search/references/cache-archive.md +13 -13
- package/skills/insane-search/references/media.md +14 -14
- package/skills/insane-search/references/playwright.md +27 -27
- package/skills/insane-search/references/research-evidence.md +36 -36
- package/skills/insane-search/references/session-handoff.md +26 -26
- package/skills/insane-search/tests/test_x_search.py +3 -3
- package/skills/luna-loop/agents/openai.yaml +7 -7
|
@@ -24,36 +24,36 @@ Each attempt dict: {"route","platform","ok","status","bytes","note"}.
|
|
|
24
24
|
"""
|
|
25
25
|
from __future__ import annotations
|
|
26
26
|
|
|
27
|
-
import importlib.util
|
|
28
|
-
import html
|
|
29
|
-
import json
|
|
30
|
-
import os
|
|
31
|
-
import re
|
|
32
|
-
import shutil
|
|
33
|
-
import subprocess
|
|
34
|
-
import sys
|
|
35
|
-
import tempfile
|
|
36
|
-
from pathlib import Path
|
|
37
|
-
from typing import Optional
|
|
38
|
-
from urllib.parse import quote, urlsplit
|
|
27
|
+
import importlib.util
|
|
28
|
+
import html
|
|
29
|
+
import json
|
|
30
|
+
import os
|
|
31
|
+
import re
|
|
32
|
+
import shutil
|
|
33
|
+
import subprocess
|
|
34
|
+
import sys
|
|
35
|
+
import tempfile
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
from typing import Optional
|
|
38
|
+
from urllib.parse import quote, urlsplit
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
# --- low-level helpers -------------------------------------------------------
|
|
42
|
-
def _cffi_get(url: str, *, impersonate: str = "safari", timeout: int = 15,
|
|
43
|
-
proxy: Optional[str] = None):
|
|
44
|
-
from curl_cffi import requests as r # lazy: engine works even if missing
|
|
45
|
-
kwargs = dict(
|
|
46
|
-
impersonate=impersonate, # type: ignore[arg-type]
|
|
47
|
-
timeout=timeout,
|
|
42
|
+
def _cffi_get(url: str, *, impersonate: str = "safari", timeout: int = 15,
|
|
43
|
+
proxy: Optional[str] = None):
|
|
44
|
+
from curl_cffi import requests as r # lazy: engine works even if missing
|
|
45
|
+
kwargs = dict(
|
|
46
|
+
impersonate=impersonate, # type: ignore[arg-type]
|
|
47
|
+
timeout=timeout,
|
|
48
48
|
headers={
|
|
49
49
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
50
50
|
"Accept-Language": "en-US,en;q=0.9,ko;q=0.8",
|
|
51
|
-
},
|
|
52
|
-
allow_redirects=True,
|
|
53
|
-
)
|
|
54
|
-
if proxy:
|
|
55
|
-
kwargs["proxy"] = proxy
|
|
56
|
-
return r.get(url, **kwargs)
|
|
51
|
+
},
|
|
52
|
+
allow_redirects=True,
|
|
53
|
+
)
|
|
54
|
+
if proxy:
|
|
55
|
+
kwargs["proxy"] = proxy
|
|
56
|
+
return r.get(url, **kwargs)
|
|
57
57
|
|
|
58
58
|
|
|
59
59
|
def _host(url: str) -> str:
|
|
@@ -83,7 +83,7 @@ def _detect(url: str) -> Optional[str]:
|
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
# --- reddit ------------------------------------------------------------------
|
|
86
|
-
def _reddit(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
86
|
+
def _reddit(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
87
87
|
attempts: list[dict] = []
|
|
88
88
|
base = url.split("?", 1)[0].rstrip("/")
|
|
89
89
|
# Build an .rss / .json target from the path (works for /r/<sub> and post URLs).
|
|
@@ -92,7 +92,7 @@ def _reddit(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
|
92
92
|
|
|
93
93
|
# Route 1: RSS (the route that actually survives — Reddit gates the JSON API).
|
|
94
94
|
try:
|
|
95
|
-
x = _cffi_get(rss_url, timeout=timeout, proxy=proxy)
|
|
95
|
+
x = _cffi_get(rss_url, timeout=timeout, proxy=proxy)
|
|
96
96
|
ok = x.status_code == 200 and ("<rss" in x.text or "<feed" in x.text)
|
|
97
97
|
attempts.append(_attempt("reddit", "rss", ok, x.status_code, x.text,
|
|
98
98
|
"feed" if ok else "no-feed-markers"))
|
|
@@ -104,7 +104,7 @@ def _reddit(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
|
104
104
|
|
|
105
105
|
# Route 2: JSON via curl_cffi (often 403 now, but try — cheap).
|
|
106
106
|
try:
|
|
107
|
-
x = _cffi_get(json_url, timeout=timeout, proxy=proxy)
|
|
107
|
+
x = _cffi_get(json_url, timeout=timeout, proxy=proxy)
|
|
108
108
|
ok = x.status_code == 200 and x.text.lstrip().startswith(("{", "["))
|
|
109
109
|
attempts.append(_attempt("reddit", "json", ok, x.status_code, x.text,
|
|
110
110
|
"json" if ok else f"status={x.status_code}"))
|
|
@@ -122,15 +122,15 @@ def _reddit(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
|
122
122
|
_TWEET_ID_RE = re.compile(r"/status(?:es)?/(\d+)")
|
|
123
123
|
|
|
124
124
|
|
|
125
|
-
def _x(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
125
|
+
def _x(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
126
126
|
attempts: list[dict] = []
|
|
127
127
|
m = _TWEET_ID_RE.search(url)
|
|
128
128
|
|
|
129
129
|
if m: # single tweet → tweet-result + oembed (both no-auth, reliable)
|
|
130
130
|
tid = m.group(1)
|
|
131
131
|
try:
|
|
132
|
-
x = _cffi_get(f"https://cdn.syndication.twimg.com/tweet-result?id={tid}&token=a",
|
|
133
|
-
timeout=timeout, proxy=proxy)
|
|
132
|
+
x = _cffi_get(f"https://cdn.syndication.twimg.com/tweet-result?id={tid}&token=a",
|
|
133
|
+
timeout=timeout, proxy=proxy)
|
|
134
134
|
d = x.json() if x.status_code == 200 else {}
|
|
135
135
|
ok = bool(d.get("text"))
|
|
136
136
|
attempts.append(_attempt("x", "tweet-result", ok, x.status_code, x.text,
|
|
@@ -142,7 +142,7 @@ def _x(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
|
142
142
|
attempts.append(_attempt("x", "tweet-result", False, 0, "", f"{type(e).__name__}"))
|
|
143
143
|
try:
|
|
144
144
|
ourl = f"https://publish.twitter.com/oembed?url=https://twitter.com/i/status/{tid}&omit_script=1"
|
|
145
|
-
x = _cffi_get(ourl, timeout=timeout, proxy=proxy)
|
|
145
|
+
x = _cffi_get(ourl, timeout=timeout, proxy=proxy)
|
|
146
146
|
d = x.json() if x.status_code == 200 else {}
|
|
147
147
|
ok = bool(d.get("html"))
|
|
148
148
|
attempts.append(_attempt("x", "oembed", ok, x.status_code, x.text,
|
|
@@ -159,7 +159,7 @@ def _x(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
|
159
159
|
surl = f"https://syndication.twitter.com/srv/timeline-profile/screen-name/{handle}"
|
|
160
160
|
for attempt_no in range(2):
|
|
161
161
|
try:
|
|
162
|
-
x = _cffi_get(surl, timeout=timeout, proxy=proxy)
|
|
162
|
+
x = _cffi_get(surl, timeout=timeout, proxy=proxy)
|
|
163
163
|
ok = x.status_code == 200 and "__NEXT_DATA__" in x.text
|
|
164
164
|
attempts.append(_attempt("x", f"syndication-timeline#{attempt_no+1}", ok,
|
|
165
165
|
x.status_code, x.text,
|
|
@@ -175,7 +175,7 @@ def _x(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
|
175
175
|
|
|
176
176
|
|
|
177
177
|
# --- youtube -----------------------------------------------------------------
|
|
178
|
-
def _ytdlp_argv() -> Optional[list[str]]:
|
|
178
|
+
def _ytdlp_argv() -> Optional[list[str]]:
|
|
179
179
|
"""Best yt-dlp invocation for this environment, or None if unavailable.
|
|
180
180
|
|
|
181
181
|
Prefers the ``yt-dlp`` console script on PATH; falls back to
|
|
@@ -192,261 +192,261 @@ def _ytdlp_argv() -> Optional[list[str]]:
|
|
|
192
192
|
return None
|
|
193
193
|
|
|
194
194
|
|
|
195
|
-
def _clean_caption_line(line: str) -> str:
|
|
196
|
-
line = re.sub(r"<[^>]+>", "", line)
|
|
197
|
-
line = html.unescape(line).strip()
|
|
198
|
-
return "" if line.startswith(("Kind:", "Language:")) else line
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
def _vtt_segments(vtt: str) -> list[dict[str, str]]:
|
|
202
|
-
"""Parse bounded WebVTT cues so final citations can include timestamps."""
|
|
203
|
-
source = (vtt or "")[:2_000_000].splitlines()
|
|
204
|
-
segments: list[dict[str, str]] = []
|
|
205
|
-
total = 0
|
|
206
|
-
index = 0
|
|
207
|
-
while index < len(source):
|
|
208
|
-
timing = source[index].strip()
|
|
209
|
-
if "-->" not in timing:
|
|
210
|
-
index += 1
|
|
211
|
-
continue
|
|
212
|
-
start, end = [part.strip().split()[0] for part in timing.split("-->", 1)]
|
|
213
|
-
index += 1
|
|
214
|
-
cue: list[str] = []
|
|
215
|
-
while index < len(source) and source[index].strip():
|
|
216
|
-
line = _clean_caption_line(source[index])
|
|
217
|
-
if line and (not cue or cue[-1] != line):
|
|
218
|
-
cue.append(line)
|
|
219
|
-
index += 1
|
|
220
|
-
text = " ".join(cue).strip()
|
|
221
|
-
if text and segments and text.startswith(segments[-1]["text"]):
|
|
222
|
-
text = text[len(segments[-1]["text"]):].strip()
|
|
223
|
-
if text and (not segments or segments[-1]["text"] != text):
|
|
224
|
-
take = text[:max(0, 500_000 - total)]
|
|
225
|
-
if take:
|
|
226
|
-
segments.append({"start": start, "end": end, "text": take})
|
|
227
|
-
total += len(take)
|
|
228
|
-
if total >= 500_000 or len(segments) >= 5000:
|
|
229
|
-
break
|
|
230
|
-
index += 1
|
|
231
|
-
return segments
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
def _vtt_to_text(vtt: str) -> str:
|
|
235
|
-
lines: list[str] = []
|
|
236
|
-
for segment in _vtt_segments(vtt):
|
|
237
|
-
text = segment["text"]
|
|
238
|
-
if not lines or lines[-1] != text:
|
|
239
|
-
lines.append(text)
|
|
240
|
-
return "\n".join(lines)[:500_000]
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
def _choose_subtitle_language(metadata: dict, preferred: tuple[str, ...] = ("ko", "en")) -> tuple[str, str] | None:
|
|
244
|
-
"""Choose a creator subtitle first, then an automatic caption."""
|
|
245
|
-
for key, source in (("subtitles", "creator"), ("automatic_captions", "automatic")):
|
|
246
|
-
available = metadata.get(key) or {}
|
|
247
|
-
for wanted in preferred:
|
|
248
|
-
if wanted in available:
|
|
249
|
-
return wanted, source
|
|
250
|
-
match = next((lang for lang in available if lang.lower().startswith(wanted.lower() + "-")), None)
|
|
251
|
-
if match:
|
|
252
|
-
return match, source
|
|
253
|
-
return None
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
_YTDLP_ARGV_CACHE: Optional[list[str]] = None
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
def _ensure_ytdlp_argv(allow_install: bool) -> Optional[list[str]]:
|
|
260
|
-
"""Resolve yt-dlp, optionally installing it into an isolated first-use venv."""
|
|
261
|
-
global _YTDLP_ARGV_CACHE
|
|
262
|
-
available = _ytdlp_argv()
|
|
263
|
-
if available is not None:
|
|
264
|
-
return available
|
|
265
|
-
if not allow_install:
|
|
266
|
-
return None
|
|
267
|
-
if os.environ.get("INSANE_AUTO_INSTALL", "").strip().lower() in ("0", "false", "no"):
|
|
268
|
-
return None
|
|
269
|
-
if _YTDLP_ARGV_CACHE is not None:
|
|
270
|
-
return _YTDLP_ARGV_CACHE or None
|
|
271
|
-
root = os.path.expanduser("~/.insane-search/media-venv")
|
|
272
|
-
python = os.path.join(
|
|
273
|
-
root, "Scripts", "python.exe") if os.name == "nt" else os.path.join(root, "bin", "python")
|
|
274
|
-
try:
|
|
275
|
-
if not os.path.isfile(python):
|
|
276
|
-
subprocess.run(
|
|
277
|
-
[sys.executable, "-m", "venv", root], capture_output=True,
|
|
278
|
-
text=True, timeout=180, check=False)
|
|
279
|
-
probe = subprocess.run(
|
|
280
|
-
[python, "-m", "yt_dlp", "--version"], capture_output=True,
|
|
281
|
-
text=True, timeout=30, check=False)
|
|
282
|
-
if probe.returncode != 0:
|
|
283
|
-
installed = subprocess.run(
|
|
284
|
-
[python, "-m", "pip", "install", "yt-dlp", "-q"],
|
|
285
|
-
capture_output=True, text=True, timeout=300, check=False)
|
|
286
|
-
if installed.returncode != 0:
|
|
287
|
-
_YTDLP_ARGV_CACHE = []
|
|
288
|
-
return None
|
|
289
|
-
_YTDLP_ARGV_CACHE = [python, "-m", "yt_dlp"]
|
|
290
|
-
return list(_YTDLP_ARGV_CACHE)
|
|
291
|
-
except Exception:
|
|
292
|
-
_YTDLP_ARGV_CACHE = []
|
|
293
|
-
return None
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
def _attach_transcript(metadata: dict, url: str, argv: list[str], timeout: int,
|
|
297
|
-
proxy: Optional[str], cookie_path: Optional[str] = None) -> dict:
|
|
298
|
-
choice = _choose_subtitle_language(metadata)
|
|
299
|
-
if choice is None:
|
|
300
|
-
metadata["research_transcript"] = {"available": False, "reason": "no_subtitles"}
|
|
301
|
-
return metadata
|
|
302
|
-
language, source = choice
|
|
303
|
-
try:
|
|
304
|
-
with tempfile.TemporaryDirectory(prefix="insane-media-") as root:
|
|
305
|
-
command = argv + [
|
|
306
|
-
"--write-subs", "--write-auto-subs", "--sub-langs", language,
|
|
307
|
-
"--sub-format", "vtt", "--skip-download", "--no-playlist",
|
|
308
|
-
"--paths", root, "-o", "%(id)s.%(ext)s",
|
|
309
|
-
]
|
|
310
|
-
if proxy:
|
|
311
|
-
command += ["--proxy", proxy]
|
|
312
|
-
if cookie_path:
|
|
313
|
-
command += ["--cookies", cookie_path]
|
|
314
|
-
completed = subprocess.run(
|
|
315
|
-
command + [url], capture_output=True, text=True,
|
|
316
|
-
timeout=max(timeout, 90), check=False)
|
|
317
|
-
files = sorted(Path(root).glob("*.vtt"))
|
|
318
|
-
text = ""
|
|
319
|
-
segments: list[dict[str, str]] = []
|
|
320
|
-
if completed.returncode == 0 and files:
|
|
321
|
-
raw_vtt = files[0].read_text(encoding="utf-8", errors="replace")
|
|
322
|
-
segments = _vtt_segments(raw_vtt)
|
|
323
|
-
text = _vtt_to_text(raw_vtt)
|
|
324
|
-
metadata["research_transcript"] = {
|
|
325
|
-
"available": bool(text),
|
|
326
|
-
"language": language,
|
|
327
|
-
"source": source,
|
|
328
|
-
"text": text,
|
|
329
|
-
"segments": segments,
|
|
330
|
-
"reason": "" if text else "subtitle_download_failed",
|
|
331
|
-
}
|
|
332
|
-
except subprocess.TimeoutExpired:
|
|
333
|
-
metadata["research_transcript"] = {"available": False, "reason": "subtitle_timeout"}
|
|
334
|
-
except Exception as exc:
|
|
335
|
-
metadata["research_transcript"] = {
|
|
336
|
-
"available": False, "reason": f"subtitle_error:{type(exc).__name__}"}
|
|
337
|
-
return metadata
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
def _compact_media_metadata(metadata: dict) -> dict:
|
|
341
|
-
"""Drop signed streams and extractor internals that waste research tokens."""
|
|
342
|
-
keys = (
|
|
343
|
-
"id", "title", "description", "uploader", "uploader_id", "channel",
|
|
344
|
-
"channel_id", "duration", "duration_string", "upload_date", "timestamp",
|
|
345
|
-
"release_date", "webpage_url", "original_url", "availability", "live_status",
|
|
346
|
-
"view_count", "like_count", "comment_count", "categories", "tags", "chapters",
|
|
347
|
-
)
|
|
348
|
-
compact = {key: metadata[key] for key in keys if metadata.get(key) is not None}
|
|
349
|
-
compact["subtitle_languages"] = sorted((metadata.get("subtitles") or {}).keys())[:100]
|
|
350
|
-
compact["automatic_caption_languages"] = sorted(
|
|
351
|
-
(metadata.get("automatic_captions") or {}).keys())[:100]
|
|
352
|
-
if "research_transcript" in metadata:
|
|
353
|
-
compact["research_transcript"] = metadata["research_transcript"]
|
|
354
|
-
return compact
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
def _write_cookie_jar(cookies: list[dict]) -> str:
|
|
358
|
-
handle = tempfile.NamedTemporaryFile(
|
|
359
|
-
"w", encoding="utf-8", prefix="insane-media-cookies-", suffix=".txt",
|
|
360
|
-
delete=False)
|
|
361
|
-
try:
|
|
362
|
-
handle.write("# Netscape HTTP Cookie File\n")
|
|
363
|
-
for cookie in cookies:
|
|
364
|
-
domain = str(cookie.get("domain") or "")
|
|
365
|
-
include_subdomains = "TRUE" if domain.startswith(".") else "FALSE"
|
|
366
|
-
fields = (
|
|
367
|
-
domain, include_subdomains, str(cookie.get("path") or "/"),
|
|
368
|
-
"TRUE" if cookie.get("secure") else "FALSE",
|
|
369
|
-
str(int(cookie.get("expires") or 0)), str(cookie.get("name") or ""),
|
|
370
|
-
str(cookie.get("value") or ""),
|
|
371
|
-
)
|
|
372
|
-
handle.write("\t".join(fields) + "\n")
|
|
373
|
-
return handle.name
|
|
374
|
-
finally:
|
|
375
|
-
handle.close()
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
def _media(url: str, timeout: int, proxy: Optional[str] = None,
|
|
379
|
-
include_transcript: bool = False, platform: str = "media",
|
|
380
|
-
cookies: Optional[list[dict]] = None,
|
|
381
|
-
auto_install: bool = False) -> dict:
|
|
382
|
-
attempts: list[dict] = []
|
|
383
|
-
argv = _ensure_ytdlp_argv(auto_install)
|
|
384
|
-
if argv is None:
|
|
385
|
-
attempts.append(_attempt(platform, "yt-dlp", False, 0, "", "yt-dlp not installed"))
|
|
386
|
-
return {"platform": platform, "ok": False, "route": None, "content": "",
|
|
387
|
-
"final_url": url, "attempts": attempts}
|
|
388
|
-
cookie_path = _write_cookie_jar(cookies) if cookies else None
|
|
389
|
-
try:
|
|
390
|
-
command = argv + ["--dump-json", "--skip-download"]
|
|
391
|
-
if proxy:
|
|
392
|
-
command += ["--proxy", proxy]
|
|
393
|
-
if cookie_path:
|
|
394
|
-
command += ["--cookies", cookie_path]
|
|
395
|
-
p = subprocess.run(
|
|
396
|
-
command + [url],
|
|
397
|
-
capture_output=True, text=True, timeout=max(timeout, 60),
|
|
398
|
-
)
|
|
399
|
-
ok = p.returncode == 0 and p.stdout.strip().startswith("{")
|
|
400
|
-
diagnostic = p.stderr or ""
|
|
401
|
-
if cookie_path:
|
|
402
|
-
diagnostic = diagnostic.replace(cookie_path, "<cookie-file>")
|
|
403
|
-
if cookies:
|
|
404
|
-
from .session_input import redact_cookie_values
|
|
405
|
-
diagnostic = redact_cookie_values(diagnostic, cookies) or ""
|
|
406
|
-
note = "json" if ok else diagnostic.strip()[:80]
|
|
407
|
-
attempts.append(_attempt(platform, "yt-dlp", ok, 200 if ok else 0, p.stdout, note))
|
|
408
|
-
if ok:
|
|
409
|
-
metadata = json.loads(p.stdout)
|
|
410
|
-
if include_transcript:
|
|
411
|
-
metadata = _attach_transcript(
|
|
412
|
-
metadata, url, argv, timeout, proxy, cookie_path)
|
|
413
|
-
if include_transcript or platform == "media":
|
|
414
|
-
metadata = _compact_media_metadata(metadata)
|
|
415
|
-
transcript_available = bool(
|
|
416
|
-
(metadata.get("research_transcript") or {}).get("available"))
|
|
417
|
-
return {"platform": platform, "ok": True, "route": "yt-dlp",
|
|
418
|
-
"content": json.dumps(metadata, ensure_ascii=False),
|
|
419
|
-
"final_url": url, "attempts": attempts,
|
|
420
|
-
"content_kind": ("media+transcript" if transcript_available
|
|
421
|
-
else "media_metadata"),
|
|
422
|
-
"transcript": transcript_available}
|
|
423
|
-
except FileNotFoundError:
|
|
424
|
-
attempts.append(_attempt(platform, "yt-dlp", False, 0, "", "yt-dlp not installed"))
|
|
425
|
-
except Exception as e:
|
|
426
|
-
attempts.append(_attempt(platform, "yt-dlp", False, 0, "", f"{type(e).__name__}"))
|
|
427
|
-
finally:
|
|
428
|
-
if cookie_path:
|
|
429
|
-
try:
|
|
430
|
-
Path(cookie_path).unlink()
|
|
431
|
-
except OSError:
|
|
432
|
-
pass
|
|
433
|
-
return {"platform": platform, "ok": False, "route": None, "content": "",
|
|
434
|
-
"final_url": url, "attempts": attempts}
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
def _youtube(url: str, timeout: int, proxy: Optional[str] = None,
|
|
438
|
-
include_transcript: bool = False,
|
|
439
|
-
cookies: Optional[list[dict]] = None) -> dict:
|
|
440
|
-
return _media(
|
|
441
|
-
url, timeout, proxy, include_transcript, platform="youtube", cookies=cookies,
|
|
442
|
-
auto_install=include_transcript)
|
|
195
|
+
def _clean_caption_line(line: str) -> str:
|
|
196
|
+
line = re.sub(r"<[^>]+>", "", line)
|
|
197
|
+
line = html.unescape(line).strip()
|
|
198
|
+
return "" if line.startswith(("Kind:", "Language:")) else line
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _vtt_segments(vtt: str) -> list[dict[str, str]]:
|
|
202
|
+
"""Parse bounded WebVTT cues so final citations can include timestamps."""
|
|
203
|
+
source = (vtt or "")[:2_000_000].splitlines()
|
|
204
|
+
segments: list[dict[str, str]] = []
|
|
205
|
+
total = 0
|
|
206
|
+
index = 0
|
|
207
|
+
while index < len(source):
|
|
208
|
+
timing = source[index].strip()
|
|
209
|
+
if "-->" not in timing:
|
|
210
|
+
index += 1
|
|
211
|
+
continue
|
|
212
|
+
start, end = [part.strip().split()[0] for part in timing.split("-->", 1)]
|
|
213
|
+
index += 1
|
|
214
|
+
cue: list[str] = []
|
|
215
|
+
while index < len(source) and source[index].strip():
|
|
216
|
+
line = _clean_caption_line(source[index])
|
|
217
|
+
if line and (not cue or cue[-1] != line):
|
|
218
|
+
cue.append(line)
|
|
219
|
+
index += 1
|
|
220
|
+
text = " ".join(cue).strip()
|
|
221
|
+
if text and segments and text.startswith(segments[-1]["text"]):
|
|
222
|
+
text = text[len(segments[-1]["text"]):].strip()
|
|
223
|
+
if text and (not segments or segments[-1]["text"] != text):
|
|
224
|
+
take = text[:max(0, 500_000 - total)]
|
|
225
|
+
if take:
|
|
226
|
+
segments.append({"start": start, "end": end, "text": take})
|
|
227
|
+
total += len(take)
|
|
228
|
+
if total >= 500_000 or len(segments) >= 5000:
|
|
229
|
+
break
|
|
230
|
+
index += 1
|
|
231
|
+
return segments
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _vtt_to_text(vtt: str) -> str:
|
|
235
|
+
lines: list[str] = []
|
|
236
|
+
for segment in _vtt_segments(vtt):
|
|
237
|
+
text = segment["text"]
|
|
238
|
+
if not lines or lines[-1] != text:
|
|
239
|
+
lines.append(text)
|
|
240
|
+
return "\n".join(lines)[:500_000]
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _choose_subtitle_language(metadata: dict, preferred: tuple[str, ...] = ("ko", "en")) -> tuple[str, str] | None:
|
|
244
|
+
"""Choose a creator subtitle first, then an automatic caption."""
|
|
245
|
+
for key, source in (("subtitles", "creator"), ("automatic_captions", "automatic")):
|
|
246
|
+
available = metadata.get(key) or {}
|
|
247
|
+
for wanted in preferred:
|
|
248
|
+
if wanted in available:
|
|
249
|
+
return wanted, source
|
|
250
|
+
match = next((lang for lang in available if lang.lower().startswith(wanted.lower() + "-")), None)
|
|
251
|
+
if match:
|
|
252
|
+
return match, source
|
|
253
|
+
return None
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
_YTDLP_ARGV_CACHE: Optional[list[str]] = None
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _ensure_ytdlp_argv(allow_install: bool) -> Optional[list[str]]:
|
|
260
|
+
"""Resolve yt-dlp, optionally installing it into an isolated first-use venv."""
|
|
261
|
+
global _YTDLP_ARGV_CACHE
|
|
262
|
+
available = _ytdlp_argv()
|
|
263
|
+
if available is not None:
|
|
264
|
+
return available
|
|
265
|
+
if not allow_install:
|
|
266
|
+
return None
|
|
267
|
+
if os.environ.get("INSANE_AUTO_INSTALL", "").strip().lower() in ("0", "false", "no"):
|
|
268
|
+
return None
|
|
269
|
+
if _YTDLP_ARGV_CACHE is not None:
|
|
270
|
+
return _YTDLP_ARGV_CACHE or None
|
|
271
|
+
root = os.path.expanduser("~/.insane-search/media-venv")
|
|
272
|
+
python = os.path.join(
|
|
273
|
+
root, "Scripts", "python.exe") if os.name == "nt" else os.path.join(root, "bin", "python")
|
|
274
|
+
try:
|
|
275
|
+
if not os.path.isfile(python):
|
|
276
|
+
subprocess.run(
|
|
277
|
+
[sys.executable, "-m", "venv", root], capture_output=True,
|
|
278
|
+
text=True, timeout=180, check=False)
|
|
279
|
+
probe = subprocess.run(
|
|
280
|
+
[python, "-m", "yt_dlp", "--version"], capture_output=True,
|
|
281
|
+
text=True, timeout=30, check=False)
|
|
282
|
+
if probe.returncode != 0:
|
|
283
|
+
installed = subprocess.run(
|
|
284
|
+
[python, "-m", "pip", "install", "yt-dlp", "-q"],
|
|
285
|
+
capture_output=True, text=True, timeout=300, check=False)
|
|
286
|
+
if installed.returncode != 0:
|
|
287
|
+
_YTDLP_ARGV_CACHE = []
|
|
288
|
+
return None
|
|
289
|
+
_YTDLP_ARGV_CACHE = [python, "-m", "yt_dlp"]
|
|
290
|
+
return list(_YTDLP_ARGV_CACHE)
|
|
291
|
+
except Exception:
|
|
292
|
+
_YTDLP_ARGV_CACHE = []
|
|
293
|
+
return None
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _attach_transcript(metadata: dict, url: str, argv: list[str], timeout: int,
|
|
297
|
+
proxy: Optional[str], cookie_path: Optional[str] = None) -> dict:
|
|
298
|
+
choice = _choose_subtitle_language(metadata)
|
|
299
|
+
if choice is None:
|
|
300
|
+
metadata["research_transcript"] = {"available": False, "reason": "no_subtitles"}
|
|
301
|
+
return metadata
|
|
302
|
+
language, source = choice
|
|
303
|
+
try:
|
|
304
|
+
with tempfile.TemporaryDirectory(prefix="insane-media-") as root:
|
|
305
|
+
command = argv + [
|
|
306
|
+
"--write-subs", "--write-auto-subs", "--sub-langs", language,
|
|
307
|
+
"--sub-format", "vtt", "--skip-download", "--no-playlist",
|
|
308
|
+
"--paths", root, "-o", "%(id)s.%(ext)s",
|
|
309
|
+
]
|
|
310
|
+
if proxy:
|
|
311
|
+
command += ["--proxy", proxy]
|
|
312
|
+
if cookie_path:
|
|
313
|
+
command += ["--cookies", cookie_path]
|
|
314
|
+
completed = subprocess.run(
|
|
315
|
+
command + [url], capture_output=True, text=True,
|
|
316
|
+
timeout=max(timeout, 90), check=False)
|
|
317
|
+
files = sorted(Path(root).glob("*.vtt"))
|
|
318
|
+
text = ""
|
|
319
|
+
segments: list[dict[str, str]] = []
|
|
320
|
+
if completed.returncode == 0 and files:
|
|
321
|
+
raw_vtt = files[0].read_text(encoding="utf-8", errors="replace")
|
|
322
|
+
segments = _vtt_segments(raw_vtt)
|
|
323
|
+
text = _vtt_to_text(raw_vtt)
|
|
324
|
+
metadata["research_transcript"] = {
|
|
325
|
+
"available": bool(text),
|
|
326
|
+
"language": language,
|
|
327
|
+
"source": source,
|
|
328
|
+
"text": text,
|
|
329
|
+
"segments": segments,
|
|
330
|
+
"reason": "" if text else "subtitle_download_failed",
|
|
331
|
+
}
|
|
332
|
+
except subprocess.TimeoutExpired:
|
|
333
|
+
metadata["research_transcript"] = {"available": False, "reason": "subtitle_timeout"}
|
|
334
|
+
except Exception as exc:
|
|
335
|
+
metadata["research_transcript"] = {
|
|
336
|
+
"available": False, "reason": f"subtitle_error:{type(exc).__name__}"}
|
|
337
|
+
return metadata
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _compact_media_metadata(metadata: dict) -> dict:
|
|
341
|
+
"""Drop signed streams and extractor internals that waste research tokens."""
|
|
342
|
+
keys = (
|
|
343
|
+
"id", "title", "description", "uploader", "uploader_id", "channel",
|
|
344
|
+
"channel_id", "duration", "duration_string", "upload_date", "timestamp",
|
|
345
|
+
"release_date", "webpage_url", "original_url", "availability", "live_status",
|
|
346
|
+
"view_count", "like_count", "comment_count", "categories", "tags", "chapters",
|
|
347
|
+
)
|
|
348
|
+
compact = {key: metadata[key] for key in keys if metadata.get(key) is not None}
|
|
349
|
+
compact["subtitle_languages"] = sorted((metadata.get("subtitles") or {}).keys())[:100]
|
|
350
|
+
compact["automatic_caption_languages"] = sorted(
|
|
351
|
+
(metadata.get("automatic_captions") or {}).keys())[:100]
|
|
352
|
+
if "research_transcript" in metadata:
|
|
353
|
+
compact["research_transcript"] = metadata["research_transcript"]
|
|
354
|
+
return compact
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _write_cookie_jar(cookies: list[dict]) -> str:
|
|
358
|
+
handle = tempfile.NamedTemporaryFile(
|
|
359
|
+
"w", encoding="utf-8", prefix="insane-media-cookies-", suffix=".txt",
|
|
360
|
+
delete=False)
|
|
361
|
+
try:
|
|
362
|
+
handle.write("# Netscape HTTP Cookie File\n")
|
|
363
|
+
for cookie in cookies:
|
|
364
|
+
domain = str(cookie.get("domain") or "")
|
|
365
|
+
include_subdomains = "TRUE" if domain.startswith(".") else "FALSE"
|
|
366
|
+
fields = (
|
|
367
|
+
domain, include_subdomains, str(cookie.get("path") or "/"),
|
|
368
|
+
"TRUE" if cookie.get("secure") else "FALSE",
|
|
369
|
+
str(int(cookie.get("expires") or 0)), str(cookie.get("name") or ""),
|
|
370
|
+
str(cookie.get("value") or ""),
|
|
371
|
+
)
|
|
372
|
+
handle.write("\t".join(fields) + "\n")
|
|
373
|
+
return handle.name
|
|
374
|
+
finally:
|
|
375
|
+
handle.close()
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _media(url: str, timeout: int, proxy: Optional[str] = None,
|
|
379
|
+
include_transcript: bool = False, platform: str = "media",
|
|
380
|
+
cookies: Optional[list[dict]] = None,
|
|
381
|
+
auto_install: bool = False) -> dict:
|
|
382
|
+
attempts: list[dict] = []
|
|
383
|
+
argv = _ensure_ytdlp_argv(auto_install)
|
|
384
|
+
if argv is None:
|
|
385
|
+
attempts.append(_attempt(platform, "yt-dlp", False, 0, "", "yt-dlp not installed"))
|
|
386
|
+
return {"platform": platform, "ok": False, "route": None, "content": "",
|
|
387
|
+
"final_url": url, "attempts": attempts}
|
|
388
|
+
cookie_path = _write_cookie_jar(cookies) if cookies else None
|
|
389
|
+
try:
|
|
390
|
+
command = argv + ["--dump-json", "--skip-download"]
|
|
391
|
+
if proxy:
|
|
392
|
+
command += ["--proxy", proxy]
|
|
393
|
+
if cookie_path:
|
|
394
|
+
command += ["--cookies", cookie_path]
|
|
395
|
+
p = subprocess.run(
|
|
396
|
+
command + [url],
|
|
397
|
+
capture_output=True, text=True, timeout=max(timeout, 60),
|
|
398
|
+
)
|
|
399
|
+
ok = p.returncode == 0 and p.stdout.strip().startswith("{")
|
|
400
|
+
diagnostic = p.stderr or ""
|
|
401
|
+
if cookie_path:
|
|
402
|
+
diagnostic = diagnostic.replace(cookie_path, "<cookie-file>")
|
|
403
|
+
if cookies:
|
|
404
|
+
from .session_input import redact_cookie_values
|
|
405
|
+
diagnostic = redact_cookie_values(diagnostic, cookies) or ""
|
|
406
|
+
note = "json" if ok else diagnostic.strip()[:80]
|
|
407
|
+
attempts.append(_attempt(platform, "yt-dlp", ok, 200 if ok else 0, p.stdout, note))
|
|
408
|
+
if ok:
|
|
409
|
+
metadata = json.loads(p.stdout)
|
|
410
|
+
if include_transcript:
|
|
411
|
+
metadata = _attach_transcript(
|
|
412
|
+
metadata, url, argv, timeout, proxy, cookie_path)
|
|
413
|
+
if include_transcript or platform == "media":
|
|
414
|
+
metadata = _compact_media_metadata(metadata)
|
|
415
|
+
transcript_available = bool(
|
|
416
|
+
(metadata.get("research_transcript") or {}).get("available"))
|
|
417
|
+
return {"platform": platform, "ok": True, "route": "yt-dlp",
|
|
418
|
+
"content": json.dumps(metadata, ensure_ascii=False),
|
|
419
|
+
"final_url": url, "attempts": attempts,
|
|
420
|
+
"content_kind": ("media+transcript" if transcript_available
|
|
421
|
+
else "media_metadata"),
|
|
422
|
+
"transcript": transcript_available}
|
|
423
|
+
except FileNotFoundError:
|
|
424
|
+
attempts.append(_attempt(platform, "yt-dlp", False, 0, "", "yt-dlp not installed"))
|
|
425
|
+
except Exception as e:
|
|
426
|
+
attempts.append(_attempt(platform, "yt-dlp", False, 0, "", f"{type(e).__name__}"))
|
|
427
|
+
finally:
|
|
428
|
+
if cookie_path:
|
|
429
|
+
try:
|
|
430
|
+
Path(cookie_path).unlink()
|
|
431
|
+
except OSError:
|
|
432
|
+
pass
|
|
433
|
+
return {"platform": platform, "ok": False, "route": None, "content": "",
|
|
434
|
+
"final_url": url, "attempts": attempts}
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _youtube(url: str, timeout: int, proxy: Optional[str] = None,
|
|
438
|
+
include_transcript: bool = False,
|
|
439
|
+
cookies: Optional[list[dict]] = None) -> dict:
|
|
440
|
+
return _media(
|
|
441
|
+
url, timeout, proxy, include_transcript, platform="youtube", cookies=cookies,
|
|
442
|
+
auto_install=include_transcript)
|
|
443
443
|
|
|
444
444
|
|
|
445
445
|
# --- threads -----------------------------------------------------------------
|
|
446
446
|
_THREADS_POST_RE = re.compile(r"/post/([A-Za-z0-9_-]+)")
|
|
447
447
|
|
|
448
448
|
|
|
449
|
-
def _threads(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
449
|
+
def _threads(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
450
450
|
"""Threads video post → signed CDN URLs from the page's inline JSON.
|
|
451
451
|
|
|
452
452
|
yt-dlp has no Threads extractor, but an anonymous GET with a curl_cffi
|
|
@@ -463,7 +463,7 @@ def _threads(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
|
|
|
463
463
|
"final_url": url, "attempts": attempts}
|
|
464
464
|
code = m.group(1)
|
|
465
465
|
try:
|
|
466
|
-
x = _cffi_get(url, timeout=timeout, proxy=proxy)
|
|
466
|
+
x = _cffi_get(url, timeout=timeout, proxy=proxy)
|
|
467
467
|
raw = x.text if x.status_code == 200 else ""
|
|
468
468
|
code_pos = [c.start() for c in re.finditer(r'"code"\s*:\s*"%s"' % re.escape(code), raw)]
|
|
469
469
|
blocks = list(re.finditer(r'"video_versions"\s*:\s*\[(.*?)\]', raw))
|
|
@@ -498,100 +498,100 @@ _ROUTERS = {"reddit": _reddit, "x": _x, "youtube": _youtube, "threads": _threads
|
|
|
498
498
|
|
|
499
499
|
|
|
500
500
|
# --- public entrypoint -------------------------------------------------------
|
|
501
|
-
def route_media(url: str, *, timeout: int = 15, proxy: Optional[str] = None,
|
|
502
|
-
include_transcript: bool = False,
|
|
503
|
-
cookies: Optional[list[dict]] = None) -> dict:
|
|
504
|
-
return _media(
|
|
505
|
-
url, timeout, proxy, include_transcript, platform="media", cookies=cookies,
|
|
506
|
-
auto_install=True)
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
class _ArchiveResponse:
|
|
510
|
-
def __init__(self, status: int, text: str, url: str, transport: str):
|
|
511
|
-
self.status_code = status
|
|
512
|
-
self.text = text
|
|
513
|
-
self.url = url
|
|
514
|
-
self.transport = transport
|
|
515
|
-
|
|
516
|
-
def json(self):
|
|
517
|
-
return json.loads(self.text)
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
def _archive_get(url: str, timeout: int, proxy: Optional[str]) -> _ArchiveResponse:
|
|
521
|
-
try:
|
|
522
|
-
response = _cffi_get(url, timeout=timeout, proxy=proxy)
|
|
523
|
-
response.transport = "curl_cffi"
|
|
524
|
-
return response
|
|
525
|
-
except Exception:
|
|
526
|
-
from .executor import run_playwright_fallback
|
|
527
|
-
attempt, body = run_playwright_fallback(
|
|
528
|
-
url, profile_id="unknown_challenge", force_executor="scrapling",
|
|
529
|
-
timeout=max(timeout, 60), proxy=proxy)
|
|
530
|
-
inner = str(getattr(attempt, "_inner_text", "") or "").strip()
|
|
531
|
-
if inner:
|
|
532
|
-
text = inner
|
|
533
|
-
else:
|
|
534
|
-
text = html.unescape(re.sub(r"(?s)<[^>]+>", " ", body or ""))
|
|
535
|
-
text = re.sub(r"\s+", " ", text).strip()
|
|
536
|
-
return _ArchiveResponse(
|
|
537
|
-
int(getattr(attempt, "status", 0) or 0), text if text else body,
|
|
538
|
-
str(getattr(attempt, "url", "") or url), "system_browser")
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
def route_archive(url: str, *, timeout: int = 15, proxy: Optional[str] = None) -> dict:
|
|
542
|
-
"""Retrieve the closest public Wayback snapshot without treating it as current."""
|
|
543
|
-
from .safety import allow_private_default, classify_url
|
|
544
|
-
|
|
545
|
-
attempts: list[dict] = []
|
|
546
|
-
safe, reason = classify_url(url, allow_private_default())
|
|
547
|
-
if not safe:
|
|
548
|
-
attempts.append(_attempt("archive", "wayback-available", False, 0, "", reason))
|
|
549
|
-
return {"platform": "archive", "ok": False, "route": None, "content": "",
|
|
550
|
-
"final_url": url, "attempts": attempts, "timestamp": ""}
|
|
551
|
-
try:
|
|
552
|
-
api_url = "https://archive.org/wayback/available?url=" + quote(url, safe="")
|
|
553
|
-
response = _archive_get(api_url, timeout=timeout, proxy=proxy)
|
|
554
|
-
data = response.json() if response.status_code == 200 else {}
|
|
555
|
-
closest = ((data.get("archived_snapshots") or {}).get("closest") or {})
|
|
556
|
-
snapshot = str(closest.get("url") or "")
|
|
557
|
-
timestamp = str(closest.get("timestamp") or "")
|
|
558
|
-
valid_snapshot = (
|
|
559
|
-
closest.get("available") is True
|
|
560
|
-
and str(closest.get("status") or "") == "200"
|
|
561
|
-
and urlsplit(snapshot).hostname == "web.archive.org"
|
|
562
|
-
)
|
|
563
|
-
attempts.append(_attempt(
|
|
564
|
-
"archive", "wayback-available", valid_snapshot, response.status_code,
|
|
565
|
-
response.text,
|
|
566
|
-
(("snapshot:" + getattr(response, "transport", "unknown")) if valid_snapshot
|
|
567
|
-
else (f"status={response.status_code}" if response.status_code != 200
|
|
568
|
-
else "no_snapshot"))))
|
|
569
|
-
if not valid_snapshot:
|
|
570
|
-
return {"platform": "archive", "ok": False, "route": None, "content": "",
|
|
571
|
-
"final_url": url, "attempts": attempts, "timestamp": ""}
|
|
572
|
-
page = _archive_get(snapshot, timeout=max(timeout, 30), proxy=proxy)
|
|
573
|
-
ok = page.status_code == 200 and len(page.text or "") >= 200
|
|
574
|
-
attempts.append(_attempt(
|
|
575
|
-
"archive", "wayback-snapshot", ok, page.status_code, page.text,
|
|
576
|
-
(("historical_snapshot:" + getattr(page, "transport", "unknown"))
|
|
577
|
-
if ok else "snapshot_fetch_failed")))
|
|
578
|
-
return {"platform": "archive", "ok": ok,
|
|
579
|
-
"route": "wayback" if ok else None,
|
|
580
|
-
"content": page.text if ok else "", "final_url": snapshot if ok else url,
|
|
581
|
-
"attempts": attempts, "timestamp": timestamp if ok else ""}
|
|
582
|
-
except Exception as exc:
|
|
583
|
-
attempts.append(_attempt(
|
|
584
|
-
"archive", "wayback-available", False, 0, "", f"{type(exc).__name__}"))
|
|
585
|
-
return {"platform": "archive", "ok": False, "route": None, "content": "",
|
|
586
|
-
"final_url": url, "attempts": attempts, "timestamp": ""}
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
def route(url: str, *, timeout: int = 15, proxy: Optional[str] = None,
|
|
590
|
-
include_media_transcript: bool = False,
|
|
591
|
-
cookies: Optional[list[dict]] = None) -> Optional[dict]:
|
|
592
|
-
platform = _detect(url)
|
|
593
|
-
if platform is None:
|
|
594
|
-
return None
|
|
595
|
-
if platform == "youtube":
|
|
596
|
-
return _youtube(url, timeout, proxy, include_media_transcript, cookies)
|
|
597
|
-
return _ROUTERS[platform](url, timeout, proxy)
|
|
501
|
+
def route_media(url: str, *, timeout: int = 15, proxy: Optional[str] = None,
|
|
502
|
+
include_transcript: bool = False,
|
|
503
|
+
cookies: Optional[list[dict]] = None) -> dict:
|
|
504
|
+
return _media(
|
|
505
|
+
url, timeout, proxy, include_transcript, platform="media", cookies=cookies,
|
|
506
|
+
auto_install=True)
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
class _ArchiveResponse:
|
|
510
|
+
def __init__(self, status: int, text: str, url: str, transport: str):
|
|
511
|
+
self.status_code = status
|
|
512
|
+
self.text = text
|
|
513
|
+
self.url = url
|
|
514
|
+
self.transport = transport
|
|
515
|
+
|
|
516
|
+
def json(self):
|
|
517
|
+
return json.loads(self.text)
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _archive_get(url: str, timeout: int, proxy: Optional[str]) -> _ArchiveResponse:
|
|
521
|
+
try:
|
|
522
|
+
response = _cffi_get(url, timeout=timeout, proxy=proxy)
|
|
523
|
+
response.transport = "curl_cffi"
|
|
524
|
+
return response
|
|
525
|
+
except Exception:
|
|
526
|
+
from .executor import run_playwright_fallback
|
|
527
|
+
attempt, body = run_playwright_fallback(
|
|
528
|
+
url, profile_id="unknown_challenge", force_executor="scrapling",
|
|
529
|
+
timeout=max(timeout, 60), proxy=proxy)
|
|
530
|
+
inner = str(getattr(attempt, "_inner_text", "") or "").strip()
|
|
531
|
+
if inner:
|
|
532
|
+
text = inner
|
|
533
|
+
else:
|
|
534
|
+
text = html.unescape(re.sub(r"(?s)<[^>]+>", " ", body or ""))
|
|
535
|
+
text = re.sub(r"\s+", " ", text).strip()
|
|
536
|
+
return _ArchiveResponse(
|
|
537
|
+
int(getattr(attempt, "status", 0) or 0), text if text else body,
|
|
538
|
+
str(getattr(attempt, "url", "") or url), "system_browser")
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def route_archive(url: str, *, timeout: int = 15, proxy: Optional[str] = None) -> dict:
|
|
542
|
+
"""Retrieve the closest public Wayback snapshot without treating it as current."""
|
|
543
|
+
from .safety import allow_private_default, classify_url
|
|
544
|
+
|
|
545
|
+
attempts: list[dict] = []
|
|
546
|
+
safe, reason = classify_url(url, allow_private_default())
|
|
547
|
+
if not safe:
|
|
548
|
+
attempts.append(_attempt("archive", "wayback-available", False, 0, "", reason))
|
|
549
|
+
return {"platform": "archive", "ok": False, "route": None, "content": "",
|
|
550
|
+
"final_url": url, "attempts": attempts, "timestamp": ""}
|
|
551
|
+
try:
|
|
552
|
+
api_url = "https://archive.org/wayback/available?url=" + quote(url, safe="")
|
|
553
|
+
response = _archive_get(api_url, timeout=timeout, proxy=proxy)
|
|
554
|
+
data = response.json() if response.status_code == 200 else {}
|
|
555
|
+
closest = ((data.get("archived_snapshots") or {}).get("closest") or {})
|
|
556
|
+
snapshot = str(closest.get("url") or "")
|
|
557
|
+
timestamp = str(closest.get("timestamp") or "")
|
|
558
|
+
valid_snapshot = (
|
|
559
|
+
closest.get("available") is True
|
|
560
|
+
and str(closest.get("status") or "") == "200"
|
|
561
|
+
and urlsplit(snapshot).hostname == "web.archive.org"
|
|
562
|
+
)
|
|
563
|
+
attempts.append(_attempt(
|
|
564
|
+
"archive", "wayback-available", valid_snapshot, response.status_code,
|
|
565
|
+
response.text,
|
|
566
|
+
(("snapshot:" + getattr(response, "transport", "unknown")) if valid_snapshot
|
|
567
|
+
else (f"status={response.status_code}" if response.status_code != 200
|
|
568
|
+
else "no_snapshot"))))
|
|
569
|
+
if not valid_snapshot:
|
|
570
|
+
return {"platform": "archive", "ok": False, "route": None, "content": "",
|
|
571
|
+
"final_url": url, "attempts": attempts, "timestamp": ""}
|
|
572
|
+
page = _archive_get(snapshot, timeout=max(timeout, 30), proxy=proxy)
|
|
573
|
+
ok = page.status_code == 200 and len(page.text or "") >= 200
|
|
574
|
+
attempts.append(_attempt(
|
|
575
|
+
"archive", "wayback-snapshot", ok, page.status_code, page.text,
|
|
576
|
+
(("historical_snapshot:" + getattr(page, "transport", "unknown"))
|
|
577
|
+
if ok else "snapshot_fetch_failed")))
|
|
578
|
+
return {"platform": "archive", "ok": ok,
|
|
579
|
+
"route": "wayback" if ok else None,
|
|
580
|
+
"content": page.text if ok else "", "final_url": snapshot if ok else url,
|
|
581
|
+
"attempts": attempts, "timestamp": timestamp if ok else ""}
|
|
582
|
+
except Exception as exc:
|
|
583
|
+
attempts.append(_attempt(
|
|
584
|
+
"archive", "wayback-available", False, 0, "", f"{type(exc).__name__}"))
|
|
585
|
+
return {"platform": "archive", "ok": False, "route": None, "content": "",
|
|
586
|
+
"final_url": url, "attempts": attempts, "timestamp": ""}
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def route(url: str, *, timeout: int = 15, proxy: Optional[str] = None,
|
|
590
|
+
include_media_transcript: bool = False,
|
|
591
|
+
cookies: Optional[list[dict]] = None) -> Optional[dict]:
|
|
592
|
+
platform = _detect(url)
|
|
593
|
+
if platform is None:
|
|
594
|
+
return None
|
|
595
|
+
if platform == "youtube":
|
|
596
|
+
return _youtube(url, timeout, proxy, include_media_transcript, cookies)
|
|
597
|
+
return _ROUTERS[platform](url, timeout, proxy)
|