devez-vibe 1.10.21 → 1.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +73 -73
  3. package/bin/dvz.exe +0 -0
  4. package/bin/dvz.js +35 -35
  5. package/bridge/claude-agent-sdk-bridge.mjs +364 -35
  6. package/install-skills.mjs +11 -11
  7. package/package.json +50 -50
  8. package/prepare-search-deps.mjs +42 -42
  9. package/skills/insane-search/SKILL.md +36 -36
  10. package/skills/insane-search/engine/__main__.py +44 -44
  11. package/skills/insane-search/engine/executor.py +246 -246
  12. package/skills/insane-search/engine/fetch_chain.py +443 -443
  13. package/skills/insane-search/engine/network.py +108 -108
  14. package/skills/insane-search/engine/phase0.py +380 -380
  15. package/skills/insane-search/engine/session_input.py +120 -120
  16. package/skills/insane-search/engine/templates/invisible_playwright_fetch.py +57 -57
  17. package/skills/insane-search/engine/templates/nodriver_fetch.py +9 -9
  18. package/skills/insane-search/engine/templates/patchright_fetch.py +9 -9
  19. package/skills/insane-search/engine/templates/playwright_mobile_chrome.js +9 -9
  20. package/skills/insane-search/engine/templates/playwright_real_chrome.js +7 -7
  21. package/skills/insane-search/engine/templates/scrapling_fetch.py +46 -46
  22. package/skills/insane-search/engine/tests/test_smoke.py +8 -8
  23. package/skills/insane-search/engine/tests/test_t10_artifact_routes.py +126 -126
  24. package/skills/insane-search/engine/tests/test_t11_session_handoff.py +157 -157
  25. package/skills/insane-search/engine/tests/test_t5_pdfplumber.py +11 -11
  26. package/skills/insane-search/engine/tests/test_t7_browser_gate.py +3 -3
  27. package/skills/insane-search/engine/tests/test_t8_network_routes.py +207 -207
  28. package/skills/insane-search/engine/tests/test_t9_evidence_bundle.py +89 -89
  29. package/skills/insane-search/engine/tests/test_u5.py +9 -9
  30. package/skills/insane-search/engine/transport.py +94 -94
  31. package/skills/insane-search/engine/waf_profiles.yaml +38 -38
  32. package/skills/insane-search/references/cache-archive.md +13 -13
  33. package/skills/insane-search/references/media.md +14 -14
  34. package/skills/insane-search/references/playwright.md +27 -27
  35. package/skills/insane-search/references/research-evidence.md +36 -36
  36. package/skills/insane-search/references/session-handoff.md +26 -26
  37. package/skills/insane-search/tests/test_x_search.py +3 -3
  38. package/skills/luna-loop/agents/openai.yaml +7 -7
@@ -24,36 +24,36 @@ Each attempt dict: {"route","platform","ok","status","bytes","note"}.
24
24
  """
25
25
  from __future__ import annotations
26
26
 
27
- import importlib.util
28
- import html
29
- import json
30
- import os
31
- import re
32
- import shutil
33
- import subprocess
34
- import sys
35
- import tempfile
36
- from pathlib import Path
37
- from typing import Optional
38
- from urllib.parse import quote, urlsplit
27
+ import importlib.util
28
+ import html
29
+ import json
30
+ import os
31
+ import re
32
+ import shutil
33
+ import subprocess
34
+ import sys
35
+ import tempfile
36
+ from pathlib import Path
37
+ from typing import Optional
38
+ from urllib.parse import quote, urlsplit
39
39
 
40
40
 
41
41
  # --- low-level helpers -------------------------------------------------------
42
- def _cffi_get(url: str, *, impersonate: str = "safari", timeout: int = 15,
43
- proxy: Optional[str] = None):
44
- from curl_cffi import requests as r # lazy: engine works even if missing
45
- kwargs = dict(
46
- impersonate=impersonate, # type: ignore[arg-type]
47
- timeout=timeout,
42
+ def _cffi_get(url: str, *, impersonate: str = "safari", timeout: int = 15,
43
+ proxy: Optional[str] = None):
44
+ from curl_cffi import requests as r # lazy: engine works even if missing
45
+ kwargs = dict(
46
+ impersonate=impersonate, # type: ignore[arg-type]
47
+ timeout=timeout,
48
48
  headers={
49
49
  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
50
50
  "Accept-Language": "en-US,en;q=0.9,ko;q=0.8",
51
- },
52
- allow_redirects=True,
53
- )
54
- if proxy:
55
- kwargs["proxy"] = proxy
56
- return r.get(url, **kwargs)
51
+ },
52
+ allow_redirects=True,
53
+ )
54
+ if proxy:
55
+ kwargs["proxy"] = proxy
56
+ return r.get(url, **kwargs)
57
57
 
58
58
 
59
59
  def _host(url: str) -> str:
@@ -83,7 +83,7 @@ def _detect(url: str) -> Optional[str]:
83
83
 
84
84
 
85
85
  # --- reddit ------------------------------------------------------------------
86
- def _reddit(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
86
+ def _reddit(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
87
87
  attempts: list[dict] = []
88
88
  base = url.split("?", 1)[0].rstrip("/")
89
89
  # Build an .rss / .json target from the path (works for /r/<sub> and post URLs).
@@ -92,7 +92,7 @@ def _reddit(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
92
92
 
93
93
  # Route 1: RSS (the route that actually survives — Reddit gates the JSON API).
94
94
  try:
95
- x = _cffi_get(rss_url, timeout=timeout, proxy=proxy)
95
+ x = _cffi_get(rss_url, timeout=timeout, proxy=proxy)
96
96
  ok = x.status_code == 200 and ("<rss" in x.text or "<feed" in x.text)
97
97
  attempts.append(_attempt("reddit", "rss", ok, x.status_code, x.text,
98
98
  "feed" if ok else "no-feed-markers"))
@@ -104,7 +104,7 @@ def _reddit(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
104
104
 
105
105
  # Route 2: JSON via curl_cffi (often 403 now, but try — cheap).
106
106
  try:
107
- x = _cffi_get(json_url, timeout=timeout, proxy=proxy)
107
+ x = _cffi_get(json_url, timeout=timeout, proxy=proxy)
108
108
  ok = x.status_code == 200 and x.text.lstrip().startswith(("{", "["))
109
109
  attempts.append(_attempt("reddit", "json", ok, x.status_code, x.text,
110
110
  "json" if ok else f"status={x.status_code}"))
@@ -122,15 +122,15 @@ def _reddit(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
122
122
  _TWEET_ID_RE = re.compile(r"/status(?:es)?/(\d+)")
123
123
 
124
124
 
125
- def _x(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
125
+ def _x(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
126
126
  attempts: list[dict] = []
127
127
  m = _TWEET_ID_RE.search(url)
128
128
 
129
129
  if m: # single tweet → tweet-result + oembed (both no-auth, reliable)
130
130
  tid = m.group(1)
131
131
  try:
132
- x = _cffi_get(f"https://cdn.syndication.twimg.com/tweet-result?id={tid}&token=a",
133
- timeout=timeout, proxy=proxy)
132
+ x = _cffi_get(f"https://cdn.syndication.twimg.com/tweet-result?id={tid}&token=a",
133
+ timeout=timeout, proxy=proxy)
134
134
  d = x.json() if x.status_code == 200 else {}
135
135
  ok = bool(d.get("text"))
136
136
  attempts.append(_attempt("x", "tweet-result", ok, x.status_code, x.text,
@@ -142,7 +142,7 @@ def _x(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
142
142
  attempts.append(_attempt("x", "tweet-result", False, 0, "", f"{type(e).__name__}"))
143
143
  try:
144
144
  ourl = f"https://publish.twitter.com/oembed?url=https://twitter.com/i/status/{tid}&omit_script=1"
145
- x = _cffi_get(ourl, timeout=timeout, proxy=proxy)
145
+ x = _cffi_get(ourl, timeout=timeout, proxy=proxy)
146
146
  d = x.json() if x.status_code == 200 else {}
147
147
  ok = bool(d.get("html"))
148
148
  attempts.append(_attempt("x", "oembed", ok, x.status_code, x.text,
@@ -159,7 +159,7 @@ def _x(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
159
159
  surl = f"https://syndication.twitter.com/srv/timeline-profile/screen-name/{handle}"
160
160
  for attempt_no in range(2):
161
161
  try:
162
- x = _cffi_get(surl, timeout=timeout, proxy=proxy)
162
+ x = _cffi_get(surl, timeout=timeout, proxy=proxy)
163
163
  ok = x.status_code == 200 and "__NEXT_DATA__" in x.text
164
164
  attempts.append(_attempt("x", f"syndication-timeline#{attempt_no+1}", ok,
165
165
  x.status_code, x.text,
@@ -175,7 +175,7 @@ def _x(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
175
175
 
176
176
 
177
177
  # --- youtube -----------------------------------------------------------------
178
- def _ytdlp_argv() -> Optional[list[str]]:
178
+ def _ytdlp_argv() -> Optional[list[str]]:
179
179
  """Best yt-dlp invocation for this environment, or None if unavailable.
180
180
 
181
181
  Prefers the ``yt-dlp`` console script on PATH; falls back to
@@ -192,261 +192,261 @@ def _ytdlp_argv() -> Optional[list[str]]:
192
192
  return None
193
193
 
194
194
 
195
- def _clean_caption_line(line: str) -> str:
196
- line = re.sub(r"<[^>]+>", "", line)
197
- line = html.unescape(line).strip()
198
- return "" if line.startswith(("Kind:", "Language:")) else line
199
-
200
-
201
- def _vtt_segments(vtt: str) -> list[dict[str, str]]:
202
- """Parse bounded WebVTT cues so final citations can include timestamps."""
203
- source = (vtt or "")[:2_000_000].splitlines()
204
- segments: list[dict[str, str]] = []
205
- total = 0
206
- index = 0
207
- while index < len(source):
208
- timing = source[index].strip()
209
- if "-->" not in timing:
210
- index += 1
211
- continue
212
- start, end = [part.strip().split()[0] for part in timing.split("-->", 1)]
213
- index += 1
214
- cue: list[str] = []
215
- while index < len(source) and source[index].strip():
216
- line = _clean_caption_line(source[index])
217
- if line and (not cue or cue[-1] != line):
218
- cue.append(line)
219
- index += 1
220
- text = " ".join(cue).strip()
221
- if text and segments and text.startswith(segments[-1]["text"]):
222
- text = text[len(segments[-1]["text"]):].strip()
223
- if text and (not segments or segments[-1]["text"] != text):
224
- take = text[:max(0, 500_000 - total)]
225
- if take:
226
- segments.append({"start": start, "end": end, "text": take})
227
- total += len(take)
228
- if total >= 500_000 or len(segments) >= 5000:
229
- break
230
- index += 1
231
- return segments
232
-
233
-
234
- def _vtt_to_text(vtt: str) -> str:
235
- lines: list[str] = []
236
- for segment in _vtt_segments(vtt):
237
- text = segment["text"]
238
- if not lines or lines[-1] != text:
239
- lines.append(text)
240
- return "\n".join(lines)[:500_000]
241
-
242
-
243
- def _choose_subtitle_language(metadata: dict, preferred: tuple[str, ...] = ("ko", "en")) -> tuple[str, str] | None:
244
- """Choose a creator subtitle first, then an automatic caption."""
245
- for key, source in (("subtitles", "creator"), ("automatic_captions", "automatic")):
246
- available = metadata.get(key) or {}
247
- for wanted in preferred:
248
- if wanted in available:
249
- return wanted, source
250
- match = next((lang for lang in available if lang.lower().startswith(wanted.lower() + "-")), None)
251
- if match:
252
- return match, source
253
- return None
254
-
255
-
256
- _YTDLP_ARGV_CACHE: Optional[list[str]] = None
257
-
258
-
259
- def _ensure_ytdlp_argv(allow_install: bool) -> Optional[list[str]]:
260
- """Resolve yt-dlp, optionally installing it into an isolated first-use venv."""
261
- global _YTDLP_ARGV_CACHE
262
- available = _ytdlp_argv()
263
- if available is not None:
264
- return available
265
- if not allow_install:
266
- return None
267
- if os.environ.get("INSANE_AUTO_INSTALL", "").strip().lower() in ("0", "false", "no"):
268
- return None
269
- if _YTDLP_ARGV_CACHE is not None:
270
- return _YTDLP_ARGV_CACHE or None
271
- root = os.path.expanduser("~/.insane-search/media-venv")
272
- python = os.path.join(
273
- root, "Scripts", "python.exe") if os.name == "nt" else os.path.join(root, "bin", "python")
274
- try:
275
- if not os.path.isfile(python):
276
- subprocess.run(
277
- [sys.executable, "-m", "venv", root], capture_output=True,
278
- text=True, timeout=180, check=False)
279
- probe = subprocess.run(
280
- [python, "-m", "yt_dlp", "--version"], capture_output=True,
281
- text=True, timeout=30, check=False)
282
- if probe.returncode != 0:
283
- installed = subprocess.run(
284
- [python, "-m", "pip", "install", "yt-dlp", "-q"],
285
- capture_output=True, text=True, timeout=300, check=False)
286
- if installed.returncode != 0:
287
- _YTDLP_ARGV_CACHE = []
288
- return None
289
- _YTDLP_ARGV_CACHE = [python, "-m", "yt_dlp"]
290
- return list(_YTDLP_ARGV_CACHE)
291
- except Exception:
292
- _YTDLP_ARGV_CACHE = []
293
- return None
294
-
295
-
296
- def _attach_transcript(metadata: dict, url: str, argv: list[str], timeout: int,
297
- proxy: Optional[str], cookie_path: Optional[str] = None) -> dict:
298
- choice = _choose_subtitle_language(metadata)
299
- if choice is None:
300
- metadata["research_transcript"] = {"available": False, "reason": "no_subtitles"}
301
- return metadata
302
- language, source = choice
303
- try:
304
- with tempfile.TemporaryDirectory(prefix="insane-media-") as root:
305
- command = argv + [
306
- "--write-subs", "--write-auto-subs", "--sub-langs", language,
307
- "--sub-format", "vtt", "--skip-download", "--no-playlist",
308
- "--paths", root, "-o", "%(id)s.%(ext)s",
309
- ]
310
- if proxy:
311
- command += ["--proxy", proxy]
312
- if cookie_path:
313
- command += ["--cookies", cookie_path]
314
- completed = subprocess.run(
315
- command + [url], capture_output=True, text=True,
316
- timeout=max(timeout, 90), check=False)
317
- files = sorted(Path(root).glob("*.vtt"))
318
- text = ""
319
- segments: list[dict[str, str]] = []
320
- if completed.returncode == 0 and files:
321
- raw_vtt = files[0].read_text(encoding="utf-8", errors="replace")
322
- segments = _vtt_segments(raw_vtt)
323
- text = _vtt_to_text(raw_vtt)
324
- metadata["research_transcript"] = {
325
- "available": bool(text),
326
- "language": language,
327
- "source": source,
328
- "text": text,
329
- "segments": segments,
330
- "reason": "" if text else "subtitle_download_failed",
331
- }
332
- except subprocess.TimeoutExpired:
333
- metadata["research_transcript"] = {"available": False, "reason": "subtitle_timeout"}
334
- except Exception as exc:
335
- metadata["research_transcript"] = {
336
- "available": False, "reason": f"subtitle_error:{type(exc).__name__}"}
337
- return metadata
338
-
339
-
340
- def _compact_media_metadata(metadata: dict) -> dict:
341
- """Drop signed streams and extractor internals that waste research tokens."""
342
- keys = (
343
- "id", "title", "description", "uploader", "uploader_id", "channel",
344
- "channel_id", "duration", "duration_string", "upload_date", "timestamp",
345
- "release_date", "webpage_url", "original_url", "availability", "live_status",
346
- "view_count", "like_count", "comment_count", "categories", "tags", "chapters",
347
- )
348
- compact = {key: metadata[key] for key in keys if metadata.get(key) is not None}
349
- compact["subtitle_languages"] = sorted((metadata.get("subtitles") or {}).keys())[:100]
350
- compact["automatic_caption_languages"] = sorted(
351
- (metadata.get("automatic_captions") or {}).keys())[:100]
352
- if "research_transcript" in metadata:
353
- compact["research_transcript"] = metadata["research_transcript"]
354
- return compact
355
-
356
-
357
- def _write_cookie_jar(cookies: list[dict]) -> str:
358
- handle = tempfile.NamedTemporaryFile(
359
- "w", encoding="utf-8", prefix="insane-media-cookies-", suffix=".txt",
360
- delete=False)
361
- try:
362
- handle.write("# Netscape HTTP Cookie File\n")
363
- for cookie in cookies:
364
- domain = str(cookie.get("domain") or "")
365
- include_subdomains = "TRUE" if domain.startswith(".") else "FALSE"
366
- fields = (
367
- domain, include_subdomains, str(cookie.get("path") or "/"),
368
- "TRUE" if cookie.get("secure") else "FALSE",
369
- str(int(cookie.get("expires") or 0)), str(cookie.get("name") or ""),
370
- str(cookie.get("value") or ""),
371
- )
372
- handle.write("\t".join(fields) + "\n")
373
- return handle.name
374
- finally:
375
- handle.close()
376
-
377
-
378
- def _media(url: str, timeout: int, proxy: Optional[str] = None,
379
- include_transcript: bool = False, platform: str = "media",
380
- cookies: Optional[list[dict]] = None,
381
- auto_install: bool = False) -> dict:
382
- attempts: list[dict] = []
383
- argv = _ensure_ytdlp_argv(auto_install)
384
- if argv is None:
385
- attempts.append(_attempt(platform, "yt-dlp", False, 0, "", "yt-dlp not installed"))
386
- return {"platform": platform, "ok": False, "route": None, "content": "",
387
- "final_url": url, "attempts": attempts}
388
- cookie_path = _write_cookie_jar(cookies) if cookies else None
389
- try:
390
- command = argv + ["--dump-json", "--skip-download"]
391
- if proxy:
392
- command += ["--proxy", proxy]
393
- if cookie_path:
394
- command += ["--cookies", cookie_path]
395
- p = subprocess.run(
396
- command + [url],
397
- capture_output=True, text=True, timeout=max(timeout, 60),
398
- )
399
- ok = p.returncode == 0 and p.stdout.strip().startswith("{")
400
- diagnostic = p.stderr or ""
401
- if cookie_path:
402
- diagnostic = diagnostic.replace(cookie_path, "<cookie-file>")
403
- if cookies:
404
- from .session_input import redact_cookie_values
405
- diagnostic = redact_cookie_values(diagnostic, cookies) or ""
406
- note = "json" if ok else diagnostic.strip()[:80]
407
- attempts.append(_attempt(platform, "yt-dlp", ok, 200 if ok else 0, p.stdout, note))
408
- if ok:
409
- metadata = json.loads(p.stdout)
410
- if include_transcript:
411
- metadata = _attach_transcript(
412
- metadata, url, argv, timeout, proxy, cookie_path)
413
- if include_transcript or platform == "media":
414
- metadata = _compact_media_metadata(metadata)
415
- transcript_available = bool(
416
- (metadata.get("research_transcript") or {}).get("available"))
417
- return {"platform": platform, "ok": True, "route": "yt-dlp",
418
- "content": json.dumps(metadata, ensure_ascii=False),
419
- "final_url": url, "attempts": attempts,
420
- "content_kind": ("media+transcript" if transcript_available
421
- else "media_metadata"),
422
- "transcript": transcript_available}
423
- except FileNotFoundError:
424
- attempts.append(_attempt(platform, "yt-dlp", False, 0, "", "yt-dlp not installed"))
425
- except Exception as e:
426
- attempts.append(_attempt(platform, "yt-dlp", False, 0, "", f"{type(e).__name__}"))
427
- finally:
428
- if cookie_path:
429
- try:
430
- Path(cookie_path).unlink()
431
- except OSError:
432
- pass
433
- return {"platform": platform, "ok": False, "route": None, "content": "",
434
- "final_url": url, "attempts": attempts}
435
-
436
-
437
- def _youtube(url: str, timeout: int, proxy: Optional[str] = None,
438
- include_transcript: bool = False,
439
- cookies: Optional[list[dict]] = None) -> dict:
440
- return _media(
441
- url, timeout, proxy, include_transcript, platform="youtube", cookies=cookies,
442
- auto_install=include_transcript)
195
+ def _clean_caption_line(line: str) -> str:
196
+ line = re.sub(r"<[^>]+>", "", line)
197
+ line = html.unescape(line).strip()
198
+ return "" if line.startswith(("Kind:", "Language:")) else line
199
+
200
+
201
+ def _vtt_segments(vtt: str) -> list[dict[str, str]]:
202
+ """Parse bounded WebVTT cues so final citations can include timestamps."""
203
+ source = (vtt or "")[:2_000_000].splitlines()
204
+ segments: list[dict[str, str]] = []
205
+ total = 0
206
+ index = 0
207
+ while index < len(source):
208
+ timing = source[index].strip()
209
+ if "-->" not in timing:
210
+ index += 1
211
+ continue
212
+ start, end = [part.strip().split()[0] for part in timing.split("-->", 1)]
213
+ index += 1
214
+ cue: list[str] = []
215
+ while index < len(source) and source[index].strip():
216
+ line = _clean_caption_line(source[index])
217
+ if line and (not cue or cue[-1] != line):
218
+ cue.append(line)
219
+ index += 1
220
+ text = " ".join(cue).strip()
221
+ if text and segments and text.startswith(segments[-1]["text"]):
222
+ text = text[len(segments[-1]["text"]):].strip()
223
+ if text and (not segments or segments[-1]["text"] != text):
224
+ take = text[:max(0, 500_000 - total)]
225
+ if take:
226
+ segments.append({"start": start, "end": end, "text": take})
227
+ total += len(take)
228
+ if total >= 500_000 or len(segments) >= 5000:
229
+ break
230
+ index += 1
231
+ return segments
232
+
233
+
234
+ def _vtt_to_text(vtt: str) -> str:
235
+ lines: list[str] = []
236
+ for segment in _vtt_segments(vtt):
237
+ text = segment["text"]
238
+ if not lines or lines[-1] != text:
239
+ lines.append(text)
240
+ return "\n".join(lines)[:500_000]
241
+
242
+
243
+ def _choose_subtitle_language(metadata: dict, preferred: tuple[str, ...] = ("ko", "en")) -> tuple[str, str] | None:
244
+ """Choose a creator subtitle first, then an automatic caption."""
245
+ for key, source in (("subtitles", "creator"), ("automatic_captions", "automatic")):
246
+ available = metadata.get(key) or {}
247
+ for wanted in preferred:
248
+ if wanted in available:
249
+ return wanted, source
250
+ match = next((lang for lang in available if lang.lower().startswith(wanted.lower() + "-")), None)
251
+ if match:
252
+ return match, source
253
+ return None
254
+
255
+
256
+ _YTDLP_ARGV_CACHE: Optional[list[str]] = None
257
+
258
+
259
+ def _ensure_ytdlp_argv(allow_install: bool) -> Optional[list[str]]:
260
+ """Resolve yt-dlp, optionally installing it into an isolated first-use venv."""
261
+ global _YTDLP_ARGV_CACHE
262
+ available = _ytdlp_argv()
263
+ if available is not None:
264
+ return available
265
+ if not allow_install:
266
+ return None
267
+ if os.environ.get("INSANE_AUTO_INSTALL", "").strip().lower() in ("0", "false", "no"):
268
+ return None
269
+ if _YTDLP_ARGV_CACHE is not None:
270
+ return _YTDLP_ARGV_CACHE or None
271
+ root = os.path.expanduser("~/.insane-search/media-venv")
272
+ python = os.path.join(
273
+ root, "Scripts", "python.exe") if os.name == "nt" else os.path.join(root, "bin", "python")
274
+ try:
275
+ if not os.path.isfile(python):
276
+ subprocess.run(
277
+ [sys.executable, "-m", "venv", root], capture_output=True,
278
+ text=True, timeout=180, check=False)
279
+ probe = subprocess.run(
280
+ [python, "-m", "yt_dlp", "--version"], capture_output=True,
281
+ text=True, timeout=30, check=False)
282
+ if probe.returncode != 0:
283
+ installed = subprocess.run(
284
+ [python, "-m", "pip", "install", "yt-dlp", "-q"],
285
+ capture_output=True, text=True, timeout=300, check=False)
286
+ if installed.returncode != 0:
287
+ _YTDLP_ARGV_CACHE = []
288
+ return None
289
+ _YTDLP_ARGV_CACHE = [python, "-m", "yt_dlp"]
290
+ return list(_YTDLP_ARGV_CACHE)
291
+ except Exception:
292
+ _YTDLP_ARGV_CACHE = []
293
+ return None
294
+
295
+
296
+ def _attach_transcript(metadata: dict, url: str, argv: list[str], timeout: int,
297
+ proxy: Optional[str], cookie_path: Optional[str] = None) -> dict:
298
+ choice = _choose_subtitle_language(metadata)
299
+ if choice is None:
300
+ metadata["research_transcript"] = {"available": False, "reason": "no_subtitles"}
301
+ return metadata
302
+ language, source = choice
303
+ try:
304
+ with tempfile.TemporaryDirectory(prefix="insane-media-") as root:
305
+ command = argv + [
306
+ "--write-subs", "--write-auto-subs", "--sub-langs", language,
307
+ "--sub-format", "vtt", "--skip-download", "--no-playlist",
308
+ "--paths", root, "-o", "%(id)s.%(ext)s",
309
+ ]
310
+ if proxy:
311
+ command += ["--proxy", proxy]
312
+ if cookie_path:
313
+ command += ["--cookies", cookie_path]
314
+ completed = subprocess.run(
315
+ command + [url], capture_output=True, text=True,
316
+ timeout=max(timeout, 90), check=False)
317
+ files = sorted(Path(root).glob("*.vtt"))
318
+ text = ""
319
+ segments: list[dict[str, str]] = []
320
+ if completed.returncode == 0 and files:
321
+ raw_vtt = files[0].read_text(encoding="utf-8", errors="replace")
322
+ segments = _vtt_segments(raw_vtt)
323
+ text = _vtt_to_text(raw_vtt)
324
+ metadata["research_transcript"] = {
325
+ "available": bool(text),
326
+ "language": language,
327
+ "source": source,
328
+ "text": text,
329
+ "segments": segments,
330
+ "reason": "" if text else "subtitle_download_failed",
331
+ }
332
+ except subprocess.TimeoutExpired:
333
+ metadata["research_transcript"] = {"available": False, "reason": "subtitle_timeout"}
334
+ except Exception as exc:
335
+ metadata["research_transcript"] = {
336
+ "available": False, "reason": f"subtitle_error:{type(exc).__name__}"}
337
+ return metadata
338
+
339
+
340
+ def _compact_media_metadata(metadata: dict) -> dict:
341
+ """Drop signed streams and extractor internals that waste research tokens."""
342
+ keys = (
343
+ "id", "title", "description", "uploader", "uploader_id", "channel",
344
+ "channel_id", "duration", "duration_string", "upload_date", "timestamp",
345
+ "release_date", "webpage_url", "original_url", "availability", "live_status",
346
+ "view_count", "like_count", "comment_count", "categories", "tags", "chapters",
347
+ )
348
+ compact = {key: metadata[key] for key in keys if metadata.get(key) is not None}
349
+ compact["subtitle_languages"] = sorted((metadata.get("subtitles") or {}).keys())[:100]
350
+ compact["automatic_caption_languages"] = sorted(
351
+ (metadata.get("automatic_captions") or {}).keys())[:100]
352
+ if "research_transcript" in metadata:
353
+ compact["research_transcript"] = metadata["research_transcript"]
354
+ return compact
355
+
356
+
357
+ def _write_cookie_jar(cookies: list[dict]) -> str:
358
+ handle = tempfile.NamedTemporaryFile(
359
+ "w", encoding="utf-8", prefix="insane-media-cookies-", suffix=".txt",
360
+ delete=False)
361
+ try:
362
+ handle.write("# Netscape HTTP Cookie File\n")
363
+ for cookie in cookies:
364
+ domain = str(cookie.get("domain") or "")
365
+ include_subdomains = "TRUE" if domain.startswith(".") else "FALSE"
366
+ fields = (
367
+ domain, include_subdomains, str(cookie.get("path") or "/"),
368
+ "TRUE" if cookie.get("secure") else "FALSE",
369
+ str(int(cookie.get("expires") or 0)), str(cookie.get("name") or ""),
370
+ str(cookie.get("value") or ""),
371
+ )
372
+ handle.write("\t".join(fields) + "\n")
373
+ return handle.name
374
+ finally:
375
+ handle.close()
376
+
377
+
378
+ def _media(url: str, timeout: int, proxy: Optional[str] = None,
379
+ include_transcript: bool = False, platform: str = "media",
380
+ cookies: Optional[list[dict]] = None,
381
+ auto_install: bool = False) -> dict:
382
+ attempts: list[dict] = []
383
+ argv = _ensure_ytdlp_argv(auto_install)
384
+ if argv is None:
385
+ attempts.append(_attempt(platform, "yt-dlp", False, 0, "", "yt-dlp not installed"))
386
+ return {"platform": platform, "ok": False, "route": None, "content": "",
387
+ "final_url": url, "attempts": attempts}
388
+ cookie_path = _write_cookie_jar(cookies) if cookies else None
389
+ try:
390
+ command = argv + ["--dump-json", "--skip-download"]
391
+ if proxy:
392
+ command += ["--proxy", proxy]
393
+ if cookie_path:
394
+ command += ["--cookies", cookie_path]
395
+ p = subprocess.run(
396
+ command + [url],
397
+ capture_output=True, text=True, timeout=max(timeout, 60),
398
+ )
399
+ ok = p.returncode == 0 and p.stdout.strip().startswith("{")
400
+ diagnostic = p.stderr or ""
401
+ if cookie_path:
402
+ diagnostic = diagnostic.replace(cookie_path, "<cookie-file>")
403
+ if cookies:
404
+ from .session_input import redact_cookie_values
405
+ diagnostic = redact_cookie_values(diagnostic, cookies) or ""
406
+ note = "json" if ok else diagnostic.strip()[:80]
407
+ attempts.append(_attempt(platform, "yt-dlp", ok, 200 if ok else 0, p.stdout, note))
408
+ if ok:
409
+ metadata = json.loads(p.stdout)
410
+ if include_transcript:
411
+ metadata = _attach_transcript(
412
+ metadata, url, argv, timeout, proxy, cookie_path)
413
+ if include_transcript or platform == "media":
414
+ metadata = _compact_media_metadata(metadata)
415
+ transcript_available = bool(
416
+ (metadata.get("research_transcript") or {}).get("available"))
417
+ return {"platform": platform, "ok": True, "route": "yt-dlp",
418
+ "content": json.dumps(metadata, ensure_ascii=False),
419
+ "final_url": url, "attempts": attempts,
420
+ "content_kind": ("media+transcript" if transcript_available
421
+ else "media_metadata"),
422
+ "transcript": transcript_available}
423
+ except FileNotFoundError:
424
+ attempts.append(_attempt(platform, "yt-dlp", False, 0, "", "yt-dlp not installed"))
425
+ except Exception as e:
426
+ attempts.append(_attempt(platform, "yt-dlp", False, 0, "", f"{type(e).__name__}"))
427
+ finally:
428
+ if cookie_path:
429
+ try:
430
+ Path(cookie_path).unlink()
431
+ except OSError:
432
+ pass
433
+ return {"platform": platform, "ok": False, "route": None, "content": "",
434
+ "final_url": url, "attempts": attempts}
435
+
436
+
437
+ def _youtube(url: str, timeout: int, proxy: Optional[str] = None,
438
+ include_transcript: bool = False,
439
+ cookies: Optional[list[dict]] = None) -> dict:
440
+ return _media(
441
+ url, timeout, proxy, include_transcript, platform="youtube", cookies=cookies,
442
+ auto_install=include_transcript)
443
443
 
444
444
 
445
445
  # --- threads -----------------------------------------------------------------
446
446
  _THREADS_POST_RE = re.compile(r"/post/([A-Za-z0-9_-]+)")
447
447
 
448
448
 
449
- def _threads(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
449
+ def _threads(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
450
450
  """Threads video post → signed CDN URLs from the page's inline JSON.
451
451
 
452
452
  yt-dlp has no Threads extractor, but an anonymous GET with a curl_cffi
@@ -463,7 +463,7 @@ def _threads(url: str, timeout: int, proxy: Optional[str] = None) -> dict:
463
463
  "final_url": url, "attempts": attempts}
464
464
  code = m.group(1)
465
465
  try:
466
- x = _cffi_get(url, timeout=timeout, proxy=proxy)
466
+ x = _cffi_get(url, timeout=timeout, proxy=proxy)
467
467
  raw = x.text if x.status_code == 200 else ""
468
468
  code_pos = [c.start() for c in re.finditer(r'"code"\s*:\s*"%s"' % re.escape(code), raw)]
469
469
  blocks = list(re.finditer(r'"video_versions"\s*:\s*\[(.*?)\]', raw))
@@ -498,100 +498,100 @@ _ROUTERS = {"reddit": _reddit, "x": _x, "youtube": _youtube, "threads": _threads
498
498
 
499
499
 
500
500
  # --- public entrypoint -------------------------------------------------------
501
- def route_media(url: str, *, timeout: int = 15, proxy: Optional[str] = None,
502
- include_transcript: bool = False,
503
- cookies: Optional[list[dict]] = None) -> dict:
504
- return _media(
505
- url, timeout, proxy, include_transcript, platform="media", cookies=cookies,
506
- auto_install=True)
507
-
508
-
509
- class _ArchiveResponse:
510
- def __init__(self, status: int, text: str, url: str, transport: str):
511
- self.status_code = status
512
- self.text = text
513
- self.url = url
514
- self.transport = transport
515
-
516
- def json(self):
517
- return json.loads(self.text)
518
-
519
-
520
- def _archive_get(url: str, timeout: int, proxy: Optional[str]) -> _ArchiveResponse:
521
- try:
522
- response = _cffi_get(url, timeout=timeout, proxy=proxy)
523
- response.transport = "curl_cffi"
524
- return response
525
- except Exception:
526
- from .executor import run_playwright_fallback
527
- attempt, body = run_playwright_fallback(
528
- url, profile_id="unknown_challenge", force_executor="scrapling",
529
- timeout=max(timeout, 60), proxy=proxy)
530
- inner = str(getattr(attempt, "_inner_text", "") or "").strip()
531
- if inner:
532
- text = inner
533
- else:
534
- text = html.unescape(re.sub(r"(?s)<[^>]+>", " ", body or ""))
535
- text = re.sub(r"\s+", " ", text).strip()
536
- return _ArchiveResponse(
537
- int(getattr(attempt, "status", 0) or 0), text if text else body,
538
- str(getattr(attempt, "url", "") or url), "system_browser")
539
-
540
-
541
- def route_archive(url: str, *, timeout: int = 15, proxy: Optional[str] = None) -> dict:
542
- """Retrieve the closest public Wayback snapshot without treating it as current."""
543
- from .safety import allow_private_default, classify_url
544
-
545
- attempts: list[dict] = []
546
- safe, reason = classify_url(url, allow_private_default())
547
- if not safe:
548
- attempts.append(_attempt("archive", "wayback-available", False, 0, "", reason))
549
- return {"platform": "archive", "ok": False, "route": None, "content": "",
550
- "final_url": url, "attempts": attempts, "timestamp": ""}
551
- try:
552
- api_url = "https://archive.org/wayback/available?url=" + quote(url, safe="")
553
- response = _archive_get(api_url, timeout=timeout, proxy=proxy)
554
- data = response.json() if response.status_code == 200 else {}
555
- closest = ((data.get("archived_snapshots") or {}).get("closest") or {})
556
- snapshot = str(closest.get("url") or "")
557
- timestamp = str(closest.get("timestamp") or "")
558
- valid_snapshot = (
559
- closest.get("available") is True
560
- and str(closest.get("status") or "") == "200"
561
- and urlsplit(snapshot).hostname == "web.archive.org"
562
- )
563
- attempts.append(_attempt(
564
- "archive", "wayback-available", valid_snapshot, response.status_code,
565
- response.text,
566
- (("snapshot:" + getattr(response, "transport", "unknown")) if valid_snapshot
567
- else (f"status={response.status_code}" if response.status_code != 200
568
- else "no_snapshot"))))
569
- if not valid_snapshot:
570
- return {"platform": "archive", "ok": False, "route": None, "content": "",
571
- "final_url": url, "attempts": attempts, "timestamp": ""}
572
- page = _archive_get(snapshot, timeout=max(timeout, 30), proxy=proxy)
573
- ok = page.status_code == 200 and len(page.text or "") >= 200
574
- attempts.append(_attempt(
575
- "archive", "wayback-snapshot", ok, page.status_code, page.text,
576
- (("historical_snapshot:" + getattr(page, "transport", "unknown"))
577
- if ok else "snapshot_fetch_failed")))
578
- return {"platform": "archive", "ok": ok,
579
- "route": "wayback" if ok else None,
580
- "content": page.text if ok else "", "final_url": snapshot if ok else url,
581
- "attempts": attempts, "timestamp": timestamp if ok else ""}
582
- except Exception as exc:
583
- attempts.append(_attempt(
584
- "archive", "wayback-available", False, 0, "", f"{type(exc).__name__}"))
585
- return {"platform": "archive", "ok": False, "route": None, "content": "",
586
- "final_url": url, "attempts": attempts, "timestamp": ""}
587
-
588
-
589
- def route(url: str, *, timeout: int = 15, proxy: Optional[str] = None,
590
- include_media_transcript: bool = False,
591
- cookies: Optional[list[dict]] = None) -> Optional[dict]:
592
- platform = _detect(url)
593
- if platform is None:
594
- return None
595
- if platform == "youtube":
596
- return _youtube(url, timeout, proxy, include_media_transcript, cookies)
597
- return _ROUTERS[platform](url, timeout, proxy)
501
+ def route_media(url: str, *, timeout: int = 15, proxy: Optional[str] = None,
502
+ include_transcript: bool = False,
503
+ cookies: Optional[list[dict]] = None) -> dict:
504
+ return _media(
505
+ url, timeout, proxy, include_transcript, platform="media", cookies=cookies,
506
+ auto_install=True)
507
+
508
+
509
+ class _ArchiveResponse:
510
+ def __init__(self, status: int, text: str, url: str, transport: str):
511
+ self.status_code = status
512
+ self.text = text
513
+ self.url = url
514
+ self.transport = transport
515
+
516
+ def json(self):
517
+ return json.loads(self.text)
518
+
519
+
520
+ def _archive_get(url: str, timeout: int, proxy: Optional[str]) -> _ArchiveResponse:
521
+ try:
522
+ response = _cffi_get(url, timeout=timeout, proxy=proxy)
523
+ response.transport = "curl_cffi"
524
+ return response
525
+ except Exception:
526
+ from .executor import run_playwright_fallback
527
+ attempt, body = run_playwright_fallback(
528
+ url, profile_id="unknown_challenge", force_executor="scrapling",
529
+ timeout=max(timeout, 60), proxy=proxy)
530
+ inner = str(getattr(attempt, "_inner_text", "") or "").strip()
531
+ if inner:
532
+ text = inner
533
+ else:
534
+ text = html.unescape(re.sub(r"(?s)<[^>]+>", " ", body or ""))
535
+ text = re.sub(r"\s+", " ", text).strip()
536
+ return _ArchiveResponse(
537
+ int(getattr(attempt, "status", 0) or 0), text if text else body,
538
+ str(getattr(attempt, "url", "") or url), "system_browser")
539
+
540
+
541
+ def route_archive(url: str, *, timeout: int = 15, proxy: Optional[str] = None) -> dict:
542
+ """Retrieve the closest public Wayback snapshot without treating it as current."""
543
+ from .safety import allow_private_default, classify_url
544
+
545
+ attempts: list[dict] = []
546
+ safe, reason = classify_url(url, allow_private_default())
547
+ if not safe:
548
+ attempts.append(_attempt("archive", "wayback-available", False, 0, "", reason))
549
+ return {"platform": "archive", "ok": False, "route": None, "content": "",
550
+ "final_url": url, "attempts": attempts, "timestamp": ""}
551
+ try:
552
+ api_url = "https://archive.org/wayback/available?url=" + quote(url, safe="")
553
+ response = _archive_get(api_url, timeout=timeout, proxy=proxy)
554
+ data = response.json() if response.status_code == 200 else {}
555
+ closest = ((data.get("archived_snapshots") or {}).get("closest") or {})
556
+ snapshot = str(closest.get("url") or "")
557
+ timestamp = str(closest.get("timestamp") or "")
558
+ valid_snapshot = (
559
+ closest.get("available") is True
560
+ and str(closest.get("status") or "") == "200"
561
+ and urlsplit(snapshot).hostname == "web.archive.org"
562
+ )
563
+ attempts.append(_attempt(
564
+ "archive", "wayback-available", valid_snapshot, response.status_code,
565
+ response.text,
566
+ (("snapshot:" + getattr(response, "transport", "unknown")) if valid_snapshot
567
+ else (f"status={response.status_code}" if response.status_code != 200
568
+ else "no_snapshot"))))
569
+ if not valid_snapshot:
570
+ return {"platform": "archive", "ok": False, "route": None, "content": "",
571
+ "final_url": url, "attempts": attempts, "timestamp": ""}
572
+ page = _archive_get(snapshot, timeout=max(timeout, 30), proxy=proxy)
573
+ ok = page.status_code == 200 and len(page.text or "") >= 200
574
+ attempts.append(_attempt(
575
+ "archive", "wayback-snapshot", ok, page.status_code, page.text,
576
+ (("historical_snapshot:" + getattr(page, "transport", "unknown"))
577
+ if ok else "snapshot_fetch_failed")))
578
+ return {"platform": "archive", "ok": ok,
579
+ "route": "wayback" if ok else None,
580
+ "content": page.text if ok else "", "final_url": snapshot if ok else url,
581
+ "attempts": attempts, "timestamp": timestamp if ok else ""}
582
+ except Exception as exc:
583
+ attempts.append(_attempt(
584
+ "archive", "wayback-available", False, 0, "", f"{type(exc).__name__}"))
585
+ return {"platform": "archive", "ok": False, "route": None, "content": "",
586
+ "final_url": url, "attempts": attempts, "timestamp": ""}
587
+
588
+
589
+ def route(url: str, *, timeout: int = 15, proxy: Optional[str] = None,
590
+ include_media_transcript: bool = False,
591
+ cookies: Optional[list[dict]] = None) -> Optional[dict]:
592
+ platform = _detect(url)
593
+ if platform is None:
594
+ return None
595
+ if platform == "youtube":
596
+ return _youtube(url, timeout, proxy, include_media_transcript, cookies)
597
+ return _ROUTERS[platform](url, timeout, proxy)