devez-vibe 1.10.21 → 1.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +73 -73
  3. package/bin/dvz.exe +0 -0
  4. package/bin/dvz.js +35 -35
  5. package/bridge/claude-agent-sdk-bridge.mjs +364 -35
  6. package/install-skills.mjs +11 -11
  7. package/package.json +50 -50
  8. package/prepare-search-deps.mjs +42 -42
  9. package/skills/insane-search/SKILL.md +36 -36
  10. package/skills/insane-search/engine/__main__.py +44 -44
  11. package/skills/insane-search/engine/executor.py +246 -246
  12. package/skills/insane-search/engine/fetch_chain.py +443 -443
  13. package/skills/insane-search/engine/network.py +108 -108
  14. package/skills/insane-search/engine/phase0.py +380 -380
  15. package/skills/insane-search/engine/session_input.py +120 -120
  16. package/skills/insane-search/engine/templates/invisible_playwright_fetch.py +57 -57
  17. package/skills/insane-search/engine/templates/nodriver_fetch.py +9 -9
  18. package/skills/insane-search/engine/templates/patchright_fetch.py +9 -9
  19. package/skills/insane-search/engine/templates/playwright_mobile_chrome.js +9 -9
  20. package/skills/insane-search/engine/templates/playwright_real_chrome.js +7 -7
  21. package/skills/insane-search/engine/templates/scrapling_fetch.py +46 -46
  22. package/skills/insane-search/engine/tests/test_smoke.py +8 -8
  23. package/skills/insane-search/engine/tests/test_t10_artifact_routes.py +126 -126
  24. package/skills/insane-search/engine/tests/test_t11_session_handoff.py +157 -157
  25. package/skills/insane-search/engine/tests/test_t5_pdfplumber.py +11 -11
  26. package/skills/insane-search/engine/tests/test_t7_browser_gate.py +3 -3
  27. package/skills/insane-search/engine/tests/test_t8_network_routes.py +207 -207
  28. package/skills/insane-search/engine/tests/test_t9_evidence_bundle.py +89 -89
  29. package/skills/insane-search/engine/tests/test_u5.py +9 -9
  30. package/skills/insane-search/engine/transport.py +94 -94
  31. package/skills/insane-search/engine/waf_profiles.yaml +38 -38
  32. package/skills/insane-search/references/cache-archive.md +13 -13
  33. package/skills/insane-search/references/media.md +14 -14
  34. package/skills/insane-search/references/playwright.md +27 -27
  35. package/skills/insane-search/references/research-evidence.md +36 -36
  36. package/skills/insane-search/references/session-handoff.md +26 -26
  37. package/skills/insane-search/tests/test_x_search.py +3 -3
  38. package/skills/luna-loop/agents/openai.yaml +7 -7
@@ -31,16 +31,16 @@ No site-specific branching. Site knowledge enters only via:
31
31
  """
32
32
  from __future__ import annotations
33
33
 
34
- import os
35
- import random
36
- import time
37
- import hashlib
38
- from dataclasses import dataclass, field, asdict
39
- from datetime import datetime, timezone
40
- from typing import Any, Optional
41
-
42
- from .content_safety import ContentSafetyReport, analyze_untrusted_content, wrap_untrusted_content
43
- from .network import diagnose_trace, normalize_proxy, redact_proxy
34
+ import os
35
+ import random
36
+ import time
37
+ import hashlib
38
+ from dataclasses import dataclass, field, asdict
39
+ from datetime import datetime, timezone
40
+ from typing import Any, Optional
41
+
42
+ from .content_safety import ContentSafetyReport, analyze_untrusted_content, wrap_untrusted_content
43
+ from .network import diagnose_trace, normalize_proxy, redact_proxy
44
44
  from .validators import Verdict, validate, TERMINAL_NONSUCCESS
45
45
  from .waf_detector import detect, load_profile, _load_profiles, last_load_error
46
46
  from .url_transforms import iter_transformed
@@ -130,21 +130,21 @@ class FetchResult:
130
130
  # "bot_detection" = routes disagree or a WAF/challenge signal → bypassable
131
131
  # (browser / more routes may help); "infra_or_auth" = every route uniformly
132
132
  # 401/404 → a real wall stealth cannot clear.
133
- block_class: str = ""
134
- proxy_used: bool = False
135
- network_diagnosis: dict = field(default_factory=dict)
136
- session_seeded: bool = False
137
- retrieved_at: str = ""
138
- content_sha256: str = ""
139
-
140
- def __post_init__(self) -> None:
141
- if not self.retrieved_at:
142
- self.retrieved_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
143
- if not self.content_sha256:
144
- self.content_sha256 = hashlib.sha256(
145
- self.content.encode("utf-8", "surrogatepass")
146
- ).hexdigest()
147
- report = analyze_untrusted_content(self.content, source_url=self.final_url)
133
+ block_class: str = ""
134
+ proxy_used: bool = False
135
+ network_diagnosis: dict = field(default_factory=dict)
136
+ session_seeded: bool = False
137
+ retrieved_at: str = ""
138
+ content_sha256: str = ""
139
+
140
+ def __post_init__(self) -> None:
141
+ if not self.retrieved_at:
142
+ self.retrieved_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
143
+ if not self.content_sha256:
144
+ self.content_sha256 = hashlib.sha256(
145
+ self.content.encode("utf-8", "surrogatepass")
146
+ ).hexdigest()
147
+ report = analyze_untrusted_content(self.content, source_url=self.final_url)
148
148
  if not self.content_trust:
149
149
  self.content_trust = report.content_trust
150
150
  if not self.prompt_injection_risk:
@@ -154,7 +154,7 @@ class FetchResult:
154
154
  if not self.untrusted_content_boundary:
155
155
  self.untrusted_content_boundary = dict(report.untrusted_content_boundary)
156
156
 
157
- def to_untrusted_text(self) -> str:
157
+ def to_untrusted_text(self) -> str:
158
158
  report = ContentSafetyReport(
159
159
  content_trust=self.content_trust,
160
160
  prompt_injection_risk=self.prompt_injection_risk,
@@ -164,65 +164,65 @@ class FetchResult:
164
164
  "end": self.untrusted_content_boundary["end"],
165
165
  },
166
166
  )
167
- return wrap_untrusted_content(self.content, report=report, source_url=self.final_url)
168
-
169
- def evidence_record(self) -> dict[str, Any]:
170
- """Return provenance metadata without exposing fetched content or secrets."""
171
- from .url_masking import mask_url
172
-
173
- requested_url = self.trace[0].url if self.trace else self.final_url
174
- source = self.extraction_source or "raw"
175
- if not self.ok:
176
- source_kind = "unknown"
177
- currentness = "unavailable"
178
- elif source.startswith("archive+"):
179
- source_kind = "archive"
180
- currentness = "historical_snapshot"
181
- elif source.startswith("pdf"):
182
- source_kind = "document"
183
- currentness = "current"
184
- elif (self.profile_used or "").startswith(("phase0:youtube", "phase0:media", "phase0:threads")):
185
- source_kind = "media"
186
- currentness = "current"
187
- else:
188
- source_kind = "web"
189
- currentness = "current"
190
- winning_attempt = next(
191
- (attempt for attempt in reversed(self.trace) if attempt.verdict in _OK_VALUES),
192
- self.trace[-1] if self.trace else None,
193
- )
194
- access_method = winning_attempt.executor if winning_attempt else "unknown"
195
- return {
196
- "requested_url": mask_url(requested_url),
197
- "final_url": mask_url(self.final_url),
198
- "retrieved_at": self.retrieved_at,
199
- "retrieval_status": "succeeded" if self.ok else "failed",
200
- "source_timestamp": self.extraction_meta.get("snapshot_timestamp", ""),
201
- "source_kind": source_kind,
202
- "currentness": currentness,
203
- "access_method": access_method,
204
- "source_route": self.profile_used or "",
205
- "content_sha256": self.content_sha256,
206
- "content_length": len(self.content),
207
- "extraction_source": source,
208
- "extraction_quality": self.extraction_quality,
209
- "content_trust": self.content_trust,
210
- "prompt_injection_risk": self.prompt_injection_risk,
211
- "session_seeded": self.session_seeded,
212
- "proxy_used": self.proxy_used,
213
- }
214
-
215
- def to_evidence_bundle(self) -> dict[str, Any]:
216
- """Pair compact provenance with boundary-wrapped, untrusted content."""
217
- operational = self.to_dict()
218
- operational.pop("evidence", None)
219
- operational.pop("retrieved_at", None)
220
- operational.pop("content_sha256", None)
221
- return {
222
- "evidence": self.evidence_record(),
223
- "result": operational,
224
- "untrusted_content": self.to_untrusted_text(),
225
- }
167
+ return wrap_untrusted_content(self.content, report=report, source_url=self.final_url)
168
+
169
+ def evidence_record(self) -> dict[str, Any]:
170
+ """Return provenance metadata without exposing fetched content or secrets."""
171
+ from .url_masking import mask_url
172
+
173
+ requested_url = self.trace[0].url if self.trace else self.final_url
174
+ source = self.extraction_source or "raw"
175
+ if not self.ok:
176
+ source_kind = "unknown"
177
+ currentness = "unavailable"
178
+ elif source.startswith("archive+"):
179
+ source_kind = "archive"
180
+ currentness = "historical_snapshot"
181
+ elif source.startswith("pdf"):
182
+ source_kind = "document"
183
+ currentness = "current"
184
+ elif (self.profile_used or "").startswith(("phase0:youtube", "phase0:media", "phase0:threads")):
185
+ source_kind = "media"
186
+ currentness = "current"
187
+ else:
188
+ source_kind = "web"
189
+ currentness = "current"
190
+ winning_attempt = next(
191
+ (attempt for attempt in reversed(self.trace) if attempt.verdict in _OK_VALUES),
192
+ self.trace[-1] if self.trace else None,
193
+ )
194
+ access_method = winning_attempt.executor if winning_attempt else "unknown"
195
+ return {
196
+ "requested_url": mask_url(requested_url),
197
+ "final_url": mask_url(self.final_url),
198
+ "retrieved_at": self.retrieved_at,
199
+ "retrieval_status": "succeeded" if self.ok else "failed",
200
+ "source_timestamp": self.extraction_meta.get("snapshot_timestamp", ""),
201
+ "source_kind": source_kind,
202
+ "currentness": currentness,
203
+ "access_method": access_method,
204
+ "source_route": self.profile_used or "",
205
+ "content_sha256": self.content_sha256,
206
+ "content_length": len(self.content),
207
+ "extraction_source": source,
208
+ "extraction_quality": self.extraction_quality,
209
+ "content_trust": self.content_trust,
210
+ "prompt_injection_risk": self.prompt_injection_risk,
211
+ "session_seeded": self.session_seeded,
212
+ "proxy_used": self.proxy_used,
213
+ }
214
+
215
+ def to_evidence_bundle(self) -> dict[str, Any]:
216
+ """Pair compact provenance with boundary-wrapped, untrusted content."""
217
+ operational = self.to_dict()
218
+ operational.pop("evidence", None)
219
+ operational.pop("retrieved_at", None)
220
+ operational.pop("content_sha256", None)
221
+ return {
222
+ "evidence": self.evidence_record(),
223
+ "result": operational,
224
+ "untrusted_content": self.to_untrusted_text(),
225
+ }
226
226
 
227
227
  def to_dict(self) -> dict:
228
228
  return {
@@ -246,14 +246,14 @@ class FetchResult:
246
246
  "extraction_quality": self.extraction_quality,
247
247
  "extraction_source": self.extraction_source,
248
248
  "extraction_meta": self.extraction_meta,
249
- "block_class": self.block_class,
250
- "proxy_used": self.proxy_used,
251
- "network_diagnosis": self.network_diagnosis,
252
- "session_seeded": self.session_seeded,
253
- "retrieved_at": self.retrieved_at,
254
- "content_sha256": self.content_sha256,
255
- "evidence": self.evidence_record(),
256
- }
249
+ "block_class": self.block_class,
250
+ "proxy_used": self.proxy_used,
251
+ "network_diagnosis": self.network_diagnosis,
252
+ "session_seeded": self.session_seeded,
253
+ "retrieved_at": self.retrieved_at,
254
+ "content_sha256": self.content_sha256,
255
+ "evidence": self.evidence_record(),
256
+ }
257
257
 
258
258
 
259
259
  # --- Content rescue extraction -----------------------------------------------
@@ -266,13 +266,13 @@ class FetchResult:
266
266
  # * SPA shells whose visible text is thinner than their JSON-LD articleBody
267
267
  # * Playwright render-merge: rendered innerText wins over a thinner body
268
268
  # pypdf is an optional import; every path degrades to the raw text.
269
- import io as _io
270
- import json as _json
271
- import re as _re
272
- import shutil as _shutil
273
- import subprocess as _subprocess
274
- import tempfile as _tempfile
275
- from pathlib import Path as _Path
269
+ import io as _io
270
+ import json as _json
271
+ import re as _re
272
+ import shutil as _shutil
273
+ import subprocess as _subprocess
274
+ import tempfile as _tempfile
275
+ from pathlib import Path as _Path
276
276
 
277
277
  try:
278
278
  from pypdf import PdfReader as _PdfReader
@@ -318,11 +318,11 @@ _SCAN_LIMIT = 2_000_000 # chars of body the rescue regexes may scan
318
318
  _JSONLD_MAX_BLOCKS = 10 # ld+json blocks parsed per page
319
319
  _JSONLD_MAX_BLOB = 200_000 # chars of a single ld+json blob given to json.loads
320
320
  _RESCUE_MAX_TEXT = 1_000_000 # chars any rescue path may return as content
321
- _PDF_MAX_BYTES = 25 * 1024 * 1024 # PDF bodies above this are never parsed
322
- _INNER_TEXT_MAX = 1_000_000 # chars of Playwright innerText accepted
323
- _PDF_TABLE_MAX_CHARS = 200_000
324
- _OCR_MAX_PAGES = 20
325
- _OCR_TOTAL_TIMEOUT = 120.0
321
+ _PDF_MAX_BYTES = 25 * 1024 * 1024 # PDF bodies above this are never parsed
322
+ _INNER_TEXT_MAX = 1_000_000 # chars of Playwright innerText accepted
323
+ _PDF_TABLE_MAX_CHARS = 200_000
324
+ _OCR_MAX_PAGES = 20
325
+ _OCR_TOTAL_TIMEOUT = 120.0
326
326
 
327
327
 
328
328
  def _quality_score(md: str) -> float:
@@ -344,7 +344,7 @@ def _visible_text(html: str) -> str:
344
344
  return _re.sub(r"\s+", " ", t).strip()
345
345
 
346
346
 
347
- def _extract_json_ld_text(html: str) -> str:
347
+ def _extract_json_ld_text(html: str) -> str:
348
348
  """Pull articleBody / description from <script type=application/ld+json>.
349
349
 
350
350
  Bounded: at most _JSONLD_MAX_BLOCKS blocks are parsed, a blob larger than
@@ -379,34 +379,34 @@ def _extract_json_ld_text(html: str) -> str:
379
379
  break
380
380
  except Exception:
381
381
  pass
382
- return "\n\n".join(out)
383
-
384
-
385
- def _pdf_table_to_markdown(table: list[list[Any]]) -> str:
386
- """Convert one bounded pdfplumber table to Markdown without losing units."""
387
- if not table:
388
- return ""
389
- rows: list[list[str]] = []
390
- width = min(max((len(row or []) for row in table), default=0), 20)
391
- if width == 0:
392
- return ""
393
- for raw_row in table[:100]:
394
- raw_row = raw_row or []
395
- row = []
396
- for cell in list(raw_row[:width]) + [""] * max(0, width - len(raw_row)):
397
- value = _re.sub(r"\s+", " ", str(cell or "")).strip().replace("|", r"\|")
398
- row.append(value)
399
- rows.append(row)
400
- if not any(any(cell for cell in row) for row in rows):
401
- return ""
402
- header = rows[0]
403
- lines = ["| " + " | ".join(header) + " |",
404
- "| " + " | ".join("---" for _ in header) + " |"]
405
- lines.extend("| " + " | ".join(row) + " |" for row in rows[1:])
406
- return "\n".join(lines)
407
-
408
-
409
- def _extract_pdf_pdfplumber(body: bytes) -> tuple[str, str]:
382
+ return "\n\n".join(out)
383
+
384
+
385
+ def _pdf_table_to_markdown(table: list[list[Any]]) -> str:
386
+ """Convert one bounded pdfplumber table to Markdown without losing units."""
387
+ if not table:
388
+ return ""
389
+ rows: list[list[str]] = []
390
+ width = min(max((len(row or []) for row in table), default=0), 20)
391
+ if width == 0:
392
+ return ""
393
+ for raw_row in table[:100]:
394
+ raw_row = raw_row or []
395
+ row = []
396
+ for cell in list(raw_row[:width]) + [""] * max(0, width - len(raw_row)):
397
+ value = _re.sub(r"\s+", " ", str(cell or "")).strip().replace("|", r"\|")
398
+ row.append(value)
399
+ rows.append(row)
400
+ if not any(any(cell for cell in row) for row in rows):
401
+ return ""
402
+ header = rows[0]
403
+ lines = ["| " + " | ".join(header) + " |",
404
+ "| " + " | ".join("---" for _ in header) + " |"]
405
+ lines.extend("| " + " | ".join(row) + " |" for row in rows[1:])
406
+ return "\n".join(lines)
407
+
408
+
409
+ def _extract_pdf_pdfplumber(body: bytes) -> tuple[str, str]:
410
410
  """(title, text) via pdfplumber, or ("", "") on failure / no text layer.
411
411
  Same bounds as the pypdf path: ≤80 pages, text capped at _RESCUE_MAX_TEXT."""
412
412
  if _pdfplumber is None:
@@ -420,37 +420,37 @@ def _extract_pdf_pdfplumber(body: bytes) -> tuple[str, str]:
420
420
  title = str(md["Title"])[:300]
421
421
  except Exception:
422
422
  pass
423
- parts: list[str] = []
424
- total = 0
425
- for page in pdf.pages[:80]:
423
+ parts: list[str] = []
424
+ total = 0
425
+ for page in pdf.pages[:80]:
426
426
  try:
427
427
  t = page.extract_text() or ""
428
428
  except Exception:
429
429
  t = ""
430
430
  take = t[:_RESCUE_MAX_TEXT - total]
431
- parts.append(take)
432
- total += len(take)
433
- if total < _RESCUE_MAX_TEXT:
434
- try:
435
- tables = page.extract_tables() or []
436
- except Exception:
437
- tables = []
438
- for table in tables[:20]:
439
- markdown = _pdf_table_to_markdown(table)
440
- if not markdown:
441
- continue
442
- take_table = markdown[:min(
443
- _PDF_TABLE_MAX_CHARS,
444
- _RESCUE_MAX_TEXT - total,
445
- )]
446
- if take_table:
447
- parts.append("[PDF table]\n" + take_table)
448
- total += len(take_table)
449
- if total >= _RESCUE_MAX_TEXT:
450
- break
451
- if total >= _RESCUE_MAX_TEXT:
452
- break
453
- return title, "\n\n".join(p for p in parts if p).strip()
431
+ parts.append(take)
432
+ total += len(take)
433
+ if total < _RESCUE_MAX_TEXT:
434
+ try:
435
+ tables = page.extract_tables() or []
436
+ except Exception:
437
+ tables = []
438
+ for table in tables[:20]:
439
+ markdown = _pdf_table_to_markdown(table)
440
+ if not markdown:
441
+ continue
442
+ take_table = markdown[:min(
443
+ _PDF_TABLE_MAX_CHARS,
444
+ _RESCUE_MAX_TEXT - total,
445
+ )]
446
+ if take_table:
447
+ parts.append("[PDF table]\n" + take_table)
448
+ total += len(take_table)
449
+ if total >= _RESCUE_MAX_TEXT:
450
+ break
451
+ if total >= _RESCUE_MAX_TEXT:
452
+ break
453
+ return title, "\n\n".join(p for p in parts if p).strip()
454
454
  except Exception:
455
455
  return "", ""
456
456
 
@@ -484,66 +484,66 @@ def _extract_pdf_pypdf(body: bytes) -> tuple[str, str, str]:
484
484
  return "", "", f"pdf_error:{type(e).__name__}"
485
485
 
486
486
 
487
- def _extract_pdf_ocr(body: bytes) -> tuple[str, str]:
488
- """OCR a scanned PDF with local Poppler and Tesseract when explicitly enabled."""
489
- pdftoppm = _shutil.which("pdftoppm")
490
- tesseract = _shutil.which("tesseract")
491
- if not pdftoppm or not tesseract:
492
- return "", "pdf_ocr_tools_missing"
493
- started = time.monotonic()
494
- try:
495
- with _tempfile.TemporaryDirectory(prefix="insane-pdf-ocr-") as root:
496
- root_path = _Path(root)
497
- source = root_path / "source.pdf"
498
- source.write_bytes(body)
499
- rendered = _subprocess.run(
500
- [pdftoppm, "-f", "1", "-l", str(_OCR_MAX_PAGES), "-r", "150",
501
- "-png", str(source), str(root_path / "page")],
502
- capture_output=True, timeout=60, check=False,
503
- )
504
- if rendered.returncode != 0:
505
- return "", "pdf_ocr_render_failed"
506
-
507
- language = "eng"
508
- try:
509
- listed = _subprocess.run(
510
- [tesseract, "--list-langs"], capture_output=True, text=True,
511
- encoding="utf-8", errors="replace", timeout=10, check=False)
512
- languages = set((listed.stdout or "").split())
513
- if "kor" in languages:
514
- language = "kor+eng" if "eng" in languages else "kor"
515
- except Exception:
516
- pass
517
-
518
- parts: list[str] = []
519
- total = 0
520
- for image in sorted(root_path.glob("page-*.png"))[:_OCR_MAX_PAGES]:
521
- remaining = _OCR_TOTAL_TIMEOUT - (time.monotonic() - started)
522
- if remaining <= 1:
523
- break
524
- completed = _subprocess.run(
525
- [tesseract, str(image), "stdout", "-l", language],
526
- capture_output=True, text=True, encoding="utf-8", errors="replace",
527
- timeout=min(20, remaining), check=False,
528
- )
529
- if completed.returncode != 0:
530
- continue
531
- text = (completed.stdout or "").strip()
532
- take = text[:_RESCUE_MAX_TEXT - total]
533
- if take:
534
- parts.append(take)
535
- total += len(take)
536
- if total >= _RESCUE_MAX_TEXT:
537
- break
538
- output = "\n\n".join(parts).strip()
539
- return (output, "") if output else ("", "pdf_ocr_no_text")
540
- except _subprocess.TimeoutExpired:
541
- return "", "pdf_ocr_timeout"
542
- except Exception as exc:
543
- return "", f"pdf_ocr_error:{type(exc).__name__}"
544
-
545
-
546
- def _extract_pdf(body: bytes, url: str, enable_ocr: bool = False) -> tuple[str, str, float, str]:
487
+ def _extract_pdf_ocr(body: bytes) -> tuple[str, str]:
488
+ """OCR a scanned PDF with local Poppler and Tesseract when explicitly enabled."""
489
+ pdftoppm = _shutil.which("pdftoppm")
490
+ tesseract = _shutil.which("tesseract")
491
+ if not pdftoppm or not tesseract:
492
+ return "", "pdf_ocr_tools_missing"
493
+ started = time.monotonic()
494
+ try:
495
+ with _tempfile.TemporaryDirectory(prefix="insane-pdf-ocr-") as root:
496
+ root_path = _Path(root)
497
+ source = root_path / "source.pdf"
498
+ source.write_bytes(body)
499
+ rendered = _subprocess.run(
500
+ [pdftoppm, "-f", "1", "-l", str(_OCR_MAX_PAGES), "-r", "150",
501
+ "-png", str(source), str(root_path / "page")],
502
+ capture_output=True, timeout=60, check=False,
503
+ )
504
+ if rendered.returncode != 0:
505
+ return "", "pdf_ocr_render_failed"
506
+
507
+ language = "eng"
508
+ try:
509
+ listed = _subprocess.run(
510
+ [tesseract, "--list-langs"], capture_output=True, text=True,
511
+ encoding="utf-8", errors="replace", timeout=10, check=False)
512
+ languages = set((listed.stdout or "").split())
513
+ if "kor" in languages:
514
+ language = "kor+eng" if "eng" in languages else "kor"
515
+ except Exception:
516
+ pass
517
+
518
+ parts: list[str] = []
519
+ total = 0
520
+ for image in sorted(root_path.glob("page-*.png"))[:_OCR_MAX_PAGES]:
521
+ remaining = _OCR_TOTAL_TIMEOUT - (time.monotonic() - started)
522
+ if remaining <= 1:
523
+ break
524
+ completed = _subprocess.run(
525
+ [tesseract, str(image), "stdout", "-l", language],
526
+ capture_output=True, text=True, encoding="utf-8", errors="replace",
527
+ timeout=min(20, remaining), check=False,
528
+ )
529
+ if completed.returncode != 0:
530
+ continue
531
+ text = (completed.stdout or "").strip()
532
+ take = text[:_RESCUE_MAX_TEXT - total]
533
+ if take:
534
+ parts.append(take)
535
+ total += len(take)
536
+ if total >= _RESCUE_MAX_TEXT:
537
+ break
538
+ output = "\n\n".join(parts).strip()
539
+ return (output, "") if output else ("", "pdf_ocr_no_text")
540
+ except _subprocess.TimeoutExpired:
541
+ return "", "pdf_ocr_timeout"
542
+ except Exception as exc:
543
+ return "", f"pdf_ocr_error:{type(exc).__name__}"
544
+
545
+
546
+ def _extract_pdf(body: bytes, url: str, enable_ocr: bool = False) -> tuple[str, str, float, str]:
547
547
  """Returns (title, text, quality, error_code). error_code is "" on success.
548
548
  Caps at 80 pages to keep token budget sane; reports pdf_no_text_layer for
549
549
  scanned PDFs (so the caller knows rendering will not help either).
@@ -554,8 +554,8 @@ def _extract_pdf(body: bytes, url: str, enable_ocr: bool = False) -> tuple[str,
554
554
  the extracted text is capped at _RESCUE_MAX_TEXT."""
555
555
  if len(body) > _PDF_MAX_BYTES:
556
556
  return "", "", 0.0, "pdf_too_large"
557
- if _pdfplumber is None and _PdfReader is None and not enable_ocr:
558
- return "", "", 0.0, "pdf_no_extractor"
557
+ if _pdfplumber is None and _PdfReader is None and not enable_ocr:
558
+ return "", "", 0.0, "pdf_no_extractor"
559
559
 
560
560
  # 1) pdfplumber (preferred). Only adopt when it yields text.
561
561
  p_title, p_text = _extract_pdf_pdfplumber(body)
@@ -566,14 +566,14 @@ def _extract_pdf(body: bytes, url: str, enable_ocr: bool = False) -> tuple[str,
566
566
  y_title, y_text, y_err = _extract_pdf_pypdf(body)
567
567
  if y_text:
568
568
  return y_title, y_text, _quality_score(y_text), ""
569
- if y_err and y_err != "pypdf_missing":
570
- return y_title, "", 0.0, y_err
571
-
572
- if enable_ocr:
573
- ocr_text, ocr_error = _extract_pdf_ocr(body)
574
- if ocr_text:
575
- return (p_title or y_title), ocr_text, _quality_score(ocr_text), "pdf_ocr"
576
- return (p_title or y_title), "", 0.0, ocr_error
569
+ if y_err and y_err != "pypdf_missing":
570
+ return y_title, "", 0.0, y_err
571
+
572
+ if enable_ocr:
573
+ ocr_text, ocr_error = _extract_pdf_ocr(body)
574
+ if ocr_text:
575
+ return (p_title or y_title), ocr_text, _quality_score(ocr_text), "pdf_ocr"
576
+ return (p_title or y_title), "", 0.0, ocr_error
577
577
 
578
578
  # Neither produced text: prefer any title we found; report no text layer.
579
579
  return (p_title or y_title), "", 0.0, "pdf_no_text_layer"
@@ -644,10 +644,10 @@ class _PWResp:
644
644
  self.headers = headers or {"content-type": "text/html"}
645
645
 
646
646
 
647
- def _extract_response(resp, final_url: str, inner_text: str = "",
648
- enable_markdown: bool = True,
649
- enable_maincontent: bool = False,
650
- enable_ocr: bool = False) -> tuple[str, str, float, dict]:
647
+ def _extract_response(resp, final_url: str, inner_text: str = "",
648
+ enable_markdown: bool = True,
649
+ enable_maincontent: bool = False,
650
+ enable_ocr: bool = False) -> tuple[str, str, float, dict]:
651
651
  """Returns (title, content, quality, meta); meta = {source, error,
652
652
  inner_text_used}. The raw body wins by default — see the module-block
653
653
  comment for when a rescue path may replace it.
@@ -668,12 +668,12 @@ def _extract_response(resp, final_url: str, inner_text: str = "",
668
668
  except Exception:
669
669
  pass
670
670
  if bytes(body[:5]) == b"%PDF-" or ctype_pdf:
671
- title, text, quality, err = _extract_pdf(
672
- bytes(body), final_url, enable_ocr=enable_ocr)
673
- if text:
674
- source = "pdf+ocr" if err == "pdf_ocr" else "pdf"
675
- return title, text, quality, {"source": source, "error": "",
676
- "inner_text_used": False}
671
+ title, text, quality, err = _extract_pdf(
672
+ bytes(body), final_url, enable_ocr=enable_ocr)
673
+ if text:
674
+ source = "pdf+ocr" if err == "pdf_ocr" else "pdf"
675
+ return title, text, quality, {"source": source, "error": "",
676
+ "inner_text_used": False}
677
677
  return title, f"[PDF binary, {len(body)} bytes; extractor={err or 'ok'}]", \
678
678
  0.0, {"source": "pdf", "error": err, "inner_text_used": False}
679
679
 
@@ -736,24 +736,24 @@ def _extract_response(resp, final_url: str, inner_text: str = "",
736
736
  "inner_text_used": inner_used}
737
737
 
738
738
 
739
- def _maybe_extract(resp, final_url: str, *, enable_extraction: bool,
740
- inner_text: str = "", enable_markdown: bool = True,
741
- enable_maincontent: bool = False,
742
- enable_ocr: bool = False) -> tuple[str, str, float, dict]:
739
+ def _maybe_extract(resp, final_url: str, *, enable_extraction: bool,
740
+ inner_text: str = "", enable_markdown: bool = True,
741
+ enable_maincontent: bool = False,
742
+ enable_ocr: bool = False) -> tuple[str, str, float, dict]:
743
743
  """Run rescue extraction when enabled; otherwise raw text + consistent meta."""
744
744
  if not enable_extraction:
745
745
  return "", getattr(resp, "text", "") or "", 0.0, \
746
746
  {"source": "raw_disabled", "error": "", "inner_text_used": False}
747
- return _extract_response(resp, final_url, inner_text=inner_text,
748
- enable_markdown=enable_markdown,
749
- enable_maincontent=enable_maincontent,
750
- enable_ocr=enable_ocr)
747
+ return _extract_response(resp, final_url, inner_text=inner_text,
748
+ enable_markdown=enable_markdown,
749
+ enable_maincontent=enable_maincontent,
750
+ enable_ocr=enable_ocr)
751
751
 
752
752
 
753
753
  # --- curl_cffi probe executor ------------------------------------------------
754
- def _curl_probe(
755
- url: str, *, impersonate: str, referer: str, timeout: int = 20,
756
- enable_retry: bool = False, proxy: Optional[str] = None,
754
+ def _curl_probe(
755
+ url: str, *, impersonate: str, referer: str, timeout: int = 20,
756
+ enable_retry: bool = False, proxy: Optional[str] = None,
757
757
  ) -> tuple[Any, Optional[str]]:
758
758
  """Returns (response, error_str). response may be None on exception.
759
759
 
@@ -761,11 +761,11 @@ def _curl_probe(
761
761
  warm connection persist across attempts and across pages of the same host.
762
762
  The pool degrades to a one-shot GET when a Session can't be created.
763
763
  """
764
- from .transport import POOL
765
- response, error = POOL.request(
766
- url, impersonate=impersonate, referer=referer, timeout=timeout,
767
- max_retries=2 if enable_retry else 0, proxy=proxy)
768
- return response, redact_proxy(error, proxy)
764
+ from .transport import POOL
765
+ response, error = POOL.request(
766
+ url, impersonate=impersonate, referer=referer, timeout=timeout,
767
+ max_retries=2 if enable_retry else 0, proxy=proxy)
768
+ return response, redact_proxy(error, proxy)
769
769
 
770
770
 
771
771
  def _run_attempt(
@@ -777,15 +777,15 @@ def _run_attempt(
777
777
  success_selectors: Optional[list[str]],
778
778
  known_bad_sizes: Optional[list[int]],
779
779
  timeout: int,
780
- phase: str,
781
- enable_retry: bool = False,
782
- proxy: Optional[str] = None,
780
+ phase: str,
781
+ enable_retry: bool = False,
782
+ proxy: Optional[str] = None,
783
783
  ) -> tuple[Attempt, Any]:
784
784
  """Execute one curl_cffi attempt and produce an Attempt record."""
785
785
  referer_url = REFERER_STRATEGIES.get(referer_name, REFERER_STRATEGIES["none"])(url)
786
786
  t0 = time.time()
787
- resp, err = _curl_probe(url, impersonate=impersonate, referer=referer_url, timeout=timeout,
788
- enable_retry=enable_retry, proxy=proxy)
787
+ resp, err = _curl_probe(url, impersonate=impersonate, referer=referer_url, timeout=timeout,
788
+ enable_retry=enable_retry, proxy=proxy)
789
789
  elapsed = round(time.time() - t0, 3)
790
790
 
791
791
  att = Attempt(
@@ -937,7 +937,7 @@ def _build_plan(
937
937
 
938
938
 
939
939
  # --- Public entrypoint: self-learning wrapper (U5) ---------------------------
940
- def _winning_route(result: FetchResult) -> Optional[dict]:
940
+ def _winning_route(result: FetchResult) -> Optional[dict]:
941
941
  """Extract the curl route that produced the OK result, from the trace.
942
942
 
943
943
  Only probe/grid curl wins are learnable: Phase 0 always runs first anyway,
@@ -951,73 +951,73 @@ def _winning_route(result: FetchResult) -> Optional[dict]:
951
951
  "referer": att.referer,
952
952
  "phase": att.phase,
953
953
  }
954
- return None
955
-
956
-
957
- def _archive_fallback(
958
- base: FetchResult,
959
- url: str,
960
- *,
961
- timeout: int,
962
- proxy: Optional[str],
963
- enable_extraction: bool,
964
- enable_markdown: bool,
965
- enable_maincontent: bool,
966
- ) -> FetchResult:
967
- """Opt-in historical fallback. A snapshot is always labeled non-current."""
968
- try:
969
- from .phase0 import route_archive
970
- archived = route_archive(url, timeout=timeout, proxy=proxy)
971
- except Exception as exc:
972
- archived = {
973
- "ok": False,
974
- "attempts": [{
975
- "route": "wayback", "ok": False, "status": 0, "bytes": 0,
976
- "note": f"{type(exc).__name__}",
977
- }],
978
- }
979
- trace = list(base.trace)
980
- for item in archived.get("attempts") or []:
981
- trace.append(Attempt(
982
- phase="archive", executor=item.get("route") or "wayback",
983
- url=archived.get("final_url") or url, url_transform="original",
984
- impersonate=None, referer="", status=int(item.get("status") or 0),
985
- body_size=int(item.get("bytes") or 0),
986
- verdict=(Verdict.STRONG_OK.value if item.get("ok") else Verdict.BLOCKED.value),
987
- reasons=[item.get("note")] if item.get("note") else [],
988
- ))
989
- if not archived.get("ok"):
990
- base.trace = trace
991
- notes = [item.get("note") for item in archived.get("attempts") or [] if item.get("note")]
992
- base.summary += f"; archive_failed={notes[-1] if notes else 'unknown'}"
993
- return base
994
-
995
- final_url = str(archived.get("final_url") or url)
996
- _title, content, quality, meta = _maybe_extract(
997
- _PWResp(str(archived.get("content") or ""), final_url), final_url,
998
- enable_extraction=enable_extraction,
999
- enable_markdown=enable_markdown,
1000
- enable_maincontent=enable_maincontent,
1001
- )
1002
- source = "archive+" + (meta.get("source") or "raw")
1003
- meta = dict(meta)
1004
- meta.update({
1005
- "source": source,
1006
- "snapshot_timestamp": str(archived.get("timestamp") or ""),
1007
- })
1008
- return FetchResult(
1009
- ok=True, content=content, final_url=final_url,
1010
- verdict=Verdict.STRONG_OK.value, profile_used="archive:wayback",
1011
- trace=trace, summary="Historical archive fallback succeeded",
1012
- planned_attempts=base.planned_attempts,
1013
- executed_attempts=base.executed_attempts,
1014
- grid_exhausted=base.grid_exhausted, stop_reason="success",
1015
- extraction_quality=quality, extraction_source=source,
1016
- extraction_meta=meta,
1017
- )
1018
-
1019
-
1020
- def fetch(
954
+ return None
955
+
956
+
957
+ def _archive_fallback(
958
+ base: FetchResult,
959
+ url: str,
960
+ *,
961
+ timeout: int,
962
+ proxy: Optional[str],
963
+ enable_extraction: bool,
964
+ enable_markdown: bool,
965
+ enable_maincontent: bool,
966
+ ) -> FetchResult:
967
+ """Opt-in historical fallback. A snapshot is always labeled non-current."""
968
+ try:
969
+ from .phase0 import route_archive
970
+ archived = route_archive(url, timeout=timeout, proxy=proxy)
971
+ except Exception as exc:
972
+ archived = {
973
+ "ok": False,
974
+ "attempts": [{
975
+ "route": "wayback", "ok": False, "status": 0, "bytes": 0,
976
+ "note": f"{type(exc).__name__}",
977
+ }],
978
+ }
979
+ trace = list(base.trace)
980
+ for item in archived.get("attempts") or []:
981
+ trace.append(Attempt(
982
+ phase="archive", executor=item.get("route") or "wayback",
983
+ url=archived.get("final_url") or url, url_transform="original",
984
+ impersonate=None, referer="", status=int(item.get("status") or 0),
985
+ body_size=int(item.get("bytes") or 0),
986
+ verdict=(Verdict.STRONG_OK.value if item.get("ok") else Verdict.BLOCKED.value),
987
+ reasons=[item.get("note")] if item.get("note") else [],
988
+ ))
989
+ if not archived.get("ok"):
990
+ base.trace = trace
991
+ notes = [item.get("note") for item in archived.get("attempts") or [] if item.get("note")]
992
+ base.summary += f"; archive_failed={notes[-1] if notes else 'unknown'}"
993
+ return base
994
+
995
+ final_url = str(archived.get("final_url") or url)
996
+ _title, content, quality, meta = _maybe_extract(
997
+ _PWResp(str(archived.get("content") or ""), final_url), final_url,
998
+ enable_extraction=enable_extraction,
999
+ enable_markdown=enable_markdown,
1000
+ enable_maincontent=enable_maincontent,
1001
+ )
1002
+ source = "archive+" + (meta.get("source") or "raw")
1003
+ meta = dict(meta)
1004
+ meta.update({
1005
+ "source": source,
1006
+ "snapshot_timestamp": str(archived.get("timestamp") or ""),
1007
+ })
1008
+ return FetchResult(
1009
+ ok=True, content=content, final_url=final_url,
1010
+ verdict=Verdict.STRONG_OK.value, profile_used="archive:wayback",
1011
+ trace=trace, summary="Historical archive fallback succeeded",
1012
+ planned_attempts=base.planned_attempts,
1013
+ executed_attempts=base.executed_attempts,
1014
+ grid_exhausted=base.grid_exhausted, stop_reason="success",
1015
+ extraction_quality=quality, extraction_source=source,
1016
+ extraction_meta=meta,
1017
+ )
1018
+
1019
+
1020
+ def fetch(
1021
1021
  url: str,
1022
1022
  *,
1023
1023
  success_selectors: Optional[list[str]] = None,
@@ -1025,21 +1025,21 @@ def fetch(
1025
1025
  user_hint: Optional[dict] = None,
1026
1026
  timeout: int = 25,
1027
1027
  max_attempts: Optional[int] = None,
1028
- max_browser_attempts: int = 2,
1028
+ max_browser_attempts: int = 2,
1029
1029
  enable_playwright: bool = True,
1030
1030
  enable_phase0: bool = True,
1031
1031
  enable_learning: bool = True,
1032
1032
  enable_extraction: bool = True,
1033
1033
  enable_retry: bool = True,
1034
- enable_markdown: bool = True,
1035
- enable_maincontent: bool = False,
1036
- enable_ocr: bool = False,
1037
- enable_archive: bool = False,
1038
- force_media: bool = False,
1039
- include_media_transcript: bool = False,
1040
- proxy: Optional[str] = None,
1041
- cookie_file: Optional[str] = None,
1042
- ) -> FetchResult:
1034
+ enable_markdown: bool = True,
1035
+ enable_maincontent: bool = False,
1036
+ enable_ocr: bool = False,
1037
+ enable_archive: bool = False,
1038
+ force_media: bool = False,
1039
+ include_media_transcript: bool = False,
1040
+ proxy: Optional[str] = None,
1041
+ cookie_file: Optional[str] = None,
1042
+ ) -> FetchResult:
1043
1043
  """Public entrypoint — the generic grid wrapped with per-host self-learning.
1044
1044
 
1045
1045
  1. Before fetching, look up the route that last succeeded for this host and
@@ -1066,15 +1066,15 @@ def fetch(
1066
1066
  structure-preserving markdown via markdownify (tables → pipe tables,
1067
1067
  <pre>/<code> → fences); ``extraction_source`` becomes "raw+md". Set False
1068
1068
  for raw HTML. No-op when markdownify is not installed.
1069
- ``enable_maincontent`` (opt-in) instead strips boilerplate via resiliparse."""
1070
- proxy = normalize_proxy(proxy)
1071
- cookies: list[dict] = []
1072
- if cookie_file:
1073
- from .session_input import load_cookie_file
1074
- from .transport import POOL, _host_of
1075
- cookies = load_cookie_file(cookie_file, url)
1076
- POOL.seed_cookies(_host_of(url), cookies, proxy=proxy)
1077
- priority: Optional[dict] = None
1069
+ ``enable_maincontent`` (opt-in) instead strips boilerplate via resiliparse."""
1070
+ proxy = normalize_proxy(proxy)
1071
+ cookies: list[dict] = []
1072
+ if cookie_file:
1073
+ from .session_input import load_cookie_file
1074
+ from .transport import POOL, _host_of
1075
+ cookies = load_cookie_file(cookie_file, url)
1076
+ POOL.seed_cookies(_host_of(url), cookies, proxy=proxy)
1077
+ priority: Optional[dict] = None
1078
1078
  learned_existed = False
1079
1079
  uh = dict(user_hint or {})
1080
1080
  try:
@@ -1093,26 +1093,26 @@ def fetch(
1093
1093
  user_hint=uh, timeout=timeout, max_attempts=max_attempts,
1094
1094
  max_browser_attempts=max_browser_attempts,
1095
1095
  enable_playwright=enable_playwright, enable_phase0=enable_phase0,
1096
- priority=priority,
1097
- enable_extraction=enable_extraction, enable_retry=enable_retry,
1098
- enable_markdown=enable_markdown, enable_maincontent=enable_maincontent,
1099
- enable_ocr=enable_ocr, force_media=force_media,
1100
- include_media_transcript=include_media_transcript,
1101
- proxy=proxy, cookies=cookies,
1102
- )
1103
-
1104
- if enable_archive and not result.ok:
1105
- result = _archive_fallback(
1106
- result, url, timeout=timeout, proxy=proxy,
1107
- enable_extraction=enable_extraction,
1108
- enable_markdown=enable_markdown,
1109
- enable_maincontent=enable_maincontent,
1110
- )
1111
-
1112
- result.proxy_used = proxy is not None
1113
- result.session_seeded = bool(cookies)
1114
- if not result.ok:
1115
- result.network_diagnosis = diagnose_trace(result.trace)
1096
+ priority=priority,
1097
+ enable_extraction=enable_extraction, enable_retry=enable_retry,
1098
+ enable_markdown=enable_markdown, enable_maincontent=enable_maincontent,
1099
+ enable_ocr=enable_ocr, force_media=force_media,
1100
+ include_media_transcript=include_media_transcript,
1101
+ proxy=proxy, cookies=cookies,
1102
+ )
1103
+
1104
+ if enable_archive and not result.ok:
1105
+ result = _archive_fallback(
1106
+ result, url, timeout=timeout, proxy=proxy,
1107
+ enable_extraction=enable_extraction,
1108
+ enable_markdown=enable_markdown,
1109
+ enable_maincontent=enable_maincontent,
1110
+ )
1111
+
1112
+ result.proxy_used = proxy is not None
1113
+ result.session_seeded = bool(cookies)
1114
+ if not result.ok:
1115
+ result.network_diagnosis = diagnose_trace(result.trace)
1116
1116
 
1117
1117
  try:
1118
1118
  from . import learning
@@ -1138,7 +1138,7 @@ def fetch(
1138
1138
 
1139
1139
 
1140
1140
  # --- Main entrypoint ---------------------------------------------------------
1141
- def _fetch_core(
1141
+ def _fetch_core(
1142
1142
  url: str,
1143
1143
  *,
1144
1144
  success_selectors: Optional[list[str]] = None,
@@ -1146,20 +1146,20 @@ def _fetch_core(
1146
1146
  user_hint: Optional[dict] = None,
1147
1147
  timeout: int = 25,
1148
1148
  max_attempts: Optional[int] = None, # None = exhaustive (R6); int = budget
1149
- max_browser_attempts: int = 2,
1149
+ max_browser_attempts: int = 2,
1150
1150
  enable_playwright: bool = True,
1151
1151
  enable_phase0: bool = True,
1152
1152
  priority: Optional[dict] = None, # U5: learned route to retry first
1153
1153
  enable_extraction: bool = True,
1154
1154
  enable_retry: bool = True,
1155
- enable_markdown: bool = True,
1156
- enable_maincontent: bool = False,
1157
- enable_ocr: bool = False,
1158
- force_media: bool = False,
1159
- include_media_transcript: bool = False,
1160
- proxy: Optional[str] = None,
1161
- cookies: Optional[list[dict]] = None,
1162
- ) -> FetchResult:
1155
+ enable_markdown: bool = True,
1156
+ enable_maincontent: bool = False,
1157
+ enable_ocr: bool = False,
1158
+ force_media: bool = False,
1159
+ include_media_transcript: bool = False,
1160
+ proxy: Optional[str] = None,
1161
+ cookies: Optional[list[dict]] = None,
1162
+ ) -> FetchResult:
1163
1163
  """Fetch `url` using the generic diversity grid.
1164
1164
 
1165
1165
  max_attempts
@@ -1199,18 +1199,18 @@ def _fetch_core(
1199
1199
  # is what made Reddit/X look "blocked" (grid 403'd .json; nobody tried .rss).
1200
1200
  if enable_phase0:
1201
1201
  try:
1202
- if force_media:
1203
- from .phase0 import route_media as _phase0_route_media
1204
- p0 = _phase0_route_media(
1205
- url, timeout=timeout, proxy=proxy,
1206
- include_transcript=include_media_transcript,
1207
- cookies=cookies)
1208
- else:
1209
- from .phase0 import route as _phase0_route
1210
- p0 = _phase0_route(
1211
- url, timeout=timeout, proxy=proxy,
1212
- include_media_transcript=include_media_transcript,
1213
- cookies=cookies)
1202
+ if force_media:
1203
+ from .phase0 import route_media as _phase0_route_media
1204
+ p0 = _phase0_route_media(
1205
+ url, timeout=timeout, proxy=proxy,
1206
+ include_transcript=include_media_transcript,
1207
+ cookies=cookies)
1208
+ else:
1209
+ from .phase0 import route as _phase0_route
1210
+ p0 = _phase0_route(
1211
+ url, timeout=timeout, proxy=proxy,
1212
+ include_media_transcript=include_media_transcript,
1213
+ cookies=cookies)
1214
1214
  except Exception as e: # router must never break the generic chain
1215
1215
  p0 = None
1216
1216
  trace.append(Attempt(
@@ -1227,21 +1227,21 @@ def _fetch_core(
1227
1227
  verdict=(Verdict.STRONG_OK.value if a["ok"] else Verdict.BLOCKED.value),
1228
1228
  reasons=[a["note"]] if a.get("note") else [],
1229
1229
  ))
1230
- if p0["ok"]:
1231
- content_kind = p0.get("content_kind") or "phase0"
1232
- return FetchResult(
1233
- ok=True, content=p0["content"], final_url=p0["final_url"],
1230
+ if p0["ok"]:
1231
+ content_kind = p0.get("content_kind") or "phase0"
1232
+ return FetchResult(
1233
+ ok=True, content=p0["content"], final_url=p0["final_url"],
1234
1234
  verdict=Verdict.STRONG_OK.value,
1235
1235
  profile_used=f"phase0:{p0['platform']}", trace=trace,
1236
- summary=f"Phase 0 official route: {p0['platform']}:{p0['route']}",
1237
- stop_reason="success",
1238
- extraction_quality=_quality_score(p0["content"]),
1239
- extraction_source=content_kind,
1240
- extraction_meta={
1241
- "source": content_kind,
1242
- "transcript_available": bool(p0.get("transcript")),
1243
- },
1244
- )
1236
+ summary=f"Phase 0 official route: {p0['platform']}:{p0['route']}",
1237
+ stop_reason="success",
1238
+ extraction_quality=_quality_score(p0["content"]),
1239
+ extraction_source=content_kind,
1240
+ extraction_meta={
1241
+ "source": content_kind,
1242
+ "transcript_available": bool(p0.get("transcript")),
1243
+ },
1244
+ )
1245
1245
  # Recognised platform but every official route failed → fall through
1246
1246
  # to the generic grid (don't give up; R6).
1247
1247
 
@@ -1262,8 +1262,8 @@ def _fetch_core(
1262
1262
  if pool_enabled():
1263
1263
  _root = _root_of(url)
1264
1264
  if _root != url:
1265
- POOL.warmup(_host_of(url), base_impersonate, _root,
1266
- timeout=min(timeout, 15), proxy=proxy)
1265
+ POOL.warmup(_host_of(url), base_impersonate, _root,
1266
+ timeout=min(timeout, 15), proxy=proxy)
1267
1267
  except Exception:
1268
1268
  pass
1269
1269
 
@@ -1274,9 +1274,9 @@ def _fetch_core(
1274
1274
  probe_attempt, probe_resp = _run_attempt(
1275
1275
  url, transform_name="original", impersonate=base_impersonate,
1276
1276
  referer_name=base_referer, success_selectors=success_selectors,
1277
- known_bad_sizes=None, timeout=timeout, phase="probe",
1278
- enable_retry=enable_retry,
1279
- proxy=proxy,
1277
+ known_bad_sizes=None, timeout=timeout, phase="probe",
1278
+ enable_retry=enable_retry,
1279
+ proxy=proxy,
1280
1280
  )
1281
1281
  trace.append(probe_attempt)
1282
1282
  curl_attempts += 1
@@ -1286,10 +1286,10 @@ def _fetch_core(
1286
1286
  return _build_result(probe_resp, probe_attempt, trace, profile_used=None,
1287
1287
  planned=0, executed=curl_attempts,
1288
1288
  grid_exhausted=False, stop_reason="success",
1289
- enable_extraction=enable_extraction,
1290
- enable_markdown=enable_markdown,
1291
- enable_maincontent=enable_maincontent,
1292
- enable_ocr=enable_ocr)
1289
+ enable_extraction=enable_extraction,
1290
+ enable_markdown=enable_markdown,
1291
+ enable_maincontent=enable_maincontent,
1292
+ enable_ocr=enable_ocr)
1293
1293
  if probe_attempt.verdict == Verdict.SUSPECT_OK.value:
1294
1294
  best_suspect = (probe_resp, probe_attempt)
1295
1295
  elif probe_attempt.verdict in _TERMINAL_NONSUCCESS_VALUES:
@@ -1318,9 +1318,9 @@ def _fetch_core(
1318
1318
  att, resp = _run_attempt(
1319
1319
  cand.url, transform_name=cand.transform, impersonate=cand.impersonate,
1320
1320
  referer_name=cand.referer, success_selectors=success_selectors,
1321
- known_bad_sizes=list(cand.known_bad_sizes) if cand.known_bad_sizes else None,
1322
- timeout=timeout, phase="grid",
1323
- proxy=proxy,
1321
+ known_bad_sizes=list(cand.known_bad_sizes) if cand.known_bad_sizes else None,
1322
+ timeout=timeout, phase="grid",
1323
+ proxy=proxy,
1324
1324
  )
1325
1325
  trace.append(att)
1326
1326
  curl_attempts += 1
@@ -1330,10 +1330,10 @@ def _fetch_core(
1330
1330
  return _build_result(resp, att, trace, profile_used=cand.profile_id,
1331
1331
  planned=planned, executed=curl_attempts,
1332
1332
  grid_exhausted=False, stop_reason="success",
1333
- enable_extraction=enable_extraction,
1334
- enable_markdown=enable_markdown,
1335
- enable_maincontent=enable_maincontent,
1336
- enable_ocr=enable_ocr)
1333
+ enable_extraction=enable_extraction,
1334
+ enable_markdown=enable_markdown,
1335
+ enable_maincontent=enable_maincontent,
1336
+ enable_ocr=enable_ocr)
1337
1337
  if att.verdict == Verdict.SUSPECT_OK.value and best_suspect is None:
1338
1338
  best_suspect = (resp, att)
1339
1339
  if att.verdict in _TERMINAL_NONSUCCESS_VALUES:
@@ -1356,8 +1356,8 @@ def _fetch_core(
1356
1356
  try:
1357
1357
  from .executor import run_playwright_fallback
1358
1358
  fb_profile = load_profile(profile_used or "unknown_challenge", profiles=profiles)
1359
- fb_order = fb_profile.get("fallback_when_challenge") or [
1360
- "scrapling", "stealth_firefox", "playwright_real_chrome"]
1359
+ fb_order = fb_profile.get("fallback_when_challenge") or [
1360
+ "scrapling", "stealth_firefox", "playwright_real_chrome"]
1361
1361
  for fb_name in fb_order:
1362
1362
  if fb_name == "curl_grid_exhaust":
1363
1363
  continue
@@ -1374,13 +1374,13 @@ def _fetch_core(
1374
1374
  break
1375
1375
  pw_attempt, pw_content = run_playwright_fallback(
1376
1376
  url, profile_id=profile_used or "unknown_challenge",
1377
- success_selectors=success_selectors, device_class=device_class,
1378
- force_executor=fb_name, timeout=timeout if timeout and timeout > 30 else 90,
1379
- proxy=proxy, cookies=cookies,
1380
- )
1377
+ success_selectors=success_selectors, device_class=device_class,
1378
+ force_executor=fb_name, timeout=timeout if timeout and timeout > 30 else 90,
1379
+ proxy=proxy, cookies=cookies,
1380
+ )
1381
1381
  trace.append(pw_attempt)
1382
- if not is_mcp_stub and getattr(pw_attempt, "_executed", True):
1383
- browser_used += 1
1382
+ if not is_mcp_stub and getattr(pw_attempt, "_executed", True):
1383
+ browser_used += 1
1384
1384
  if pw_attempt.verdict in _OK_VALUES:
1385
1385
  # Render-merge: the executor stashes the rendered innerText
1386
1386
  # on the attempt; the rescue gate keeps whichever of
@@ -1388,10 +1388,10 @@ def _fetch_core(
1388
1388
  pw_inner = getattr(pw_attempt, "_inner_text", "") or ""
1389
1389
  _t, pw_out, pw_q, pw_meta = _maybe_extract(
1390
1390
  _PWResp(pw_content, pw_attempt.url), pw_attempt.url,
1391
- enable_extraction=enable_extraction, inner_text=pw_inner,
1392
- enable_markdown=enable_markdown,
1393
- enable_maincontent=enable_maincontent,
1394
- enable_ocr=enable_ocr)
1391
+ enable_extraction=enable_extraction, inner_text=pw_inner,
1392
+ enable_markdown=enable_markdown,
1393
+ enable_maincontent=enable_maincontent,
1394
+ enable_ocr=enable_ocr)
1395
1395
  return FetchResult(
1396
1396
  ok=True, content=pw_out, final_url=pw_attempt.url,
1397
1397
  verdict=pw_attempt.verdict, profile_used=profile_used,
@@ -1459,9 +1459,9 @@ def _untried_routes(stop_reason, grid_exhausted) -> tuple[list[str], bool]:
1459
1459
  return routes, must_mcp
1460
1460
 
1461
1461
 
1462
- _REAL_EXECUTORS = frozenset({
1463
- "curl_cffi", "playwright_real_chrome", "playwright_mobile_chrome",
1464
- "scrapling:stealthy_fetcher", "stealth_firefox:invisible_playwright"})
1462
+ _REAL_EXECUTORS = frozenset({
1463
+ "curl_cffi", "playwright_real_chrome", "playwright_mobile_chrome",
1464
+ "scrapling:stealthy_fetcher", "stealth_firefox:invisible_playwright"})
1465
1465
  _INFRA_AUTH_VERDICTS = frozenset({Verdict.AUTH_REQUIRED.value, Verdict.NOT_FOUND.value})
1466
1466
  _WAF_VERDICTS = frozenset({
1467
1467
  Verdict.CHALLENGE.value, Verdict.BLOCKED.value,
@@ -1548,16 +1548,16 @@ def fetch_many(urls: list[str], **kwargs) -> list[FetchResult]:
1548
1548
  return [r for r in results if r is not None]
1549
1549
 
1550
1550
 
1551
- def _build_result(resp, attempt: Attempt, trace: list[Attempt], profile_used: Optional[str],
1552
- *, planned: int, executed: int, grid_exhausted: bool, stop_reason: str,
1553
- enable_extraction: bool = True, enable_markdown: bool = True,
1554
- enable_maincontent: bool = False,
1555
- enable_ocr: bool = False) -> FetchResult:
1551
+ def _build_result(resp, attempt: Attempt, trace: list[Attempt], profile_used: Optional[str],
1552
+ *, planned: int, executed: int, grid_exhausted: bool, stop_reason: str,
1553
+ enable_extraction: bool = True, enable_markdown: bool = True,
1554
+ enable_maincontent: bool = False,
1555
+ enable_ocr: bool = False) -> FetchResult:
1556
1556
  final_url = str(getattr(resp, "url", attempt.url))
1557
- _t, content, quality, meta = _maybe_extract(
1558
- resp, final_url, enable_extraction=enable_extraction,
1559
- enable_markdown=enable_markdown, enable_maincontent=enable_maincontent,
1560
- enable_ocr=enable_ocr)
1557
+ _t, content, quality, meta = _maybe_extract(
1558
+ resp, final_url, enable_extraction=enable_extraction,
1559
+ enable_markdown=enable_markdown, enable_maincontent=enable_maincontent,
1560
+ enable_ocr=enable_ocr)
1561
1561
  return FetchResult(
1562
1562
  ok=True,
1563
1563
  content=content,