devez-vibe 1.10.21 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +73 -73
- package/bin/dvz.exe +0 -0
- package/bin/dvz.js +35 -35
- package/bridge/claude-agent-sdk-bridge.mjs +364 -35
- package/install-skills.mjs +11 -11
- package/package.json +50 -50
- package/prepare-search-deps.mjs +42 -42
- package/skills/insane-search/SKILL.md +36 -36
- package/skills/insane-search/engine/__main__.py +44 -44
- package/skills/insane-search/engine/executor.py +246 -246
- package/skills/insane-search/engine/fetch_chain.py +443 -443
- package/skills/insane-search/engine/network.py +108 -108
- package/skills/insane-search/engine/phase0.py +380 -380
- package/skills/insane-search/engine/session_input.py +120 -120
- package/skills/insane-search/engine/templates/invisible_playwright_fetch.py +57 -57
- package/skills/insane-search/engine/templates/nodriver_fetch.py +9 -9
- package/skills/insane-search/engine/templates/patchright_fetch.py +9 -9
- package/skills/insane-search/engine/templates/playwright_mobile_chrome.js +9 -9
- package/skills/insane-search/engine/templates/playwright_real_chrome.js +7 -7
- package/skills/insane-search/engine/templates/scrapling_fetch.py +46 -46
- package/skills/insane-search/engine/tests/test_smoke.py +8 -8
- package/skills/insane-search/engine/tests/test_t10_artifact_routes.py +126 -126
- package/skills/insane-search/engine/tests/test_t11_session_handoff.py +157 -157
- package/skills/insane-search/engine/tests/test_t5_pdfplumber.py +11 -11
- package/skills/insane-search/engine/tests/test_t7_browser_gate.py +3 -3
- package/skills/insane-search/engine/tests/test_t8_network_routes.py +207 -207
- package/skills/insane-search/engine/tests/test_t9_evidence_bundle.py +89 -89
- package/skills/insane-search/engine/tests/test_u5.py +9 -9
- package/skills/insane-search/engine/transport.py +94 -94
- package/skills/insane-search/engine/waf_profiles.yaml +38 -38
- package/skills/insane-search/references/cache-archive.md +13 -13
- package/skills/insane-search/references/media.md +14 -14
- package/skills/insane-search/references/playwright.md +27 -27
- package/skills/insane-search/references/research-evidence.md +36 -36
- package/skills/insane-search/references/session-handoff.md +26 -26
- package/skills/insane-search/tests/test_x_search.py +3 -3
- package/skills/luna-loop/agents/openai.yaml +7 -7
|
@@ -31,16 +31,16 @@ No site-specific branching. Site knowledge enters only via:
|
|
|
31
31
|
"""
|
|
32
32
|
from __future__ import annotations
|
|
33
33
|
|
|
34
|
-
import os
|
|
35
|
-
import random
|
|
36
|
-
import time
|
|
37
|
-
import hashlib
|
|
38
|
-
from dataclasses import dataclass, field, asdict
|
|
39
|
-
from datetime import datetime, timezone
|
|
40
|
-
from typing import Any, Optional
|
|
41
|
-
|
|
42
|
-
from .content_safety import ContentSafetyReport, analyze_untrusted_content, wrap_untrusted_content
|
|
43
|
-
from .network import diagnose_trace, normalize_proxy, redact_proxy
|
|
34
|
+
import os
|
|
35
|
+
import random
|
|
36
|
+
import time
|
|
37
|
+
import hashlib
|
|
38
|
+
from dataclasses import dataclass, field, asdict
|
|
39
|
+
from datetime import datetime, timezone
|
|
40
|
+
from typing import Any, Optional
|
|
41
|
+
|
|
42
|
+
from .content_safety import ContentSafetyReport, analyze_untrusted_content, wrap_untrusted_content
|
|
43
|
+
from .network import diagnose_trace, normalize_proxy, redact_proxy
|
|
44
44
|
from .validators import Verdict, validate, TERMINAL_NONSUCCESS
|
|
45
45
|
from .waf_detector import detect, load_profile, _load_profiles, last_load_error
|
|
46
46
|
from .url_transforms import iter_transformed
|
|
@@ -130,21 +130,21 @@ class FetchResult:
|
|
|
130
130
|
# "bot_detection" = routes disagree or a WAF/challenge signal → bypassable
|
|
131
131
|
# (browser / more routes may help); "infra_or_auth" = every route uniformly
|
|
132
132
|
# 401/404 → a real wall stealth cannot clear.
|
|
133
|
-
block_class: str = ""
|
|
134
|
-
proxy_used: bool = False
|
|
135
|
-
network_diagnosis: dict = field(default_factory=dict)
|
|
136
|
-
session_seeded: bool = False
|
|
137
|
-
retrieved_at: str = ""
|
|
138
|
-
content_sha256: str = ""
|
|
139
|
-
|
|
140
|
-
def __post_init__(self) -> None:
|
|
141
|
-
if not self.retrieved_at:
|
|
142
|
-
self.retrieved_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
143
|
-
if not self.content_sha256:
|
|
144
|
-
self.content_sha256 = hashlib.sha256(
|
|
145
|
-
self.content.encode("utf-8", "surrogatepass")
|
|
146
|
-
).hexdigest()
|
|
147
|
-
report = analyze_untrusted_content(self.content, source_url=self.final_url)
|
|
133
|
+
block_class: str = ""
|
|
134
|
+
proxy_used: bool = False
|
|
135
|
+
network_diagnosis: dict = field(default_factory=dict)
|
|
136
|
+
session_seeded: bool = False
|
|
137
|
+
retrieved_at: str = ""
|
|
138
|
+
content_sha256: str = ""
|
|
139
|
+
|
|
140
|
+
def __post_init__(self) -> None:
|
|
141
|
+
if not self.retrieved_at:
|
|
142
|
+
self.retrieved_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
143
|
+
if not self.content_sha256:
|
|
144
|
+
self.content_sha256 = hashlib.sha256(
|
|
145
|
+
self.content.encode("utf-8", "surrogatepass")
|
|
146
|
+
).hexdigest()
|
|
147
|
+
report = analyze_untrusted_content(self.content, source_url=self.final_url)
|
|
148
148
|
if not self.content_trust:
|
|
149
149
|
self.content_trust = report.content_trust
|
|
150
150
|
if not self.prompt_injection_risk:
|
|
@@ -154,7 +154,7 @@ class FetchResult:
|
|
|
154
154
|
if not self.untrusted_content_boundary:
|
|
155
155
|
self.untrusted_content_boundary = dict(report.untrusted_content_boundary)
|
|
156
156
|
|
|
157
|
-
def to_untrusted_text(self) -> str:
|
|
157
|
+
def to_untrusted_text(self) -> str:
|
|
158
158
|
report = ContentSafetyReport(
|
|
159
159
|
content_trust=self.content_trust,
|
|
160
160
|
prompt_injection_risk=self.prompt_injection_risk,
|
|
@@ -164,65 +164,65 @@ class FetchResult:
|
|
|
164
164
|
"end": self.untrusted_content_boundary["end"],
|
|
165
165
|
},
|
|
166
166
|
)
|
|
167
|
-
return wrap_untrusted_content(self.content, report=report, source_url=self.final_url)
|
|
168
|
-
|
|
169
|
-
def evidence_record(self) -> dict[str, Any]:
|
|
170
|
-
"""Return provenance metadata without exposing fetched content or secrets."""
|
|
171
|
-
from .url_masking import mask_url
|
|
172
|
-
|
|
173
|
-
requested_url = self.trace[0].url if self.trace else self.final_url
|
|
174
|
-
source = self.extraction_source or "raw"
|
|
175
|
-
if not self.ok:
|
|
176
|
-
source_kind = "unknown"
|
|
177
|
-
currentness = "unavailable"
|
|
178
|
-
elif source.startswith("archive+"):
|
|
179
|
-
source_kind = "archive"
|
|
180
|
-
currentness = "historical_snapshot"
|
|
181
|
-
elif source.startswith("pdf"):
|
|
182
|
-
source_kind = "document"
|
|
183
|
-
currentness = "current"
|
|
184
|
-
elif (self.profile_used or "").startswith(("phase0:youtube", "phase0:media", "phase0:threads")):
|
|
185
|
-
source_kind = "media"
|
|
186
|
-
currentness = "current"
|
|
187
|
-
else:
|
|
188
|
-
source_kind = "web"
|
|
189
|
-
currentness = "current"
|
|
190
|
-
winning_attempt = next(
|
|
191
|
-
(attempt for attempt in reversed(self.trace) if attempt.verdict in _OK_VALUES),
|
|
192
|
-
self.trace[-1] if self.trace else None,
|
|
193
|
-
)
|
|
194
|
-
access_method = winning_attempt.executor if winning_attempt else "unknown"
|
|
195
|
-
return {
|
|
196
|
-
"requested_url": mask_url(requested_url),
|
|
197
|
-
"final_url": mask_url(self.final_url),
|
|
198
|
-
"retrieved_at": self.retrieved_at,
|
|
199
|
-
"retrieval_status": "succeeded" if self.ok else "failed",
|
|
200
|
-
"source_timestamp": self.extraction_meta.get("snapshot_timestamp", ""),
|
|
201
|
-
"source_kind": source_kind,
|
|
202
|
-
"currentness": currentness,
|
|
203
|
-
"access_method": access_method,
|
|
204
|
-
"source_route": self.profile_used or "",
|
|
205
|
-
"content_sha256": self.content_sha256,
|
|
206
|
-
"content_length": len(self.content),
|
|
207
|
-
"extraction_source": source,
|
|
208
|
-
"extraction_quality": self.extraction_quality,
|
|
209
|
-
"content_trust": self.content_trust,
|
|
210
|
-
"prompt_injection_risk": self.prompt_injection_risk,
|
|
211
|
-
"session_seeded": self.session_seeded,
|
|
212
|
-
"proxy_used": self.proxy_used,
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
def to_evidence_bundle(self) -> dict[str, Any]:
|
|
216
|
-
"""Pair compact provenance with boundary-wrapped, untrusted content."""
|
|
217
|
-
operational = self.to_dict()
|
|
218
|
-
operational.pop("evidence", None)
|
|
219
|
-
operational.pop("retrieved_at", None)
|
|
220
|
-
operational.pop("content_sha256", None)
|
|
221
|
-
return {
|
|
222
|
-
"evidence": self.evidence_record(),
|
|
223
|
-
"result": operational,
|
|
224
|
-
"untrusted_content": self.to_untrusted_text(),
|
|
225
|
-
}
|
|
167
|
+
return wrap_untrusted_content(self.content, report=report, source_url=self.final_url)
|
|
168
|
+
|
|
169
|
+
def evidence_record(self) -> dict[str, Any]:
|
|
170
|
+
"""Return provenance metadata without exposing fetched content or secrets."""
|
|
171
|
+
from .url_masking import mask_url
|
|
172
|
+
|
|
173
|
+
requested_url = self.trace[0].url if self.trace else self.final_url
|
|
174
|
+
source = self.extraction_source or "raw"
|
|
175
|
+
if not self.ok:
|
|
176
|
+
source_kind = "unknown"
|
|
177
|
+
currentness = "unavailable"
|
|
178
|
+
elif source.startswith("archive+"):
|
|
179
|
+
source_kind = "archive"
|
|
180
|
+
currentness = "historical_snapshot"
|
|
181
|
+
elif source.startswith("pdf"):
|
|
182
|
+
source_kind = "document"
|
|
183
|
+
currentness = "current"
|
|
184
|
+
elif (self.profile_used or "").startswith(("phase0:youtube", "phase0:media", "phase0:threads")):
|
|
185
|
+
source_kind = "media"
|
|
186
|
+
currentness = "current"
|
|
187
|
+
else:
|
|
188
|
+
source_kind = "web"
|
|
189
|
+
currentness = "current"
|
|
190
|
+
winning_attempt = next(
|
|
191
|
+
(attempt for attempt in reversed(self.trace) if attempt.verdict in _OK_VALUES),
|
|
192
|
+
self.trace[-1] if self.trace else None,
|
|
193
|
+
)
|
|
194
|
+
access_method = winning_attempt.executor if winning_attempt else "unknown"
|
|
195
|
+
return {
|
|
196
|
+
"requested_url": mask_url(requested_url),
|
|
197
|
+
"final_url": mask_url(self.final_url),
|
|
198
|
+
"retrieved_at": self.retrieved_at,
|
|
199
|
+
"retrieval_status": "succeeded" if self.ok else "failed",
|
|
200
|
+
"source_timestamp": self.extraction_meta.get("snapshot_timestamp", ""),
|
|
201
|
+
"source_kind": source_kind,
|
|
202
|
+
"currentness": currentness,
|
|
203
|
+
"access_method": access_method,
|
|
204
|
+
"source_route": self.profile_used or "",
|
|
205
|
+
"content_sha256": self.content_sha256,
|
|
206
|
+
"content_length": len(self.content),
|
|
207
|
+
"extraction_source": source,
|
|
208
|
+
"extraction_quality": self.extraction_quality,
|
|
209
|
+
"content_trust": self.content_trust,
|
|
210
|
+
"prompt_injection_risk": self.prompt_injection_risk,
|
|
211
|
+
"session_seeded": self.session_seeded,
|
|
212
|
+
"proxy_used": self.proxy_used,
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
def to_evidence_bundle(self) -> dict[str, Any]:
|
|
216
|
+
"""Pair compact provenance with boundary-wrapped, untrusted content."""
|
|
217
|
+
operational = self.to_dict()
|
|
218
|
+
operational.pop("evidence", None)
|
|
219
|
+
operational.pop("retrieved_at", None)
|
|
220
|
+
operational.pop("content_sha256", None)
|
|
221
|
+
return {
|
|
222
|
+
"evidence": self.evidence_record(),
|
|
223
|
+
"result": operational,
|
|
224
|
+
"untrusted_content": self.to_untrusted_text(),
|
|
225
|
+
}
|
|
226
226
|
|
|
227
227
|
def to_dict(self) -> dict:
|
|
228
228
|
return {
|
|
@@ -246,14 +246,14 @@ class FetchResult:
|
|
|
246
246
|
"extraction_quality": self.extraction_quality,
|
|
247
247
|
"extraction_source": self.extraction_source,
|
|
248
248
|
"extraction_meta": self.extraction_meta,
|
|
249
|
-
"block_class": self.block_class,
|
|
250
|
-
"proxy_used": self.proxy_used,
|
|
251
|
-
"network_diagnosis": self.network_diagnosis,
|
|
252
|
-
"session_seeded": self.session_seeded,
|
|
253
|
-
"retrieved_at": self.retrieved_at,
|
|
254
|
-
"content_sha256": self.content_sha256,
|
|
255
|
-
"evidence": self.evidence_record(),
|
|
256
|
-
}
|
|
249
|
+
"block_class": self.block_class,
|
|
250
|
+
"proxy_used": self.proxy_used,
|
|
251
|
+
"network_diagnosis": self.network_diagnosis,
|
|
252
|
+
"session_seeded": self.session_seeded,
|
|
253
|
+
"retrieved_at": self.retrieved_at,
|
|
254
|
+
"content_sha256": self.content_sha256,
|
|
255
|
+
"evidence": self.evidence_record(),
|
|
256
|
+
}
|
|
257
257
|
|
|
258
258
|
|
|
259
259
|
# --- Content rescue extraction -----------------------------------------------
|
|
@@ -266,13 +266,13 @@ class FetchResult:
|
|
|
266
266
|
# * SPA shells whose visible text is thinner than their JSON-LD articleBody
|
|
267
267
|
# * Playwright render-merge: rendered innerText wins over a thinner body
|
|
268
268
|
# pypdf is an optional import; every path degrades to the raw text.
|
|
269
|
-
import io as _io
|
|
270
|
-
import json as _json
|
|
271
|
-
import re as _re
|
|
272
|
-
import shutil as _shutil
|
|
273
|
-
import subprocess as _subprocess
|
|
274
|
-
import tempfile as _tempfile
|
|
275
|
-
from pathlib import Path as _Path
|
|
269
|
+
import io as _io
|
|
270
|
+
import json as _json
|
|
271
|
+
import re as _re
|
|
272
|
+
import shutil as _shutil
|
|
273
|
+
import subprocess as _subprocess
|
|
274
|
+
import tempfile as _tempfile
|
|
275
|
+
from pathlib import Path as _Path
|
|
276
276
|
|
|
277
277
|
try:
|
|
278
278
|
from pypdf import PdfReader as _PdfReader
|
|
@@ -318,11 +318,11 @@ _SCAN_LIMIT = 2_000_000 # chars of body the rescue regexes may scan
|
|
|
318
318
|
_JSONLD_MAX_BLOCKS = 10 # ld+json blocks parsed per page
|
|
319
319
|
_JSONLD_MAX_BLOB = 200_000 # chars of a single ld+json blob given to json.loads
|
|
320
320
|
_RESCUE_MAX_TEXT = 1_000_000 # chars any rescue path may return as content
|
|
321
|
-
_PDF_MAX_BYTES = 25 * 1024 * 1024 # PDF bodies above this are never parsed
|
|
322
|
-
_INNER_TEXT_MAX = 1_000_000 # chars of Playwright innerText accepted
|
|
323
|
-
_PDF_TABLE_MAX_CHARS = 200_000
|
|
324
|
-
_OCR_MAX_PAGES = 20
|
|
325
|
-
_OCR_TOTAL_TIMEOUT = 120.0
|
|
321
|
+
_PDF_MAX_BYTES = 25 * 1024 * 1024 # PDF bodies above this are never parsed
|
|
322
|
+
_INNER_TEXT_MAX = 1_000_000 # chars of Playwright innerText accepted
|
|
323
|
+
_PDF_TABLE_MAX_CHARS = 200_000
|
|
324
|
+
_OCR_MAX_PAGES = 20
|
|
325
|
+
_OCR_TOTAL_TIMEOUT = 120.0
|
|
326
326
|
|
|
327
327
|
|
|
328
328
|
def _quality_score(md: str) -> float:
|
|
@@ -344,7 +344,7 @@ def _visible_text(html: str) -> str:
|
|
|
344
344
|
return _re.sub(r"\s+", " ", t).strip()
|
|
345
345
|
|
|
346
346
|
|
|
347
|
-
def _extract_json_ld_text(html: str) -> str:
|
|
347
|
+
def _extract_json_ld_text(html: str) -> str:
|
|
348
348
|
"""Pull articleBody / description from <script type=application/ld+json>.
|
|
349
349
|
|
|
350
350
|
Bounded: at most _JSONLD_MAX_BLOCKS blocks are parsed, a blob larger than
|
|
@@ -379,34 +379,34 @@ def _extract_json_ld_text(html: str) -> str:
|
|
|
379
379
|
break
|
|
380
380
|
except Exception:
|
|
381
381
|
pass
|
|
382
|
-
return "\n\n".join(out)
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
def _pdf_table_to_markdown(table: list[list[Any]]) -> str:
|
|
386
|
-
"""Convert one bounded pdfplumber table to Markdown without losing units."""
|
|
387
|
-
if not table:
|
|
388
|
-
return ""
|
|
389
|
-
rows: list[list[str]] = []
|
|
390
|
-
width = min(max((len(row or []) for row in table), default=0), 20)
|
|
391
|
-
if width == 0:
|
|
392
|
-
return ""
|
|
393
|
-
for raw_row in table[:100]:
|
|
394
|
-
raw_row = raw_row or []
|
|
395
|
-
row = []
|
|
396
|
-
for cell in list(raw_row[:width]) + [""] * max(0, width - len(raw_row)):
|
|
397
|
-
value = _re.sub(r"\s+", " ", str(cell or "")).strip().replace("|", r"\|")
|
|
398
|
-
row.append(value)
|
|
399
|
-
rows.append(row)
|
|
400
|
-
if not any(any(cell for cell in row) for row in rows):
|
|
401
|
-
return ""
|
|
402
|
-
header = rows[0]
|
|
403
|
-
lines = ["| " + " | ".join(header) + " |",
|
|
404
|
-
"| " + " | ".join("---" for _ in header) + " |"]
|
|
405
|
-
lines.extend("| " + " | ".join(row) + " |" for row in rows[1:])
|
|
406
|
-
return "\n".join(lines)
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
def _extract_pdf_pdfplumber(body: bytes) -> tuple[str, str]:
|
|
382
|
+
return "\n\n".join(out)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _pdf_table_to_markdown(table: list[list[Any]]) -> str:
|
|
386
|
+
"""Convert one bounded pdfplumber table to Markdown without losing units."""
|
|
387
|
+
if not table:
|
|
388
|
+
return ""
|
|
389
|
+
rows: list[list[str]] = []
|
|
390
|
+
width = min(max((len(row or []) for row in table), default=0), 20)
|
|
391
|
+
if width == 0:
|
|
392
|
+
return ""
|
|
393
|
+
for raw_row in table[:100]:
|
|
394
|
+
raw_row = raw_row or []
|
|
395
|
+
row = []
|
|
396
|
+
for cell in list(raw_row[:width]) + [""] * max(0, width - len(raw_row)):
|
|
397
|
+
value = _re.sub(r"\s+", " ", str(cell or "")).strip().replace("|", r"\|")
|
|
398
|
+
row.append(value)
|
|
399
|
+
rows.append(row)
|
|
400
|
+
if not any(any(cell for cell in row) for row in rows):
|
|
401
|
+
return ""
|
|
402
|
+
header = rows[0]
|
|
403
|
+
lines = ["| " + " | ".join(header) + " |",
|
|
404
|
+
"| " + " | ".join("---" for _ in header) + " |"]
|
|
405
|
+
lines.extend("| " + " | ".join(row) + " |" for row in rows[1:])
|
|
406
|
+
return "\n".join(lines)
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _extract_pdf_pdfplumber(body: bytes) -> tuple[str, str]:
|
|
410
410
|
"""(title, text) via pdfplumber, or ("", "") on failure / no text layer.
|
|
411
411
|
Same bounds as the pypdf path: ≤80 pages, text capped at _RESCUE_MAX_TEXT."""
|
|
412
412
|
if _pdfplumber is None:
|
|
@@ -420,37 +420,37 @@ def _extract_pdf_pdfplumber(body: bytes) -> tuple[str, str]:
|
|
|
420
420
|
title = str(md["Title"])[:300]
|
|
421
421
|
except Exception:
|
|
422
422
|
pass
|
|
423
|
-
parts: list[str] = []
|
|
424
|
-
total = 0
|
|
425
|
-
for page in pdf.pages[:80]:
|
|
423
|
+
parts: list[str] = []
|
|
424
|
+
total = 0
|
|
425
|
+
for page in pdf.pages[:80]:
|
|
426
426
|
try:
|
|
427
427
|
t = page.extract_text() or ""
|
|
428
428
|
except Exception:
|
|
429
429
|
t = ""
|
|
430
430
|
take = t[:_RESCUE_MAX_TEXT - total]
|
|
431
|
-
parts.append(take)
|
|
432
|
-
total += len(take)
|
|
433
|
-
if total < _RESCUE_MAX_TEXT:
|
|
434
|
-
try:
|
|
435
|
-
tables = page.extract_tables() or []
|
|
436
|
-
except Exception:
|
|
437
|
-
tables = []
|
|
438
|
-
for table in tables[:20]:
|
|
439
|
-
markdown = _pdf_table_to_markdown(table)
|
|
440
|
-
if not markdown:
|
|
441
|
-
continue
|
|
442
|
-
take_table = markdown[:min(
|
|
443
|
-
_PDF_TABLE_MAX_CHARS,
|
|
444
|
-
_RESCUE_MAX_TEXT - total,
|
|
445
|
-
)]
|
|
446
|
-
if take_table:
|
|
447
|
-
parts.append("[PDF table]\n" + take_table)
|
|
448
|
-
total += len(take_table)
|
|
449
|
-
if total >= _RESCUE_MAX_TEXT:
|
|
450
|
-
break
|
|
451
|
-
if total >= _RESCUE_MAX_TEXT:
|
|
452
|
-
break
|
|
453
|
-
return title, "\n\n".join(p for p in parts if p).strip()
|
|
431
|
+
parts.append(take)
|
|
432
|
+
total += len(take)
|
|
433
|
+
if total < _RESCUE_MAX_TEXT:
|
|
434
|
+
try:
|
|
435
|
+
tables = page.extract_tables() or []
|
|
436
|
+
except Exception:
|
|
437
|
+
tables = []
|
|
438
|
+
for table in tables[:20]:
|
|
439
|
+
markdown = _pdf_table_to_markdown(table)
|
|
440
|
+
if not markdown:
|
|
441
|
+
continue
|
|
442
|
+
take_table = markdown[:min(
|
|
443
|
+
_PDF_TABLE_MAX_CHARS,
|
|
444
|
+
_RESCUE_MAX_TEXT - total,
|
|
445
|
+
)]
|
|
446
|
+
if take_table:
|
|
447
|
+
parts.append("[PDF table]\n" + take_table)
|
|
448
|
+
total += len(take_table)
|
|
449
|
+
if total >= _RESCUE_MAX_TEXT:
|
|
450
|
+
break
|
|
451
|
+
if total >= _RESCUE_MAX_TEXT:
|
|
452
|
+
break
|
|
453
|
+
return title, "\n\n".join(p for p in parts if p).strip()
|
|
454
454
|
except Exception:
|
|
455
455
|
return "", ""
|
|
456
456
|
|
|
@@ -484,66 +484,66 @@ def _extract_pdf_pypdf(body: bytes) -> tuple[str, str, str]:
|
|
|
484
484
|
return "", "", f"pdf_error:{type(e).__name__}"
|
|
485
485
|
|
|
486
486
|
|
|
487
|
-
def _extract_pdf_ocr(body: bytes) -> tuple[str, str]:
|
|
488
|
-
"""OCR a scanned PDF with local Poppler and Tesseract when explicitly enabled."""
|
|
489
|
-
pdftoppm = _shutil.which("pdftoppm")
|
|
490
|
-
tesseract = _shutil.which("tesseract")
|
|
491
|
-
if not pdftoppm or not tesseract:
|
|
492
|
-
return "", "pdf_ocr_tools_missing"
|
|
493
|
-
started = time.monotonic()
|
|
494
|
-
try:
|
|
495
|
-
with _tempfile.TemporaryDirectory(prefix="insane-pdf-ocr-") as root:
|
|
496
|
-
root_path = _Path(root)
|
|
497
|
-
source = root_path / "source.pdf"
|
|
498
|
-
source.write_bytes(body)
|
|
499
|
-
rendered = _subprocess.run(
|
|
500
|
-
[pdftoppm, "-f", "1", "-l", str(_OCR_MAX_PAGES), "-r", "150",
|
|
501
|
-
"-png", str(source), str(root_path / "page")],
|
|
502
|
-
capture_output=True, timeout=60, check=False,
|
|
503
|
-
)
|
|
504
|
-
if rendered.returncode != 0:
|
|
505
|
-
return "", "pdf_ocr_render_failed"
|
|
506
|
-
|
|
507
|
-
language = "eng"
|
|
508
|
-
try:
|
|
509
|
-
listed = _subprocess.run(
|
|
510
|
-
[tesseract, "--list-langs"], capture_output=True, text=True,
|
|
511
|
-
encoding="utf-8", errors="replace", timeout=10, check=False)
|
|
512
|
-
languages = set((listed.stdout or "").split())
|
|
513
|
-
if "kor" in languages:
|
|
514
|
-
language = "kor+eng" if "eng" in languages else "kor"
|
|
515
|
-
except Exception:
|
|
516
|
-
pass
|
|
517
|
-
|
|
518
|
-
parts: list[str] = []
|
|
519
|
-
total = 0
|
|
520
|
-
for image in sorted(root_path.glob("page-*.png"))[:_OCR_MAX_PAGES]:
|
|
521
|
-
remaining = _OCR_TOTAL_TIMEOUT - (time.monotonic() - started)
|
|
522
|
-
if remaining <= 1:
|
|
523
|
-
break
|
|
524
|
-
completed = _subprocess.run(
|
|
525
|
-
[tesseract, str(image), "stdout", "-l", language],
|
|
526
|
-
capture_output=True, text=True, encoding="utf-8", errors="replace",
|
|
527
|
-
timeout=min(20, remaining), check=False,
|
|
528
|
-
)
|
|
529
|
-
if completed.returncode != 0:
|
|
530
|
-
continue
|
|
531
|
-
text = (completed.stdout or "").strip()
|
|
532
|
-
take = text[:_RESCUE_MAX_TEXT - total]
|
|
533
|
-
if take:
|
|
534
|
-
parts.append(take)
|
|
535
|
-
total += len(take)
|
|
536
|
-
if total >= _RESCUE_MAX_TEXT:
|
|
537
|
-
break
|
|
538
|
-
output = "\n\n".join(parts).strip()
|
|
539
|
-
return (output, "") if output else ("", "pdf_ocr_no_text")
|
|
540
|
-
except _subprocess.TimeoutExpired:
|
|
541
|
-
return "", "pdf_ocr_timeout"
|
|
542
|
-
except Exception as exc:
|
|
543
|
-
return "", f"pdf_ocr_error:{type(exc).__name__}"
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
def _extract_pdf(body: bytes, url: str, enable_ocr: bool = False) -> tuple[str, str, float, str]:
|
|
487
|
+
def _extract_pdf_ocr(body: bytes) -> tuple[str, str]:
|
|
488
|
+
"""OCR a scanned PDF with local Poppler and Tesseract when explicitly enabled."""
|
|
489
|
+
pdftoppm = _shutil.which("pdftoppm")
|
|
490
|
+
tesseract = _shutil.which("tesseract")
|
|
491
|
+
if not pdftoppm or not tesseract:
|
|
492
|
+
return "", "pdf_ocr_tools_missing"
|
|
493
|
+
started = time.monotonic()
|
|
494
|
+
try:
|
|
495
|
+
with _tempfile.TemporaryDirectory(prefix="insane-pdf-ocr-") as root:
|
|
496
|
+
root_path = _Path(root)
|
|
497
|
+
source = root_path / "source.pdf"
|
|
498
|
+
source.write_bytes(body)
|
|
499
|
+
rendered = _subprocess.run(
|
|
500
|
+
[pdftoppm, "-f", "1", "-l", str(_OCR_MAX_PAGES), "-r", "150",
|
|
501
|
+
"-png", str(source), str(root_path / "page")],
|
|
502
|
+
capture_output=True, timeout=60, check=False,
|
|
503
|
+
)
|
|
504
|
+
if rendered.returncode != 0:
|
|
505
|
+
return "", "pdf_ocr_render_failed"
|
|
506
|
+
|
|
507
|
+
language = "eng"
|
|
508
|
+
try:
|
|
509
|
+
listed = _subprocess.run(
|
|
510
|
+
[tesseract, "--list-langs"], capture_output=True, text=True,
|
|
511
|
+
encoding="utf-8", errors="replace", timeout=10, check=False)
|
|
512
|
+
languages = set((listed.stdout or "").split())
|
|
513
|
+
if "kor" in languages:
|
|
514
|
+
language = "kor+eng" if "eng" in languages else "kor"
|
|
515
|
+
except Exception:
|
|
516
|
+
pass
|
|
517
|
+
|
|
518
|
+
parts: list[str] = []
|
|
519
|
+
total = 0
|
|
520
|
+
for image in sorted(root_path.glob("page-*.png"))[:_OCR_MAX_PAGES]:
|
|
521
|
+
remaining = _OCR_TOTAL_TIMEOUT - (time.monotonic() - started)
|
|
522
|
+
if remaining <= 1:
|
|
523
|
+
break
|
|
524
|
+
completed = _subprocess.run(
|
|
525
|
+
[tesseract, str(image), "stdout", "-l", language],
|
|
526
|
+
capture_output=True, text=True, encoding="utf-8", errors="replace",
|
|
527
|
+
timeout=min(20, remaining), check=False,
|
|
528
|
+
)
|
|
529
|
+
if completed.returncode != 0:
|
|
530
|
+
continue
|
|
531
|
+
text = (completed.stdout or "").strip()
|
|
532
|
+
take = text[:_RESCUE_MAX_TEXT - total]
|
|
533
|
+
if take:
|
|
534
|
+
parts.append(take)
|
|
535
|
+
total += len(take)
|
|
536
|
+
if total >= _RESCUE_MAX_TEXT:
|
|
537
|
+
break
|
|
538
|
+
output = "\n\n".join(parts).strip()
|
|
539
|
+
return (output, "") if output else ("", "pdf_ocr_no_text")
|
|
540
|
+
except _subprocess.TimeoutExpired:
|
|
541
|
+
return "", "pdf_ocr_timeout"
|
|
542
|
+
except Exception as exc:
|
|
543
|
+
return "", f"pdf_ocr_error:{type(exc).__name__}"
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def _extract_pdf(body: bytes, url: str, enable_ocr: bool = False) -> tuple[str, str, float, str]:
|
|
547
547
|
"""Returns (title, text, quality, error_code). error_code is "" on success.
|
|
548
548
|
Caps at 80 pages to keep token budget sane; reports pdf_no_text_layer for
|
|
549
549
|
scanned PDFs (so the caller knows rendering will not help either).
|
|
@@ -554,8 +554,8 @@ def _extract_pdf(body: bytes, url: str, enable_ocr: bool = False) -> tuple[str,
|
|
|
554
554
|
the extracted text is capped at _RESCUE_MAX_TEXT."""
|
|
555
555
|
if len(body) > _PDF_MAX_BYTES:
|
|
556
556
|
return "", "", 0.0, "pdf_too_large"
|
|
557
|
-
if _pdfplumber is None and _PdfReader is None and not enable_ocr:
|
|
558
|
-
return "", "", 0.0, "pdf_no_extractor"
|
|
557
|
+
if _pdfplumber is None and _PdfReader is None and not enable_ocr:
|
|
558
|
+
return "", "", 0.0, "pdf_no_extractor"
|
|
559
559
|
|
|
560
560
|
# 1) pdfplumber (preferred). Only adopt when it yields text.
|
|
561
561
|
p_title, p_text = _extract_pdf_pdfplumber(body)
|
|
@@ -566,14 +566,14 @@ def _extract_pdf(body: bytes, url: str, enable_ocr: bool = False) -> tuple[str,
|
|
|
566
566
|
y_title, y_text, y_err = _extract_pdf_pypdf(body)
|
|
567
567
|
if y_text:
|
|
568
568
|
return y_title, y_text, _quality_score(y_text), ""
|
|
569
|
-
if y_err and y_err != "pypdf_missing":
|
|
570
|
-
return y_title, "", 0.0, y_err
|
|
571
|
-
|
|
572
|
-
if enable_ocr:
|
|
573
|
-
ocr_text, ocr_error = _extract_pdf_ocr(body)
|
|
574
|
-
if ocr_text:
|
|
575
|
-
return (p_title or y_title), ocr_text, _quality_score(ocr_text), "pdf_ocr"
|
|
576
|
-
return (p_title or y_title), "", 0.0, ocr_error
|
|
569
|
+
if y_err and y_err != "pypdf_missing":
|
|
570
|
+
return y_title, "", 0.0, y_err
|
|
571
|
+
|
|
572
|
+
if enable_ocr:
|
|
573
|
+
ocr_text, ocr_error = _extract_pdf_ocr(body)
|
|
574
|
+
if ocr_text:
|
|
575
|
+
return (p_title or y_title), ocr_text, _quality_score(ocr_text), "pdf_ocr"
|
|
576
|
+
return (p_title or y_title), "", 0.0, ocr_error
|
|
577
577
|
|
|
578
578
|
# Neither produced text: prefer any title we found; report no text layer.
|
|
579
579
|
return (p_title or y_title), "", 0.0, "pdf_no_text_layer"
|
|
@@ -644,10 +644,10 @@ class _PWResp:
|
|
|
644
644
|
self.headers = headers or {"content-type": "text/html"}
|
|
645
645
|
|
|
646
646
|
|
|
647
|
-
def _extract_response(resp, final_url: str, inner_text: str = "",
|
|
648
|
-
enable_markdown: bool = True,
|
|
649
|
-
enable_maincontent: bool = False,
|
|
650
|
-
enable_ocr: bool = False) -> tuple[str, str, float, dict]:
|
|
647
|
+
def _extract_response(resp, final_url: str, inner_text: str = "",
|
|
648
|
+
enable_markdown: bool = True,
|
|
649
|
+
enable_maincontent: bool = False,
|
|
650
|
+
enable_ocr: bool = False) -> tuple[str, str, float, dict]:
|
|
651
651
|
"""Returns (title, content, quality, meta); meta = {source, error,
|
|
652
652
|
inner_text_used}. The raw body wins by default — see the module-block
|
|
653
653
|
comment for when a rescue path may replace it.
|
|
@@ -668,12 +668,12 @@ def _extract_response(resp, final_url: str, inner_text: str = "",
|
|
|
668
668
|
except Exception:
|
|
669
669
|
pass
|
|
670
670
|
if bytes(body[:5]) == b"%PDF-" or ctype_pdf:
|
|
671
|
-
title, text, quality, err = _extract_pdf(
|
|
672
|
-
bytes(body), final_url, enable_ocr=enable_ocr)
|
|
673
|
-
if text:
|
|
674
|
-
source = "pdf+ocr" if err == "pdf_ocr" else "pdf"
|
|
675
|
-
return title, text, quality, {"source": source, "error": "",
|
|
676
|
-
"inner_text_used": False}
|
|
671
|
+
title, text, quality, err = _extract_pdf(
|
|
672
|
+
bytes(body), final_url, enable_ocr=enable_ocr)
|
|
673
|
+
if text:
|
|
674
|
+
source = "pdf+ocr" if err == "pdf_ocr" else "pdf"
|
|
675
|
+
return title, text, quality, {"source": source, "error": "",
|
|
676
|
+
"inner_text_used": False}
|
|
677
677
|
return title, f"[PDF binary, {len(body)} bytes; extractor={err or 'ok'}]", \
|
|
678
678
|
0.0, {"source": "pdf", "error": err, "inner_text_used": False}
|
|
679
679
|
|
|
@@ -736,24 +736,24 @@ def _extract_response(resp, final_url: str, inner_text: str = "",
|
|
|
736
736
|
"inner_text_used": inner_used}
|
|
737
737
|
|
|
738
738
|
|
|
739
|
-
def _maybe_extract(resp, final_url: str, *, enable_extraction: bool,
|
|
740
|
-
inner_text: str = "", enable_markdown: bool = True,
|
|
741
|
-
enable_maincontent: bool = False,
|
|
742
|
-
enable_ocr: bool = False) -> tuple[str, str, float, dict]:
|
|
739
|
+
def _maybe_extract(resp, final_url: str, *, enable_extraction: bool,
|
|
740
|
+
inner_text: str = "", enable_markdown: bool = True,
|
|
741
|
+
enable_maincontent: bool = False,
|
|
742
|
+
enable_ocr: bool = False) -> tuple[str, str, float, dict]:
|
|
743
743
|
"""Run rescue extraction when enabled; otherwise raw text + consistent meta."""
|
|
744
744
|
if not enable_extraction:
|
|
745
745
|
return "", getattr(resp, "text", "") or "", 0.0, \
|
|
746
746
|
{"source": "raw_disabled", "error": "", "inner_text_used": False}
|
|
747
|
-
return _extract_response(resp, final_url, inner_text=inner_text,
|
|
748
|
-
enable_markdown=enable_markdown,
|
|
749
|
-
enable_maincontent=enable_maincontent,
|
|
750
|
-
enable_ocr=enable_ocr)
|
|
747
|
+
return _extract_response(resp, final_url, inner_text=inner_text,
|
|
748
|
+
enable_markdown=enable_markdown,
|
|
749
|
+
enable_maincontent=enable_maincontent,
|
|
750
|
+
enable_ocr=enable_ocr)
|
|
751
751
|
|
|
752
752
|
|
|
753
753
|
# --- curl_cffi probe executor ------------------------------------------------
|
|
754
|
-
def _curl_probe(
|
|
755
|
-
url: str, *, impersonate: str, referer: str, timeout: int = 20,
|
|
756
|
-
enable_retry: bool = False, proxy: Optional[str] = None,
|
|
754
|
+
def _curl_probe(
|
|
755
|
+
url: str, *, impersonate: str, referer: str, timeout: int = 20,
|
|
756
|
+
enable_retry: bool = False, proxy: Optional[str] = None,
|
|
757
757
|
) -> tuple[Any, Optional[str]]:
|
|
758
758
|
"""Returns (response, error_str). response may be None on exception.
|
|
759
759
|
|
|
@@ -761,11 +761,11 @@ def _curl_probe(
|
|
|
761
761
|
warm connection persist across attempts and across pages of the same host.
|
|
762
762
|
The pool degrades to a one-shot GET when a Session can't be created.
|
|
763
763
|
"""
|
|
764
|
-
from .transport import POOL
|
|
765
|
-
response, error = POOL.request(
|
|
766
|
-
url, impersonate=impersonate, referer=referer, timeout=timeout,
|
|
767
|
-
max_retries=2 if enable_retry else 0, proxy=proxy)
|
|
768
|
-
return response, redact_proxy(error, proxy)
|
|
764
|
+
from .transport import POOL
|
|
765
|
+
response, error = POOL.request(
|
|
766
|
+
url, impersonate=impersonate, referer=referer, timeout=timeout,
|
|
767
|
+
max_retries=2 if enable_retry else 0, proxy=proxy)
|
|
768
|
+
return response, redact_proxy(error, proxy)
|
|
769
769
|
|
|
770
770
|
|
|
771
771
|
def _run_attempt(
|
|
@@ -777,15 +777,15 @@ def _run_attempt(
|
|
|
777
777
|
success_selectors: Optional[list[str]],
|
|
778
778
|
known_bad_sizes: Optional[list[int]],
|
|
779
779
|
timeout: int,
|
|
780
|
-
phase: str,
|
|
781
|
-
enable_retry: bool = False,
|
|
782
|
-
proxy: Optional[str] = None,
|
|
780
|
+
phase: str,
|
|
781
|
+
enable_retry: bool = False,
|
|
782
|
+
proxy: Optional[str] = None,
|
|
783
783
|
) -> tuple[Attempt, Any]:
|
|
784
784
|
"""Execute one curl_cffi attempt and produce an Attempt record."""
|
|
785
785
|
referer_url = REFERER_STRATEGIES.get(referer_name, REFERER_STRATEGIES["none"])(url)
|
|
786
786
|
t0 = time.time()
|
|
787
|
-
resp, err = _curl_probe(url, impersonate=impersonate, referer=referer_url, timeout=timeout,
|
|
788
|
-
enable_retry=enable_retry, proxy=proxy)
|
|
787
|
+
resp, err = _curl_probe(url, impersonate=impersonate, referer=referer_url, timeout=timeout,
|
|
788
|
+
enable_retry=enable_retry, proxy=proxy)
|
|
789
789
|
elapsed = round(time.time() - t0, 3)
|
|
790
790
|
|
|
791
791
|
att = Attempt(
|
|
@@ -937,7 +937,7 @@ def _build_plan(
|
|
|
937
937
|
|
|
938
938
|
|
|
939
939
|
# --- Public entrypoint: self-learning wrapper (U5) ---------------------------
|
|
940
|
-
def _winning_route(result: FetchResult) -> Optional[dict]:
|
|
940
|
+
def _winning_route(result: FetchResult) -> Optional[dict]:
|
|
941
941
|
"""Extract the curl route that produced the OK result, from the trace.
|
|
942
942
|
|
|
943
943
|
Only probe/grid curl wins are learnable: Phase 0 always runs first anyway,
|
|
@@ -951,73 +951,73 @@ def _winning_route(result: FetchResult) -> Optional[dict]:
|
|
|
951
951
|
"referer": att.referer,
|
|
952
952
|
"phase": att.phase,
|
|
953
953
|
}
|
|
954
|
-
return None
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
def _archive_fallback(
|
|
958
|
-
base: FetchResult,
|
|
959
|
-
url: str,
|
|
960
|
-
*,
|
|
961
|
-
timeout: int,
|
|
962
|
-
proxy: Optional[str],
|
|
963
|
-
enable_extraction: bool,
|
|
964
|
-
enable_markdown: bool,
|
|
965
|
-
enable_maincontent: bool,
|
|
966
|
-
) -> FetchResult:
|
|
967
|
-
"""Opt-in historical fallback. A snapshot is always labeled non-current."""
|
|
968
|
-
try:
|
|
969
|
-
from .phase0 import route_archive
|
|
970
|
-
archived = route_archive(url, timeout=timeout, proxy=proxy)
|
|
971
|
-
except Exception as exc:
|
|
972
|
-
archived = {
|
|
973
|
-
"ok": False,
|
|
974
|
-
"attempts": [{
|
|
975
|
-
"route": "wayback", "ok": False, "status": 0, "bytes": 0,
|
|
976
|
-
"note": f"{type(exc).__name__}",
|
|
977
|
-
}],
|
|
978
|
-
}
|
|
979
|
-
trace = list(base.trace)
|
|
980
|
-
for item in archived.get("attempts") or []:
|
|
981
|
-
trace.append(Attempt(
|
|
982
|
-
phase="archive", executor=item.get("route") or "wayback",
|
|
983
|
-
url=archived.get("final_url") or url, url_transform="original",
|
|
984
|
-
impersonate=None, referer="", status=int(item.get("status") or 0),
|
|
985
|
-
body_size=int(item.get("bytes") or 0),
|
|
986
|
-
verdict=(Verdict.STRONG_OK.value if item.get("ok") else Verdict.BLOCKED.value),
|
|
987
|
-
reasons=[item.get("note")] if item.get("note") else [],
|
|
988
|
-
))
|
|
989
|
-
if not archived.get("ok"):
|
|
990
|
-
base.trace = trace
|
|
991
|
-
notes = [item.get("note") for item in archived.get("attempts") or [] if item.get("note")]
|
|
992
|
-
base.summary += f"; archive_failed={notes[-1] if notes else 'unknown'}"
|
|
993
|
-
return base
|
|
994
|
-
|
|
995
|
-
final_url = str(archived.get("final_url") or url)
|
|
996
|
-
_title, content, quality, meta = _maybe_extract(
|
|
997
|
-
_PWResp(str(archived.get("content") or ""), final_url), final_url,
|
|
998
|
-
enable_extraction=enable_extraction,
|
|
999
|
-
enable_markdown=enable_markdown,
|
|
1000
|
-
enable_maincontent=enable_maincontent,
|
|
1001
|
-
)
|
|
1002
|
-
source = "archive+" + (meta.get("source") or "raw")
|
|
1003
|
-
meta = dict(meta)
|
|
1004
|
-
meta.update({
|
|
1005
|
-
"source": source,
|
|
1006
|
-
"snapshot_timestamp": str(archived.get("timestamp") or ""),
|
|
1007
|
-
})
|
|
1008
|
-
return FetchResult(
|
|
1009
|
-
ok=True, content=content, final_url=final_url,
|
|
1010
|
-
verdict=Verdict.STRONG_OK.value, profile_used="archive:wayback",
|
|
1011
|
-
trace=trace, summary="Historical archive fallback succeeded",
|
|
1012
|
-
planned_attempts=base.planned_attempts,
|
|
1013
|
-
executed_attempts=base.executed_attempts,
|
|
1014
|
-
grid_exhausted=base.grid_exhausted, stop_reason="success",
|
|
1015
|
-
extraction_quality=quality, extraction_source=source,
|
|
1016
|
-
extraction_meta=meta,
|
|
1017
|
-
)
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
def fetch(
|
|
954
|
+
return None
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def _archive_fallback(
|
|
958
|
+
base: FetchResult,
|
|
959
|
+
url: str,
|
|
960
|
+
*,
|
|
961
|
+
timeout: int,
|
|
962
|
+
proxy: Optional[str],
|
|
963
|
+
enable_extraction: bool,
|
|
964
|
+
enable_markdown: bool,
|
|
965
|
+
enable_maincontent: bool,
|
|
966
|
+
) -> FetchResult:
|
|
967
|
+
"""Opt-in historical fallback. A snapshot is always labeled non-current."""
|
|
968
|
+
try:
|
|
969
|
+
from .phase0 import route_archive
|
|
970
|
+
archived = route_archive(url, timeout=timeout, proxy=proxy)
|
|
971
|
+
except Exception as exc:
|
|
972
|
+
archived = {
|
|
973
|
+
"ok": False,
|
|
974
|
+
"attempts": [{
|
|
975
|
+
"route": "wayback", "ok": False, "status": 0, "bytes": 0,
|
|
976
|
+
"note": f"{type(exc).__name__}",
|
|
977
|
+
}],
|
|
978
|
+
}
|
|
979
|
+
trace = list(base.trace)
|
|
980
|
+
for item in archived.get("attempts") or []:
|
|
981
|
+
trace.append(Attempt(
|
|
982
|
+
phase="archive", executor=item.get("route") or "wayback",
|
|
983
|
+
url=archived.get("final_url") or url, url_transform="original",
|
|
984
|
+
impersonate=None, referer="", status=int(item.get("status") or 0),
|
|
985
|
+
body_size=int(item.get("bytes") or 0),
|
|
986
|
+
verdict=(Verdict.STRONG_OK.value if item.get("ok") else Verdict.BLOCKED.value),
|
|
987
|
+
reasons=[item.get("note")] if item.get("note") else [],
|
|
988
|
+
))
|
|
989
|
+
if not archived.get("ok"):
|
|
990
|
+
base.trace = trace
|
|
991
|
+
notes = [item.get("note") for item in archived.get("attempts") or [] if item.get("note")]
|
|
992
|
+
base.summary += f"; archive_failed={notes[-1] if notes else 'unknown'}"
|
|
993
|
+
return base
|
|
994
|
+
|
|
995
|
+
final_url = str(archived.get("final_url") or url)
|
|
996
|
+
_title, content, quality, meta = _maybe_extract(
|
|
997
|
+
_PWResp(str(archived.get("content") or ""), final_url), final_url,
|
|
998
|
+
enable_extraction=enable_extraction,
|
|
999
|
+
enable_markdown=enable_markdown,
|
|
1000
|
+
enable_maincontent=enable_maincontent,
|
|
1001
|
+
)
|
|
1002
|
+
source = "archive+" + (meta.get("source") or "raw")
|
|
1003
|
+
meta = dict(meta)
|
|
1004
|
+
meta.update({
|
|
1005
|
+
"source": source,
|
|
1006
|
+
"snapshot_timestamp": str(archived.get("timestamp") or ""),
|
|
1007
|
+
})
|
|
1008
|
+
return FetchResult(
|
|
1009
|
+
ok=True, content=content, final_url=final_url,
|
|
1010
|
+
verdict=Verdict.STRONG_OK.value, profile_used="archive:wayback",
|
|
1011
|
+
trace=trace, summary="Historical archive fallback succeeded",
|
|
1012
|
+
planned_attempts=base.planned_attempts,
|
|
1013
|
+
executed_attempts=base.executed_attempts,
|
|
1014
|
+
grid_exhausted=base.grid_exhausted, stop_reason="success",
|
|
1015
|
+
extraction_quality=quality, extraction_source=source,
|
|
1016
|
+
extraction_meta=meta,
|
|
1017
|
+
)
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
def fetch(
|
|
1021
1021
|
url: str,
|
|
1022
1022
|
*,
|
|
1023
1023
|
success_selectors: Optional[list[str]] = None,
|
|
@@ -1025,21 +1025,21 @@ def fetch(
|
|
|
1025
1025
|
user_hint: Optional[dict] = None,
|
|
1026
1026
|
timeout: int = 25,
|
|
1027
1027
|
max_attempts: Optional[int] = None,
|
|
1028
|
-
max_browser_attempts: int = 2,
|
|
1028
|
+
max_browser_attempts: int = 2,
|
|
1029
1029
|
enable_playwright: bool = True,
|
|
1030
1030
|
enable_phase0: bool = True,
|
|
1031
1031
|
enable_learning: bool = True,
|
|
1032
1032
|
enable_extraction: bool = True,
|
|
1033
1033
|
enable_retry: bool = True,
|
|
1034
|
-
enable_markdown: bool = True,
|
|
1035
|
-
enable_maincontent: bool = False,
|
|
1036
|
-
enable_ocr: bool = False,
|
|
1037
|
-
enable_archive: bool = False,
|
|
1038
|
-
force_media: bool = False,
|
|
1039
|
-
include_media_transcript: bool = False,
|
|
1040
|
-
proxy: Optional[str] = None,
|
|
1041
|
-
cookie_file: Optional[str] = None,
|
|
1042
|
-
) -> FetchResult:
|
|
1034
|
+
enable_markdown: bool = True,
|
|
1035
|
+
enable_maincontent: bool = False,
|
|
1036
|
+
enable_ocr: bool = False,
|
|
1037
|
+
enable_archive: bool = False,
|
|
1038
|
+
force_media: bool = False,
|
|
1039
|
+
include_media_transcript: bool = False,
|
|
1040
|
+
proxy: Optional[str] = None,
|
|
1041
|
+
cookie_file: Optional[str] = None,
|
|
1042
|
+
) -> FetchResult:
|
|
1043
1043
|
"""Public entrypoint — the generic grid wrapped with per-host self-learning.
|
|
1044
1044
|
|
|
1045
1045
|
1. Before fetching, look up the route that last succeeded for this host and
|
|
@@ -1066,15 +1066,15 @@ def fetch(
|
|
|
1066
1066
|
structure-preserving markdown via markdownify (tables → pipe tables,
|
|
1067
1067
|
<pre>/<code> → fences); ``extraction_source`` becomes "raw+md". Set False
|
|
1068
1068
|
for raw HTML. No-op when markdownify is not installed.
|
|
1069
|
-
``enable_maincontent`` (opt-in) instead strips boilerplate via resiliparse."""
|
|
1070
|
-
proxy = normalize_proxy(proxy)
|
|
1071
|
-
cookies: list[dict] = []
|
|
1072
|
-
if cookie_file:
|
|
1073
|
-
from .session_input import load_cookie_file
|
|
1074
|
-
from .transport import POOL, _host_of
|
|
1075
|
-
cookies = load_cookie_file(cookie_file, url)
|
|
1076
|
-
POOL.seed_cookies(_host_of(url), cookies, proxy=proxy)
|
|
1077
|
-
priority: Optional[dict] = None
|
|
1069
|
+
``enable_maincontent`` (opt-in) instead strips boilerplate via resiliparse."""
|
|
1070
|
+
proxy = normalize_proxy(proxy)
|
|
1071
|
+
cookies: list[dict] = []
|
|
1072
|
+
if cookie_file:
|
|
1073
|
+
from .session_input import load_cookie_file
|
|
1074
|
+
from .transport import POOL, _host_of
|
|
1075
|
+
cookies = load_cookie_file(cookie_file, url)
|
|
1076
|
+
POOL.seed_cookies(_host_of(url), cookies, proxy=proxy)
|
|
1077
|
+
priority: Optional[dict] = None
|
|
1078
1078
|
learned_existed = False
|
|
1079
1079
|
uh = dict(user_hint or {})
|
|
1080
1080
|
try:
|
|
@@ -1093,26 +1093,26 @@ def fetch(
|
|
|
1093
1093
|
user_hint=uh, timeout=timeout, max_attempts=max_attempts,
|
|
1094
1094
|
max_browser_attempts=max_browser_attempts,
|
|
1095
1095
|
enable_playwright=enable_playwright, enable_phase0=enable_phase0,
|
|
1096
|
-
priority=priority,
|
|
1097
|
-
enable_extraction=enable_extraction, enable_retry=enable_retry,
|
|
1098
|
-
enable_markdown=enable_markdown, enable_maincontent=enable_maincontent,
|
|
1099
|
-
enable_ocr=enable_ocr, force_media=force_media,
|
|
1100
|
-
include_media_transcript=include_media_transcript,
|
|
1101
|
-
proxy=proxy, cookies=cookies,
|
|
1102
|
-
)
|
|
1103
|
-
|
|
1104
|
-
if enable_archive and not result.ok:
|
|
1105
|
-
result = _archive_fallback(
|
|
1106
|
-
result, url, timeout=timeout, proxy=proxy,
|
|
1107
|
-
enable_extraction=enable_extraction,
|
|
1108
|
-
enable_markdown=enable_markdown,
|
|
1109
|
-
enable_maincontent=enable_maincontent,
|
|
1110
|
-
)
|
|
1111
|
-
|
|
1112
|
-
result.proxy_used = proxy is not None
|
|
1113
|
-
result.session_seeded = bool(cookies)
|
|
1114
|
-
if not result.ok:
|
|
1115
|
-
result.network_diagnosis = diagnose_trace(result.trace)
|
|
1096
|
+
priority=priority,
|
|
1097
|
+
enable_extraction=enable_extraction, enable_retry=enable_retry,
|
|
1098
|
+
enable_markdown=enable_markdown, enable_maincontent=enable_maincontent,
|
|
1099
|
+
enable_ocr=enable_ocr, force_media=force_media,
|
|
1100
|
+
include_media_transcript=include_media_transcript,
|
|
1101
|
+
proxy=proxy, cookies=cookies,
|
|
1102
|
+
)
|
|
1103
|
+
|
|
1104
|
+
if enable_archive and not result.ok:
|
|
1105
|
+
result = _archive_fallback(
|
|
1106
|
+
result, url, timeout=timeout, proxy=proxy,
|
|
1107
|
+
enable_extraction=enable_extraction,
|
|
1108
|
+
enable_markdown=enable_markdown,
|
|
1109
|
+
enable_maincontent=enable_maincontent,
|
|
1110
|
+
)
|
|
1111
|
+
|
|
1112
|
+
result.proxy_used = proxy is not None
|
|
1113
|
+
result.session_seeded = bool(cookies)
|
|
1114
|
+
if not result.ok:
|
|
1115
|
+
result.network_diagnosis = diagnose_trace(result.trace)
|
|
1116
1116
|
|
|
1117
1117
|
try:
|
|
1118
1118
|
from . import learning
|
|
@@ -1138,7 +1138,7 @@ def fetch(
|
|
|
1138
1138
|
|
|
1139
1139
|
|
|
1140
1140
|
# --- Main entrypoint ---------------------------------------------------------
|
|
1141
|
-
def _fetch_core(
|
|
1141
|
+
def _fetch_core(
|
|
1142
1142
|
url: str,
|
|
1143
1143
|
*,
|
|
1144
1144
|
success_selectors: Optional[list[str]] = None,
|
|
@@ -1146,20 +1146,20 @@ def _fetch_core(
|
|
|
1146
1146
|
user_hint: Optional[dict] = None,
|
|
1147
1147
|
timeout: int = 25,
|
|
1148
1148
|
max_attempts: Optional[int] = None, # None = exhaustive (R6); int = budget
|
|
1149
|
-
max_browser_attempts: int = 2,
|
|
1149
|
+
max_browser_attempts: int = 2,
|
|
1150
1150
|
enable_playwright: bool = True,
|
|
1151
1151
|
enable_phase0: bool = True,
|
|
1152
1152
|
priority: Optional[dict] = None, # U5: learned route to retry first
|
|
1153
1153
|
enable_extraction: bool = True,
|
|
1154
1154
|
enable_retry: bool = True,
|
|
1155
|
-
enable_markdown: bool = True,
|
|
1156
|
-
enable_maincontent: bool = False,
|
|
1157
|
-
enable_ocr: bool = False,
|
|
1158
|
-
force_media: bool = False,
|
|
1159
|
-
include_media_transcript: bool = False,
|
|
1160
|
-
proxy: Optional[str] = None,
|
|
1161
|
-
cookies: Optional[list[dict]] = None,
|
|
1162
|
-
) -> FetchResult:
|
|
1155
|
+
enable_markdown: bool = True,
|
|
1156
|
+
enable_maincontent: bool = False,
|
|
1157
|
+
enable_ocr: bool = False,
|
|
1158
|
+
force_media: bool = False,
|
|
1159
|
+
include_media_transcript: bool = False,
|
|
1160
|
+
proxy: Optional[str] = None,
|
|
1161
|
+
cookies: Optional[list[dict]] = None,
|
|
1162
|
+
) -> FetchResult:
|
|
1163
1163
|
"""Fetch `url` using the generic diversity grid.
|
|
1164
1164
|
|
|
1165
1165
|
max_attempts
|
|
@@ -1199,18 +1199,18 @@ def _fetch_core(
|
|
|
1199
1199
|
# is what made Reddit/X look "blocked" (grid 403'd .json; nobody tried .rss).
|
|
1200
1200
|
if enable_phase0:
|
|
1201
1201
|
try:
|
|
1202
|
-
if force_media:
|
|
1203
|
-
from .phase0 import route_media as _phase0_route_media
|
|
1204
|
-
p0 = _phase0_route_media(
|
|
1205
|
-
url, timeout=timeout, proxy=proxy,
|
|
1206
|
-
include_transcript=include_media_transcript,
|
|
1207
|
-
cookies=cookies)
|
|
1208
|
-
else:
|
|
1209
|
-
from .phase0 import route as _phase0_route
|
|
1210
|
-
p0 = _phase0_route(
|
|
1211
|
-
url, timeout=timeout, proxy=proxy,
|
|
1212
|
-
include_media_transcript=include_media_transcript,
|
|
1213
|
-
cookies=cookies)
|
|
1202
|
+
if force_media:
|
|
1203
|
+
from .phase0 import route_media as _phase0_route_media
|
|
1204
|
+
p0 = _phase0_route_media(
|
|
1205
|
+
url, timeout=timeout, proxy=proxy,
|
|
1206
|
+
include_transcript=include_media_transcript,
|
|
1207
|
+
cookies=cookies)
|
|
1208
|
+
else:
|
|
1209
|
+
from .phase0 import route as _phase0_route
|
|
1210
|
+
p0 = _phase0_route(
|
|
1211
|
+
url, timeout=timeout, proxy=proxy,
|
|
1212
|
+
include_media_transcript=include_media_transcript,
|
|
1213
|
+
cookies=cookies)
|
|
1214
1214
|
except Exception as e: # router must never break the generic chain
|
|
1215
1215
|
p0 = None
|
|
1216
1216
|
trace.append(Attempt(
|
|
@@ -1227,21 +1227,21 @@ def _fetch_core(
|
|
|
1227
1227
|
verdict=(Verdict.STRONG_OK.value if a["ok"] else Verdict.BLOCKED.value),
|
|
1228
1228
|
reasons=[a["note"]] if a.get("note") else [],
|
|
1229
1229
|
))
|
|
1230
|
-
if p0["ok"]:
|
|
1231
|
-
content_kind = p0.get("content_kind") or "phase0"
|
|
1232
|
-
return FetchResult(
|
|
1233
|
-
ok=True, content=p0["content"], final_url=p0["final_url"],
|
|
1230
|
+
if p0["ok"]:
|
|
1231
|
+
content_kind = p0.get("content_kind") or "phase0"
|
|
1232
|
+
return FetchResult(
|
|
1233
|
+
ok=True, content=p0["content"], final_url=p0["final_url"],
|
|
1234
1234
|
verdict=Verdict.STRONG_OK.value,
|
|
1235
1235
|
profile_used=f"phase0:{p0['platform']}", trace=trace,
|
|
1236
|
-
summary=f"Phase 0 official route: {p0['platform']}:{p0['route']}",
|
|
1237
|
-
stop_reason="success",
|
|
1238
|
-
extraction_quality=_quality_score(p0["content"]),
|
|
1239
|
-
extraction_source=content_kind,
|
|
1240
|
-
extraction_meta={
|
|
1241
|
-
"source": content_kind,
|
|
1242
|
-
"transcript_available": bool(p0.get("transcript")),
|
|
1243
|
-
},
|
|
1244
|
-
)
|
|
1236
|
+
summary=f"Phase 0 official route: {p0['platform']}:{p0['route']}",
|
|
1237
|
+
stop_reason="success",
|
|
1238
|
+
extraction_quality=_quality_score(p0["content"]),
|
|
1239
|
+
extraction_source=content_kind,
|
|
1240
|
+
extraction_meta={
|
|
1241
|
+
"source": content_kind,
|
|
1242
|
+
"transcript_available": bool(p0.get("transcript")),
|
|
1243
|
+
},
|
|
1244
|
+
)
|
|
1245
1245
|
# Recognised platform but every official route failed → fall through
|
|
1246
1246
|
# to the generic grid (don't give up; R6).
|
|
1247
1247
|
|
|
@@ -1262,8 +1262,8 @@ def _fetch_core(
|
|
|
1262
1262
|
if pool_enabled():
|
|
1263
1263
|
_root = _root_of(url)
|
|
1264
1264
|
if _root != url:
|
|
1265
|
-
POOL.warmup(_host_of(url), base_impersonate, _root,
|
|
1266
|
-
timeout=min(timeout, 15), proxy=proxy)
|
|
1265
|
+
POOL.warmup(_host_of(url), base_impersonate, _root,
|
|
1266
|
+
timeout=min(timeout, 15), proxy=proxy)
|
|
1267
1267
|
except Exception:
|
|
1268
1268
|
pass
|
|
1269
1269
|
|
|
@@ -1274,9 +1274,9 @@ def _fetch_core(
|
|
|
1274
1274
|
probe_attempt, probe_resp = _run_attempt(
|
|
1275
1275
|
url, transform_name="original", impersonate=base_impersonate,
|
|
1276
1276
|
referer_name=base_referer, success_selectors=success_selectors,
|
|
1277
|
-
known_bad_sizes=None, timeout=timeout, phase="probe",
|
|
1278
|
-
enable_retry=enable_retry,
|
|
1279
|
-
proxy=proxy,
|
|
1277
|
+
known_bad_sizes=None, timeout=timeout, phase="probe",
|
|
1278
|
+
enable_retry=enable_retry,
|
|
1279
|
+
proxy=proxy,
|
|
1280
1280
|
)
|
|
1281
1281
|
trace.append(probe_attempt)
|
|
1282
1282
|
curl_attempts += 1
|
|
@@ -1286,10 +1286,10 @@ def _fetch_core(
|
|
|
1286
1286
|
return _build_result(probe_resp, probe_attempt, trace, profile_used=None,
|
|
1287
1287
|
planned=0, executed=curl_attempts,
|
|
1288
1288
|
grid_exhausted=False, stop_reason="success",
|
|
1289
|
-
enable_extraction=enable_extraction,
|
|
1290
|
-
enable_markdown=enable_markdown,
|
|
1291
|
-
enable_maincontent=enable_maincontent,
|
|
1292
|
-
enable_ocr=enable_ocr)
|
|
1289
|
+
enable_extraction=enable_extraction,
|
|
1290
|
+
enable_markdown=enable_markdown,
|
|
1291
|
+
enable_maincontent=enable_maincontent,
|
|
1292
|
+
enable_ocr=enable_ocr)
|
|
1293
1293
|
if probe_attempt.verdict == Verdict.SUSPECT_OK.value:
|
|
1294
1294
|
best_suspect = (probe_resp, probe_attempt)
|
|
1295
1295
|
elif probe_attempt.verdict in _TERMINAL_NONSUCCESS_VALUES:
|
|
@@ -1318,9 +1318,9 @@ def _fetch_core(
|
|
|
1318
1318
|
att, resp = _run_attempt(
|
|
1319
1319
|
cand.url, transform_name=cand.transform, impersonate=cand.impersonate,
|
|
1320
1320
|
referer_name=cand.referer, success_selectors=success_selectors,
|
|
1321
|
-
known_bad_sizes=list(cand.known_bad_sizes) if cand.known_bad_sizes else None,
|
|
1322
|
-
timeout=timeout, phase="grid",
|
|
1323
|
-
proxy=proxy,
|
|
1321
|
+
known_bad_sizes=list(cand.known_bad_sizes) if cand.known_bad_sizes else None,
|
|
1322
|
+
timeout=timeout, phase="grid",
|
|
1323
|
+
proxy=proxy,
|
|
1324
1324
|
)
|
|
1325
1325
|
trace.append(att)
|
|
1326
1326
|
curl_attempts += 1
|
|
@@ -1330,10 +1330,10 @@ def _fetch_core(
|
|
|
1330
1330
|
return _build_result(resp, att, trace, profile_used=cand.profile_id,
|
|
1331
1331
|
planned=planned, executed=curl_attempts,
|
|
1332
1332
|
grid_exhausted=False, stop_reason="success",
|
|
1333
|
-
enable_extraction=enable_extraction,
|
|
1334
|
-
enable_markdown=enable_markdown,
|
|
1335
|
-
enable_maincontent=enable_maincontent,
|
|
1336
|
-
enable_ocr=enable_ocr)
|
|
1333
|
+
enable_extraction=enable_extraction,
|
|
1334
|
+
enable_markdown=enable_markdown,
|
|
1335
|
+
enable_maincontent=enable_maincontent,
|
|
1336
|
+
enable_ocr=enable_ocr)
|
|
1337
1337
|
if att.verdict == Verdict.SUSPECT_OK.value and best_suspect is None:
|
|
1338
1338
|
best_suspect = (resp, att)
|
|
1339
1339
|
if att.verdict in _TERMINAL_NONSUCCESS_VALUES:
|
|
@@ -1356,8 +1356,8 @@ def _fetch_core(
|
|
|
1356
1356
|
try:
|
|
1357
1357
|
from .executor import run_playwright_fallback
|
|
1358
1358
|
fb_profile = load_profile(profile_used or "unknown_challenge", profiles=profiles)
|
|
1359
|
-
fb_order = fb_profile.get("fallback_when_challenge") or [
|
|
1360
|
-
"scrapling", "stealth_firefox", "playwright_real_chrome"]
|
|
1359
|
+
fb_order = fb_profile.get("fallback_when_challenge") or [
|
|
1360
|
+
"scrapling", "stealth_firefox", "playwright_real_chrome"]
|
|
1361
1361
|
for fb_name in fb_order:
|
|
1362
1362
|
if fb_name == "curl_grid_exhaust":
|
|
1363
1363
|
continue
|
|
@@ -1374,13 +1374,13 @@ def _fetch_core(
|
|
|
1374
1374
|
break
|
|
1375
1375
|
pw_attempt, pw_content = run_playwright_fallback(
|
|
1376
1376
|
url, profile_id=profile_used or "unknown_challenge",
|
|
1377
|
-
success_selectors=success_selectors, device_class=device_class,
|
|
1378
|
-
force_executor=fb_name, timeout=timeout if timeout and timeout > 30 else 90,
|
|
1379
|
-
proxy=proxy, cookies=cookies,
|
|
1380
|
-
)
|
|
1377
|
+
success_selectors=success_selectors, device_class=device_class,
|
|
1378
|
+
force_executor=fb_name, timeout=timeout if timeout and timeout > 30 else 90,
|
|
1379
|
+
proxy=proxy, cookies=cookies,
|
|
1380
|
+
)
|
|
1381
1381
|
trace.append(pw_attempt)
|
|
1382
|
-
if not is_mcp_stub and getattr(pw_attempt, "_executed", True):
|
|
1383
|
-
browser_used += 1
|
|
1382
|
+
if not is_mcp_stub and getattr(pw_attempt, "_executed", True):
|
|
1383
|
+
browser_used += 1
|
|
1384
1384
|
if pw_attempt.verdict in _OK_VALUES:
|
|
1385
1385
|
# Render-merge: the executor stashes the rendered innerText
|
|
1386
1386
|
# on the attempt; the rescue gate keeps whichever of
|
|
@@ -1388,10 +1388,10 @@ def _fetch_core(
|
|
|
1388
1388
|
pw_inner = getattr(pw_attempt, "_inner_text", "") or ""
|
|
1389
1389
|
_t, pw_out, pw_q, pw_meta = _maybe_extract(
|
|
1390
1390
|
_PWResp(pw_content, pw_attempt.url), pw_attempt.url,
|
|
1391
|
-
enable_extraction=enable_extraction, inner_text=pw_inner,
|
|
1392
|
-
enable_markdown=enable_markdown,
|
|
1393
|
-
enable_maincontent=enable_maincontent,
|
|
1394
|
-
enable_ocr=enable_ocr)
|
|
1391
|
+
enable_extraction=enable_extraction, inner_text=pw_inner,
|
|
1392
|
+
enable_markdown=enable_markdown,
|
|
1393
|
+
enable_maincontent=enable_maincontent,
|
|
1394
|
+
enable_ocr=enable_ocr)
|
|
1395
1395
|
return FetchResult(
|
|
1396
1396
|
ok=True, content=pw_out, final_url=pw_attempt.url,
|
|
1397
1397
|
verdict=pw_attempt.verdict, profile_used=profile_used,
|
|
@@ -1459,9 +1459,9 @@ def _untried_routes(stop_reason, grid_exhausted) -> tuple[list[str], bool]:
|
|
|
1459
1459
|
return routes, must_mcp
|
|
1460
1460
|
|
|
1461
1461
|
|
|
1462
|
-
_REAL_EXECUTORS = frozenset({
|
|
1463
|
-
"curl_cffi", "playwright_real_chrome", "playwright_mobile_chrome",
|
|
1464
|
-
"scrapling:stealthy_fetcher", "stealth_firefox:invisible_playwright"})
|
|
1462
|
+
_REAL_EXECUTORS = frozenset({
|
|
1463
|
+
"curl_cffi", "playwright_real_chrome", "playwright_mobile_chrome",
|
|
1464
|
+
"scrapling:stealthy_fetcher", "stealth_firefox:invisible_playwright"})
|
|
1465
1465
|
_INFRA_AUTH_VERDICTS = frozenset({Verdict.AUTH_REQUIRED.value, Verdict.NOT_FOUND.value})
|
|
1466
1466
|
_WAF_VERDICTS = frozenset({
|
|
1467
1467
|
Verdict.CHALLENGE.value, Verdict.BLOCKED.value,
|
|
@@ -1548,16 +1548,16 @@ def fetch_many(urls: list[str], **kwargs) -> list[FetchResult]:
|
|
|
1548
1548
|
return [r for r in results if r is not None]
|
|
1549
1549
|
|
|
1550
1550
|
|
|
1551
|
-
def _build_result(resp, attempt: Attempt, trace: list[Attempt], profile_used: Optional[str],
|
|
1552
|
-
*, planned: int, executed: int, grid_exhausted: bool, stop_reason: str,
|
|
1553
|
-
enable_extraction: bool = True, enable_markdown: bool = True,
|
|
1554
|
-
enable_maincontent: bool = False,
|
|
1555
|
-
enable_ocr: bool = False) -> FetchResult:
|
|
1551
|
+
def _build_result(resp, attempt: Attempt, trace: list[Attempt], profile_used: Optional[str],
|
|
1552
|
+
*, planned: int, executed: int, grid_exhausted: bool, stop_reason: str,
|
|
1553
|
+
enable_extraction: bool = True, enable_markdown: bool = True,
|
|
1554
|
+
enable_maincontent: bool = False,
|
|
1555
|
+
enable_ocr: bool = False) -> FetchResult:
|
|
1556
1556
|
final_url = str(getattr(resp, "url", attempt.url))
|
|
1557
|
-
_t, content, quality, meta = _maybe_extract(
|
|
1558
|
-
resp, final_url, enable_extraction=enable_extraction,
|
|
1559
|
-
enable_markdown=enable_markdown, enable_maincontent=enable_maincontent,
|
|
1560
|
-
enable_ocr=enable_ocr)
|
|
1557
|
+
_t, content, quality, meta = _maybe_extract(
|
|
1558
|
+
resp, final_url, enable_extraction=enable_extraction,
|
|
1559
|
+
enable_markdown=enable_markdown, enable_maincontent=enable_maincontent,
|
|
1560
|
+
enable_ocr=enable_ocr)
|
|
1561
1561
|
return FetchResult(
|
|
1562
1562
|
ok=True,
|
|
1563
1563
|
content=content,
|