@mmmbuto/masix 0.4.2 → 0.4.3

package/packages/plugin-base/discovery/source/src/torrent.rs

@@ -0,0 +1,701 @@
+use crate::doh::DohResolver;
+use crate::magnet_cache::MagnetCache;
+use crate::transport::TransportLayer;
+use anyhow::{anyhow, Result};
+use scraper::{Html, Selector};
+use serde::Serialize;
+use std::collections::HashSet;
+
+#[derive(Debug, Clone)]
+pub struct TorrentProvider {
+    pub id: &'static str,
+    pub display_name: &'static str,
+    pub clearnet_domains: &'static [&'static str],
+    pub proxy_urls: &'static [&'static str],
+    pub onion_urls: &'static [&'static str],
+    parser: ProviderParser,
+}
+
+#[derive(Debug, Clone, Copy)]
+enum ProviderParser {
+    GenericPath(&'static str),
+    PirateBay,
+    YtsJson,
+    EztvPath,
+    SolidTorrents,
+    Bt4g,
+    Torrentz2,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct ProviderStatus {
+    pub id: String,
+    pub name: String,
+    pub supports_clearnet: bool,
+    pub supports_proxy: bool,
+    pub supports_onion: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct MirrorTorrentResult {
+    pub title: String,
+    pub url: String,
+    pub content: String,
+    pub provider: String,
+    pub via: String,
+    pub source_url: String,
+    pub magnet_links: Vec<String>,
+    pub seeds: Option<u32>,
+    pub leeches: Option<u32>,
+    pub size: Option<String>,
+    pub uploader: Option<String>,
+    pub category: Option<String>,
+}
+
+pub fn bundled_providers() -> Vec<TorrentProvider> {
+    vec![
+        TorrentProvider {
+            id: "1337x",
+            display_name: "1337x",
+            clearnet_domains: &[
+                "https://1337x.to",
+                "https://1337x.st",
+                "https://x1337x.se",
+                "https://1337xx.to",
+            ],
+            proxy_urls: &[
+                "https://unblockit.bz/1337x",
+                "https://unblockit.id/1337x",
+                "https://nocensor.lol/1337x",
+            ],
+            onion_urls: &[],
+            parser: ProviderParser::GenericPath("/search/{query}/1/"),
+        },
+        TorrentProvider {
+            id: "thepiratebay",
+            display_name: "ThePirateBay",
+            clearnet_domains: &[
+                "https://thepiratebay.org",
+                "https://thepiratesbay.com",
+                "https://tpb.party",
+            ],
+            proxy_urls: &["https://unblockit.bz/tpb", "https://nocensor.lol/tpb"],
+            onion_urls: &[],
+            parser: ProviderParser::PirateBay,
+        },
+        TorrentProvider {
+            id: "yts",
+            display_name: "YTS",
+            clearnet_domains: &["https://yts.mx", "https://yts.lt", "https://yts.am"],
+            proxy_urls: &[],
+            onion_urls: &[],
+            parser: ProviderParser::YtsJson,
+        },
+        TorrentProvider {
+            id: "torrentgalaxy",
+            display_name: "TorrentGalaxy",
+            clearnet_domains: &["https://torrentgalaxy.to", "https://torrentgalaxy.mx"],
+            proxy_urls: &[],
+            onion_urls: &[],
+            parser: ProviderParser::GenericPath("/torrents.php?search={query}"),
+        },
+        TorrentProvider {
+            id: "eztv",
+            display_name: "EZTV",
+            clearnet_domains: &["https://eztv.re", "https://eztv.ag", "https://eztv.io"],
+            proxy_urls: &[],
+            onion_urls: &[],
+            parser: ProviderParser::EztvPath,
+        },
+        TorrentProvider {
+            id: "kickass",
+            display_name: "Kickass",
+            clearnet_domains: &[
+                "https://kickasstorrents.to",
+                "https://katcr.to",
+                "https://kat.sx",
+            ],
+            proxy_urls: &[],
+            onion_urls: &[],
+            parser: ProviderParser::GenericPath("/usearch/{query}/"),
+        },
+        TorrentProvider {
+            id: "limetorrents",
+            display_name: "LimeTorrents",
+            clearnet_domains: &[
+                "https://www.limetorrents.lol",
+                "https://www.limetorrents.info",
+            ],
+            proxy_urls: &[],
+            onion_urls: &[],
+            parser: ProviderParser::GenericPath("/search/all/{query}/"),
+        },
+        TorrentProvider {
+            id: "solidtorrents",
+            display_name: "SolidTorrents",
+            clearnet_domains: &["https://solidtorrents.to"],
+            proxy_urls: &[],
+            onion_urls: &[],
+            parser: ProviderParser::SolidTorrents,
+        },
+        TorrentProvider {
+            id: "bt4g",
+            display_name: "BT4G",
+            clearnet_domains: &["https://bt4gprx.com", "https://bt4g.org"],
+            proxy_urls: &[],
+            onion_urls: &[],
+            parser: ProviderParser::Bt4g,
+        },
+        TorrentProvider {
+            id: "torrentz2",
+            display_name: "Torrentz2",
+            clearnet_domains: &["https://torrentz2eu.org"],
+            proxy_urls: &[],
+            onion_urls: &[],
+            parser: ProviderParser::Torrentz2,
+        },
+    ]
+}
+
+pub fn provider_statuses(providers: &[TorrentProvider]) -> Vec<ProviderStatus> {
+    providers
+        .iter()
+        .map(|provider| ProviderStatus {
+            id: provider.id.to_string(),
+            name: provider.display_name.to_string(),
+            supports_clearnet: !provider.clearnet_domains.is_empty(),
+            supports_proxy: !provider.proxy_urls.is_empty(),
+            supports_onion: !provider.onion_urls.is_empty(),
+        })
+        .collect()
+}
+
+pub async fn search_mirror_catalog(
+    providers: &[TorrentProvider],
+    transport: &TransportLayer,
+    doh: &DohResolver,
+    cache: Option<&MagnetCache>,
+    query: &str,
+    max_results: usize,
+    allowed: Option<&HashSet<String>>,
+    use_tor: bool,
+) -> Result<Vec<MirrorTorrentResult>> {
+    let mut output = Vec::new();
+    let normalized_allowed = allowed.map(|items| {
+        items
+            .iter()
+            .map(|value| value.trim().to_lowercase())
+            .collect::<HashSet<_>>()
+    });
+
+    for provider in providers {
+        if let Some(allowed) = &normalized_allowed {
+            if !allowed.contains(&provider.id.to_lowercase())
+                && !allowed.contains(&provider.display_name.to_lowercase())
+            {
+                continue;
+            }
+        }
+
+        match search_single_provider(provider, transport, doh, cache, query, max_results, use_tor)
+            .await
+        {
+            Ok(mut results) => output.append(&mut results),
+            Err(_) => continue,
+        }
+    }
+
+    if output.is_empty() {
+        Err(anyhow!("mirror catalog returned no results"))
+    } else {
+        Ok(output)
+    }
+}
+
+async fn search_single_provider(
+    provider: &TorrentProvider,
+    transport: &TransportLayer,
+    doh: &DohResolver,
+    cache: Option<&MagnetCache>,
+    query: &str,
+    max_results: usize,
+    use_tor: bool,
+) -> Result<Vec<MirrorTorrentResult>> {
+    let mut routes = Vec::new();
+    if use_tor && transport.tor_available() {
+        for onion in provider.onion_urls {
+            routes.push(((*onion).to_string(), "onion".to_string(), true));
+        }
+    }
+    for proxy in provider.proxy_urls {
+        routes.push(((*proxy).to_string(), "proxy".to_string(), false));
+    }
+    for domain in provider.clearnet_domains {
+        let _ = doh.resolve_with_fallback(extract_host(domain)).await;
+        routes.push(((*domain).to_string(), "clearnet".to_string(), false));
+    }
+
+    let mut last_error: Option<anyhow::Error> = None;
+    for (base, via, prefer_tor) in routes {
+        match fetch_provider_results(
+            provider,
+            transport,
+            cache,
+            query,
+            max_results,
+            &base,
+            &via,
+            prefer_tor,
+        )
+        .await
+        {
+            Ok(results) if !results.is_empty() => return Ok(results),
+            Ok(_) => continue,
+            Err(err) => last_error = Some(err),
+        }
+    }
+
+    Err(last_error.unwrap_or_else(|| anyhow!("provider {} returned no results", provider.id)))
+}
+
+async fn fetch_provider_results(
+    provider: &TorrentProvider,
+    transport: &TransportLayer,
+    cache: Option<&MagnetCache>,
+    query: &str,
+    max_results: usize,
+    base_url: &str,
+    via: &str,
+    prefer_tor: bool,
+) -> Result<Vec<MirrorTorrentResult>> {
+    match provider.parser {
+        ProviderParser::YtsJson => {
+            fetch_yts(
+                provider,
+                transport,
+                query,
+                max_results,
+                base_url,
+                via,
+                prefer_tor,
+            )
+            .await
+        }
+        parser => {
+            fetch_html(
+                provider,
+                parser,
+                transport,
+                cache,
+                query,
+                max_results,
+                base_url,
+                via,
+                prefer_tor,
+            )
+            .await
+        }
+    }
+}
+
+async fn fetch_html(
+    provider: &TorrentProvider,
+    parser: ProviderParser,
+    transport: &TransportLayer,
+    cache: Option<&MagnetCache>,
+    query: &str,
+    max_results: usize,
+    base_url: &str,
+    via: &str,
+    prefer_tor: bool,
+) -> Result<Vec<MirrorTorrentResult>> {
+    let search_url = build_search_url(base_url, parser, query);
+    let (client, resolved_via) = transport.client_for_preference(prefer_tor);
+    let response = client.get(&search_url).send().await?;
+    if !response.status().is_success() {
+        return Err(anyhow!(
+            "provider {} returned HTTP {}",
+            provider.id,
+            response.status()
+        ));
+    }
+    let html = response.text().await?;
+    parse_html_results(
+        provider,
+        parser,
+        cache,
+        client,
+        &html,
+        base_url,
+        &search_url,
+        via_or(resolved_via, via),
+        max_results,
+    )
+    .await
+}
+
+async fn fetch_yts(
+    provider: &TorrentProvider,
+    transport: &TransportLayer,
+    query: &str,
+    max_results: usize,
+    base_url: &str,
+    via: &str,
+    prefer_tor: bool,
+) -> Result<Vec<MirrorTorrentResult>> {
+    let url = format!(
+        "{}/api/v2/list_movies.json?query_term={}&limit={}",
+        base_url.trim_end_matches('/'),
+        url_encode(query),
+        max_results.min(20).max(1)
+    );
+    let (client, resolved_via) = transport.client_for_preference(prefer_tor);
+    let response = client.get(&url).send().await?;
+    if !response.status().is_success() {
+        return Err(anyhow!(
+            "provider {} returned HTTP {}",
+            provider.id,
+            response.status()
+        ));
+    }
+    let payload: serde_json::Value = response.json().await?;
+    let mut output = Vec::new();
+    for movie in payload
+        .get("data")
+        .and_then(|v| v.get("movies"))
+        .and_then(|v| v.as_array())
+        .into_iter()
+        .flatten()
+        .take(max_results.min(20))
+    {
+        let title = movie
+            .get("title_long")
+            .and_then(|v| v.as_str())
+            .unwrap_or("YTS result");
+        let source_url = movie
+            .get("url")
+            .and_then(|v| v.as_str())
+            .unwrap_or(base_url);
+        let summary = movie
+            .get("summary")
+            .and_then(|v| v.as_str())
+            .unwrap_or("YTS API result");
+        let mut magnets = Vec::new();
+        let mut seeds = None;
+        let mut size = None;
+        if let Some(torrents) = movie.get("torrents").and_then(|v| v.as_array()) {
+            if let Some(first) = torrents.first() {
+                if let Some(hash) = first.get("hash").and_then(|v| v.as_str()) {
+                    magnets.push(format!("magnet:?xt=urn:btih:{hash}"));
+                }
+                seeds = first
+                    .get("seeds")
+                    .and_then(|v| v.as_u64())
+                    .map(|v| v as u32);
+                size = first
+                    .get("size")
+                    .and_then(|v| v.as_str())
+                    .map(|v| v.to_string());
+            }
+        }
+        output.push(MirrorTorrentResult {
+            title: title.to_string(),
+            url: source_url.to_string(),
+            content: truncate_text(summary, 700),
+            provider: provider.id.to_string(),
+            via: via_or(resolved_via, via).to_string(),
+            source_url: url.clone(),
+            magnet_links: magnets,
+            seeds,
+            leeches: None,
+            size,
+            uploader: None,
+            category: None,
+        });
+    }
+    Ok(output)
+}
+
+async fn parse_html_results(
+    provider: &TorrentProvider,
+    parser: ProviderParser,
+    cache: Option<&MagnetCache>,
+    client: &reqwest::Client,
+    html: &str,
+    base_url: &str,
+    source_url: &str,
+    via: &str,
+    max_results: usize,
+) -> Result<Vec<MirrorTorrentResult>> {
+    let document = Html::parse_document(html);
+    let row_selector = match parser {
+        ProviderParser::PirateBay => "table#searchResult tr",
+        ProviderParser::EztvPath => {
+            "table.forum_header_border tr.forum_header_border, tr.forum_header_border"
+        }
+        ProviderParser::SolidTorrents => "div.card.search-result, div.search-result",
+        ProviderParser::Bt4g => "div.result-item, div.one_result, li",
+        ProviderParser::Torrentz2 => "dl, div.results dl",
+        _ => "table tbody tr, tr.odd, tr.even, div.result, article, li",
+    };
+    let row_selector =
+        Selector::parse(row_selector).map_err(|e| anyhow!("Invalid row selector: {}", e))?;
+    let link_selector =
+        Selector::parse("a").map_err(|e| anyhow!("Invalid link selector: {}", e))?;
+    let magnet_selector = Selector::parse("a[href^=\"magnet:?\"]")
+        .map_err(|e| anyhow!("Invalid magnet selector: {}", e))?;
+
+    let mut output = Vec::new();
+    let mut seen = HashSet::new();
+
+    for row in document.select(&row_selector) {
+        if output.len() >= max_results.min(20) {
+            break;
+        }
+        let row_text = compact_text(&row.text().collect::<Vec<_>>().join(" "));
+        if row_text.is_empty() {
+            continue;
+        }
+
+        let mut title = String::new();
+        let mut detail_url = None;
+        for link in row.select(&link_selector) {
+            let text = compact_text(&link.text().collect::<Vec<_>>().join(" "));
+            let href = link.value().attr("href").unwrap_or("").trim();
+            if href.is_empty() || text.len() < 3 {
+                continue;
+            }
+            if href.starts_with("magnet:?") {
+                continue;
+            }
+            title = text;
+            detail_url = Some(resolve_href(base_url, href));
+            break;
+        }
+        let Some(detail_url) = detail_url else {
+            continue;
+        };
+        if !seen.insert(detail_url.clone()) {
+            continue;
+        }
+
+        let mut magnet_links = Vec::new();
+        for magnet in row.select(&magnet_selector) {
+            if let Some(href) = magnet.value().attr("href") {
+                let cleaned = sanitize_magnet(href);
+                if cleaned.starts_with("magnet:?") {
+                    magnet_links.push(cleaned);
+                }
+            }
+        }
+        magnet_links.truncate(3);
+
+        if magnet_links.is_empty() {
+            if let Some(cache) = cache {
+                if let Some(cached) = cache.get(&detail_url).await {
+                    magnet_links.push(cached);
+                }
+            }
+        }
+
+        if magnet_links.is_empty() {
+            let fetched = fetch_detail_magnets(client, &detail_url, 3)
+                .await
+                .unwrap_or_default();
+            if let Some(first) = fetched.first() {
+                if let Some(cache) = cache {
+                    let _ = cache.set(&detail_url, first).await;
+                }
+            }
+            magnet_links = fetched;
+        }
+
+        let seeds = extract_number(&row_text);
+        output.push(MirrorTorrentResult {
+            title: if title.is_empty() {
+                format!("{} result", provider.display_name)
+            } else {
+                title
+            },
+            url: detail_url,
+            content: truncate_text(&row_text, 700),
+            provider: provider.id.to_string(),
+            via: via.to_string(),
+            source_url: source_url.to_string(),
+            magnet_links,
+            seeds,
+            leeches: extract_second_number(&row_text, seeds),
+            size: extract_size(&row_text),
+            uploader: None,
+            category: None,
+        });
+    }
+
+    Ok(output)
+}
+
+async fn fetch_detail_magnets(
+    client: &reqwest::Client,
+    url: &str,
+    max_links: usize,
+) -> Result<Vec<String>> {
+    let response = client.get(url).send().await?;
+    if !response.status().is_success() {
+        return Ok(Vec::new());
+    }
+    let html = response.text().await.unwrap_or_default();
+    Ok(extract_magnets_from_text(&html, max_links))
+}
+
+fn extract_magnets_from_text(text: &str, max_links: usize) -> Vec<String> {
+    let mut links = Vec::new();
+    let mut seen = HashSet::new();
+    let mut idx = 0usize;
+    while idx < text.len() {
+        let Some(found) = text[idx..].find("magnet:?") else {
+            break;
+        };
+        let start = idx + found;
+        let remainder = &text[start..];
+        let end_rel = remainder
+            .find(|c: char| c.is_whitespace() || matches!(c, '"' | '\'' | '<' | '>'))
+            .unwrap_or(remainder.len());
+        let raw = &remainder[..end_rel];
+        let cleaned = sanitize_magnet(raw);
+        if cleaned.starts_with("magnet:?") && seen.insert(cleaned.clone()) {
+            links.push(cleaned);
+        }
+        if links.len() >= max_links {
+            break;
+        }
+        idx = start + end_rel;
+    }
+    links
+}
+
+fn build_search_url(base_url: &str, parser: ProviderParser, query: &str) -> String {
+    let encoded = url_encode(query);
+    let base = base_url.trim_end_matches('/');
+    match parser {
+        ProviderParser::GenericPath(pattern) => {
+            format!("{base}{}", pattern.replace("{query}", &encoded))
+        }
+        ProviderParser::PirateBay => format!("{base}/search/{encoded}/1/99/0"),
+        ProviderParser::EztvPath => format!("{base}/search/{encoded}"),
+        ProviderParser::SolidTorrents => format!("{base}/search?q={encoded}"),
+        ProviderParser::Bt4g => format!("{base}/search?q={encoded}"),
+        ProviderParser::Torrentz2 => format!("{base}/search?f={encoded}"),
+        ProviderParser::YtsJson => format!("{base}/api/v2/list_movies.json?query_term={encoded}"),
+    }
+}
+
+fn resolve_href(base_url: &str, href: &str) -> String {
+    if href.starts_with("http://") || href.starts_with("https://") || href.starts_with("magnet:?") {
+        href.to_string()
+    } else {
+        format!(
+            "{}/{}",
+            base_url.trim_end_matches('/'),
+            href.trim_start_matches('/')
+        )
+    }
+}
+
+fn compact_text(value: &str) -> String {
+    value.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+fn extract_number(text: &str) -> Option<u32> {
+    text.split(|c: char| !c.is_ascii_digit())
+        .find_map(|chunk| chunk.parse::<u32>().ok())
+}
+
+/// Extract the second standalone number from row text (typically leechers, after seeds).
+fn extract_second_number(text: &str, first: Option<u32>) -> Option<u32> {
+    let first_val = first?;
+    let mut found_first = false;
+    for chunk in text.split(|c: char| !c.is_ascii_digit()) {
+        if let Ok(n) = chunk.parse::<u32>() {
+            if !found_first && n == first_val {
+                found_first = true;
+            } else if found_first && n != first_val {
+                return Some(n);
+            }
+        }
+    }
+    None
+}
+
+fn extract_size(text: &str) -> Option<String> {
+    for token in text.split_whitespace() {
+        let lower = token.to_lowercase();
+        if lower.ends_with("gb")
+            || lower.ends_with("mb")
+            || lower.ends_with("kb")
+            || lower.ends_with("tb")
+        {
+            return Some(
+                token
+                    .trim_matches(|c: char| c == ',' || c == ';')
+                    .to_string(),
+            );
+        }
+    }
+    None
+}
+
+fn sanitize_magnet(value: &str) -> String {
+    value
+        .trim()
+        .trim_matches('"')
+        .trim_matches('\'')
+        .replace("&amp;", "&")
+}
+
+fn truncate_text(value: &str, max_chars: usize) -> String {
+    if value.chars().count() <= max_chars {
+        return value.to_string();
+    }
+    let mut out = String::new();
+    for (idx, ch) in value.chars().enumerate() {
+        if idx >= max_chars {
+            break;
+        }
+        out.push(ch);
+    }
+    out.push_str("...");
+    out
+}
+
+fn extract_host(url: &str) -> &str {
+    url.trim_start_matches("https://")
+        .trim_start_matches("http://")
+        .split('/')
+        .next()
+        .unwrap_or(url)
+}
+
+fn url_encode(value: &str) -> String {
+    let mut out = String::new();
+    for b in value.as_bytes() {
+        let is_unreserved = matches!(
+            *b,
+            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~'
+        );
+        if is_unreserved {
+            out.push(*b as char);
+        } else {
+            use std::fmt::Write as _;
+            let _ = write!(&mut out, "%{:02X}", b);
+        }
+    }
+    out
+}
+
+fn via_or<'a>(resolved: &'a str, fallback: &'a str) -> &'a str {
+    if resolved == "clearnet" {
+        fallback
+    } else {
+        resolved
+    }
+}