@mmmbuto/masix 0.4.2 → 0.4.3

package/packages/plugin-base/discovery/source/src/main.rs

@@ -1,3 +1,14 @@
+mod doh;
+mod magnet_cache;
+mod torrent;
+mod transport;
+
+use crate::doh::DohResolver;
+use crate::magnet_cache::MagnetCache;
+use crate::torrent::{
+    bundled_providers, provider_statuses, search_mirror_catalog, ProviderStatus, TorrentProvider,
+};
+use crate::transport::TransportLayer;
 use anyhow::{anyhow, Result};
 use clap::{Parser, Subcommand};
 use scraper::{Html, Selector};
@@ -5,6 +16,7 @@ use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 use std::collections::{HashMap, HashSet};
 use std::io::{self, BufRead, Write};
+use std::path::PathBuf;
 use std::sync::OnceLock;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use tokio::sync::Mutex;
@@ -13,13 +25,17 @@ const DEFAULT_SEARXNG_URLS: &[&str] = &[
     "https://search.inetol.net",
     "https://searx.work",
     "https://search.privacyredirect.com",
+    "https://searx.be",
+    "https://northboot.xyz",
+    "https://searxng.site",
+    "https://etsi.me",
 ];
 const MAX_WEB_CONTENT: usize = 15_000;
 const SEARX_RETRIES: usize = 3;
 const MAX_SEARX_PARALLEL_ENDPOINTS: usize = 3;
 const MAX_ENDPOINTS_FROM_CONFIG: usize = 8;
 const DEFAULT_SEARCH_TIMEOUT_SECS: u64 = 15;
-const DEFAULT_FETCH_TIMEOUT_SECS: u64 = 20;
+const DEFAULT_TOR_TIMEOUT_SECS: u64 = 45;
 const MAX_PROVIDER_RESULTS: usize = 40;
 const MODULE_VERSION: &str = env!("CARGO_PKG_VERSION");
 const BROWSER_USER_AGENT: &str =
@@ -50,6 +66,16 @@ enum Commands {
         max_results: usize,
         #[arg(long)]
         endpoint: Option<String>,
+        #[arg(long, default_value_t = false)]
+        use_tor: bool,
+        #[arg(short, long)]
+        json: bool,
+    },
+    /// Search the web with anti-censorship transport preferences
+    TorSearch {
+        query: String,
+        #[arg(short, long, default_value_t = 5)]
+        max_results: usize,
         #[arg(short, long)]
         json: bool,
     },
@@ -64,6 +90,10 @@ enum Commands {
         endpoint: Option<String>,
         #[arg(long, default_value_t = true)]
         with_magnets: bool,
+        #[arg(long, value_delimiter = ',')]
+        providers: Vec<String>,
+        #[arg(long, default_value_t = false)]
+        use_tor: bool,
         #[arg(short, long)]
         json: bool,
     },
@@ -75,6 +105,11 @@ enum Commands {
         #[arg(short, long)]
         json: bool,
     },
+    /// Report current search transport and provider status
+    SearchStatus {
+        #[arg(short, long)]
+        json: bool,
+    },
     /// Print plugin metadata (draft)
     Manifest,
     /// Run MCP server over stdio (JSON-RPC)
@@ -113,9 +148,11 @@ struct SearchResult {
     score: Option<f64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     endpoint: Option<String>,
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    via: String,
 }
 
-#[derive(Debug, Serialize)]
+#[derive(Debug, Serialize, Clone)]
 struct TorrentSearchResult {
     title: String,
     url: String,
@@ -123,6 +160,26 @@ struct TorrentSearchResult {
     engine: String,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     magnet_links: Vec<String>,
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    provider: String,
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    via: String,
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    source_url: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    info_hash: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    size: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    size_bytes: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    seeds: Option<u32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    leeches: Option<u32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    uploader: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    category: Option<String>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -235,23 +292,93 @@ struct ArchiveDoc {
     description: Option<serde_json::Value>,
 }
 
+#[derive(Debug, Serialize)]
+struct SearchStatusSnapshot {
+    plugin_version: String,
+    tor: TorStatus,
+    doh: DohStatus,
+    magnet_cache: MagnetCacheStatus,
+    searxng_endpoints: Vec<SearxEndpointStatus>,
+    torrent_providers: Vec<ProviderStatus>,
+}
+
+#[derive(Debug, Serialize)]
+struct TorStatus {
+    available: bool,
+    port: Option<u16>,
+    mode: String,
+}
+
+#[derive(Debug, Serialize)]
+struct DohStatus {
+    primary: String,
+    fallback: String,
+}
+
+#[derive(Debug, Serialize)]
+struct MagnetCacheStatus {
+    entries: usize,
+    ttl_hours: u64,
+}
+
+#[derive(Debug, Serialize)]
+struct SearxEndpointStatus {
+    url: String,
+    successes: u32,
+    failures: u32,
+    cooldown_until: u64,
+    last_error: Option<String>,
+}
+
+#[derive(Debug)]
+struct DiscoveryState {
+    transport: TransportLayer,
+    doh: DohResolver,
+    cache: MagnetCache,
+    torrent_providers: Vec<TorrentProvider>,
+}
+
 static ENDPOINT_HEALTH: OnceLock<Mutex<HashMap<String, EndpointHealth>>> = OnceLock::new();
 
 fn endpoint_health_store() -> &'static Mutex<HashMap<String, EndpointHealth>> {
     ENDPOINT_HEALTH.get_or_init(|| Mutex::new(HashMap::new()))
 }
 
+impl DiscoveryState {
+    async fn new() -> Result<Self> {
+        let user_agent = format!("MasiXDiscovery/{MODULE_VERSION} transport");
+        let transport = TransportLayer::new(
+            &user_agent,
+            DEFAULT_SEARCH_TIMEOUT_SECS,
+            DEFAULT_TOR_TIMEOUT_SECS,
+        )
+        .await?;
+        let doh = DohResolver::new(&user_agent)?;
+        let cache = MagnetCache::new(default_data_dir().as_deref()).await?;
+        Ok(Self {
+            transport,
+            doh,
+            cache,
+            torrent_providers: bundled_providers(),
+        })
+    }
+}
+
 #[tokio::main]
 async fn main() -> Result<()> {
     let cli = Cli::parse();
+    let state = DiscoveryState::new().await?;
     match cli.command {
         Commands::WebSearch {
             query,
             max_results,
             endpoint,
+            use_tor,
             json,
         } => {
-            let results = broker_web_search(endpoint.as_deref(), &query, max_results).await?;
+            let results =
+                broker_web_search(&state, endpoint.as_deref(), &query, max_results, use_tor, None)
+                    .await?;
             if json {
                 println!("{}", serde_json::to_string_pretty(&results)?);
             } else if results.is_empty() {
@@ -282,21 +409,67 @@ async fn main() -> Result<()> {
                 }
             }
         }
+        Commands::TorSearch {
+            query,
+            max_results,
+            json,
+        } => {
+            let results = broker_web_search(&state, None, &query, max_results, true, None).await?;
+            if json {
+                println!("{}", serde_json::to_string_pretty(&results)?);
+            } else if results.is_empty() {
+                println!("No results found.");
+            } else {
+                for (i, item) in results.iter().enumerate() {
+                    println!(
+                        "{}. {} [{} | {} | {}]\n   {}\n   {}\n",
+                        i + 1,
+                        if item.title.trim().is_empty() {
+                            "(untitled)"
+                        } else {
+                            item.title.trim()
+                        },
+                        if item.engine.trim().is_empty() {
+                            "unknown"
+                        } else {
+                            item.engine.trim()
+                        },
+                        if item.provider.trim().is_empty() {
+                            "broker"
+                        } else {
+                            item.provider.trim()
+                        },
+                        if item.via.trim().is_empty() {
+                            "clearnet"
+                        } else {
+                            item.via.trim()
+                        },
+                        item.url.trim(),
+                        item.content.trim()
+                    );
+                }
+            }
+        }
         Commands::WebFetch { url } => {
-            println!("{}", web_fetch_page(&url).await?);
+            println!("{}", web_fetch_page(&url, &state.transport).await?);
         }
         Commands::TorrentSearch {
             query,
             max_results,
             endpoint,
             with_magnets,
+            providers,
+            use_tor,
             json,
         } => {
             let results = torrent_search(
+                &state,
                 endpoint.as_deref(),
                 &query,
                 max_results.min(20).max(1),
                 with_magnets,
+                &providers,
+                use_tor,
             )
             .await?;
             if json {
@@ -346,17 +519,32 @@ async fn main() -> Result<()> {
                 }
             }
         }
+        Commands::SearchStatus { json } => {
+            let status = search_status(&state).await;
+            if json {
+                println!("{}", serde_json::to_string_pretty(&status)?);
+            } else {
+                println!(
+                    "Discovery {} | tor={} port={:?} | cache_entries={} | providers={}",
+                    status.plugin_version,
+                    status.tor.available,
+                    status.tor.port,
+                    status.magnet_cache.entries,
+                    status.torrent_providers.len()
+                );
+            }
+        }
         Commands::Manifest => {
             println!("{}", include_str!("../plugin.manifest.json"));
         }
         Commands::ServeMcp => {
-            run_mcp_server().await?;
+            run_mcp_server(&state).await?;
         }
     }
     Ok(())
 }
 
-async fn run_mcp_server() -> Result<()> {
+async fn run_mcp_server(state: &DiscoveryState) -> Result<()> {
     let stdin = io::stdin();
     let mut stdout = io::stdout();
 
@@ -385,7 +573,7 @@ async fn run_mcp_server() -> Result<()> {
             }
         };
 
-        let response = handle_mcp_request(&request).await;
+        let response = handle_mcp_request(&request, state).await;
         writeln!(stdout, "{}", serde_json::to_string(&response)?)?;
         stdout.flush()?;
     }
@@ -393,7 +581,7 @@ async fn run_mcp_server() -> Result<()> {
     Ok(())
 }
 
-async fn handle_mcp_request(request: &JsonRpcRequest) -> JsonRpcResponse {
+async fn handle_mcp_request(request: &JsonRpcRequest, state: &DiscoveryState) -> JsonRpcResponse {
     match request.method.as_str() {
         "initialize" => JsonRpcResponse {
             jsonrpc: "2.0".to_string(),
@@ -434,7 +622,7 @@ async fn handle_mcp_request(request: &JsonRpcRequest) -> JsonRpcResponse {
                 .cloned()
                 .unwrap_or(serde_json::json!({}));
 
-            match handle_tool_call(tool_name, arguments).await {
+            match handle_tool_call(state, tool_name, arguments).await {
                 Ok(result) => JsonRpcResponse {
                     jsonrpc: "2.0".to_string(),
                     id: request.id.clone(),
@@ -488,6 +676,37 @@ fn get_tool_definitions() -> Vec<ToolDefinition> {
                         "type": "integer",
                         "description": "Maximum number of results (default: 5, max: 20)",
                         "default": 5
+                    },
+                    "use_tor": {
+                        "type": "boolean",
+                        "description": "Prefer Tor-capable routes when available; degrades to clearnet when Tor is unavailable",
+                        "default": false
+                    }
+                },
+                "required": ["query"]
+            }),
+        },
+        ToolDefinition {
+            name: "tor_search".to_string(),
+            description:
+                "Search using anti-censorship transport preferences. Falls back to clearnet if Tor is unavailable."
+                    .to_string(),
+            input_schema: serde_json::json!({
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "Search query"
+                    },
+                    "max_results": {
+                        "type": "integer",
+                        "description": "Maximum number of results (default: 5, max: 20)",
+                        "default": 5
+                    },
+                    "engines": {
+                        "type": "array",
+                        "items": { "type": "string" },
+                        "description": "Optional engine filter: ['ahmia', 'searxng', 'brave', 'duckduckgo', 'bing', 'wikipedia', 'news']. Omit to use all engines."
                     }
                 },
                 "required": ["query"]
@@ -528,6 +747,16 @@ fn get_tool_definitions() -> Vec<ToolDefinition> {
                         "type": "boolean",
                         "description": "Try to extract magnet links from each result URL (default: true)",
                         "default": true
+                    },
+                    "providers": {
+                        "type": "array",
+                        "items": { "type": "string" },
+                        "description": "Optional provider filter: 1337x, thepiratebay, yts, torrentgalaxy, eztv, kickass, limetorrents, solidtorrents, bt4g, torrentz2, nyaa, searx, archive"
+                    },
+                    "use_tor": {
+                        "type": "boolean",
+                        "description": "Prefer Tor-capable routes when available; degrades to clearnet when Tor is unavailable",
+                        "default": false
                     }
                 },
                 "required": ["query"]
@@ -552,10 +781,22 @@ fn get_tool_definitions() -> Vec<ToolDefinition> {
                 "required": ["url"]
             }),
         },
+        ToolDefinition {
+            name: "search_status".to_string(),
+            description: "Return Discovery transport, SearX health, and torrent provider capability status".to_string(),
+            input_schema: serde_json::json!({
+                "type": "object",
+                "properties": {}
+            }),
+        },
     ]
 }
 
-async fn handle_tool_call(name: &str, arguments: serde_json::Value) -> Result<ToolResult> {
+async fn handle_tool_call(
+    state: &DiscoveryState,
+    name: &str,
+    arguments: serde_json::Value,
+) -> Result<ToolResult> {
     match name {
         "web_search" => {
             let query = arguments
@@ -566,8 +807,46 @@ async fn handle_tool_call(name: &str, arguments: serde_json::Value) -> Result<To
                 .get("max_results")
                 .and_then(|v| v.as_u64())
                 .unwrap_or(5) as usize;
+            let use_tor = arguments
+                .get("use_tor")
+                .and_then(|v| v.as_bool())
+                .unwrap_or(false);
+
+            let results =
+                broker_web_search(state, None, query, max_results.min(20).max(1), use_tor, None).await?;
+            let text = serde_json::to_string_pretty(&results)?;
+
+            Ok(ToolResult {
+                content: vec![ToolContent {
+                    content_type: "text".to_string(),
+                    text,
+                }],
+                is_error: false,
+            })
+        }
+        "tor_search" => {
+            let query = arguments
+                .get("query")
+                .and_then(|v| v.as_str())
+                .ok_or_else(|| anyhow!("Missing 'query' parameter"))?;
+            let max_results = arguments
+                .get("max_results")
+                .and_then(|v| v.as_u64())
+                .unwrap_or(5) as usize;
+            let engines: Vec<String> = arguments
+                .get("engines")
+                .and_then(|v| v.as_array())
+                .map(|arr| {
+                    arr.iter()
+                        .filter_map(|e| e.as_str())
+                        .map(|s| s.to_lowercase())
+                        .collect()
+                })
+                .unwrap_or_default();
+            let engines_filter = if engines.is_empty() { None } else { Some(engines.as_slice()) };
 
-            let results = broker_web_search(None, query, max_results.min(20).max(1)).await?;
+            let results =
+                broker_web_search(state, None, query, max_results.min(20).max(1), true, engines_filter).await?;
             let text = serde_json::to_string_pretty(&results)?;
 
             Ok(ToolResult {
@@ -584,7 +863,7 @@ async fn handle_tool_call(name: &str, arguments: serde_json::Value) -> Result<To
                 .and_then(|v| v.as_str())
                 .ok_or_else(|| anyhow!("Missing 'url' parameter"))?;
 
-            let content = web_fetch_page(url).await?;
+            let content = web_fetch_page(url, &state.transport).await?;
 
             Ok(ToolResult {
                 content: vec![ToolContent {
@@ -607,9 +886,31 @@ async fn handle_tool_call(name: &str, arguments: serde_json::Value) -> Result<To
                 .get("with_magnets")
                 .and_then(|v| v.as_bool())
                 .unwrap_or(true);
+            let providers = arguments
+                .get("providers")
+                .and_then(|v| v.as_array())
+                .map(|items| {
+                    items
+                        .iter()
+                        .filter_map(|item| item.as_str().map(|s| s.to_string()))
+                        .collect::<Vec<_>>()
+                })
+                .unwrap_or_default();
+            let use_tor = arguments
+                .get("use_tor")
+                .and_then(|v| v.as_bool())
+                .unwrap_or(false);
 
-            let results =
-                torrent_search(None, query, max_results.min(20).max(1), with_magnets).await?;
+            let results = torrent_search(
+                state,
+                None,
+                query,
+                max_results.min(20).max(1),
+                with_magnets,
+                &providers,
+                use_tor,
+            )
+            .await?;
             let text = serde_json::to_string_pretty(&results)?;
 
             Ok(ToolResult {
@@ -641,18 +942,34 @@ async fn handle_tool_call(name: &str, arguments: serde_json::Value) -> Result<To
                 is_error: false,
             })
         }
+        "search_status" => {
+            let status = search_status(state).await;
+            let text = serde_json::to_string_pretty(&status)?;
+
+            Ok(ToolResult {
+                content: vec![ToolContent {
+                    content_type: "text".to_string(),
+                    text,
+                }],
+                is_error: false,
+            })
+        }
         _ => Err(anyhow!("Unknown tool: {}", name)),
     }
 }
 
 async fn broker_web_search(
+    state: &DiscoveryState,
     endpoint_override: Option<&str>,
     query: &str,
     max_results: usize,
+    use_tor: bool,
+    engines_filter: Option<&[String]>,
 ) -> Result<Vec<SearchResult>> {
     let max_results = max_results.min(20).max(1);
     let endpoints = resolve_searx_endpoints(endpoint_override);
-    let mut reports = collect_provider_reports(&endpoints, query, max_results).await;
+    let mut reports =
+        collect_provider_reports(state, &endpoints, query, max_results, use_tor, engines_filter).await;
     let mut merged = Vec::new();
     for report in &reports {
         merged.extend(report.items.clone());
@@ -667,7 +984,8 @@ async fn broker_web_search(
     if let Some(relaxed_query) = relax_search_query(query) {
         if relaxed_query != query {
             let retry_reports =
-                collect_provider_reports(&endpoints, &relaxed_query, max_results).await;
+                collect_provider_reports(state, &endpoints, &relaxed_query, max_results, use_tor, engines_filter)
+                    .await;
             let mut retry_merged = Vec::new();
             for report in &retry_reports {
                 retry_merged.extend(report.items.clone());
@@ -684,7 +1002,8 @@ async fn broker_web_search(
     if let Some(topic_query) = topic_focus_query(query) {
         if topic_query != query {
             let topic_reports =
-                collect_provider_reports(&endpoints, &topic_query, max_results).await;
+                collect_provider_reports(state, &endpoints, &topic_query, max_results, use_tor, engines_filter)
+                    .await;
             let mut topic_merged = Vec::new();
             for report in &topic_reports {
                 topic_merged.extend(report.items.clone());
@@ -724,31 +1043,67 @@ async fn broker_web_search(
     }
 }
 
+fn engine_allowed(filter: Option<&[String]>, name: &str) -> bool {
+    match filter {
+        None => true,
+        Some(list) if list.is_empty() => true,
+        Some(list) => list.iter().any(|e| e.eq_ignore_ascii_case(name)),
+    }
+}
+
 async fn collect_provider_reports(
+    state: &DiscoveryState,
     endpoints: &[String],
     query: &str,
     max_results: usize,
+    use_tor: bool,
+    engines_filter: Option<&[String]>,
 ) -> Vec<SearchProviderReport> {
-    let searx_future = searx_search_broker(
-        endpoints,
-        query,
-        (max_results * 3).min(MAX_PROVIDER_RESULTS),
-        MAX_SEARX_PARALLEL_ENDPOINTS,
-    );
-    let wiki_future = wikipedia_search(query, (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS));
-    let news_future = google_news_search(query, (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS));
-    let brave_future = brave_html_search(query, (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS));
-    let duckduckgo_future =
-        duckduckgo_html_search(query, (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS));
-    let bing_future = bing_rss_search(query, (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS));
-
-    let (searx, wiki, news, brave, duckduckgo, bing) = tokio::join!(
+    let sub = (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS);
+    let searx_future = async {
+        if engine_allowed(engines_filter, "searxng") {
+            searx_search_broker(endpoints, query, (max_results * 3).min(MAX_PROVIDER_RESULTS), MAX_SEARX_PARALLEL_ENDPOINTS).await
+        } else { Ok(Vec::new()) }
+    };
+    let wiki_future = async {
+        if engine_allowed(engines_filter, "wikipedia") {
+            wikipedia_search(query, sub).await
+        } else { Ok(Vec::new()) }
+    };
+    let news_future = async {
+        if engine_allowed(engines_filter, "news") {
+            google_news_search(query, sub).await
+        } else { Ok(Vec::new()) }
+    };
+    let brave_future = async {
+        if engine_allowed(engines_filter, "brave") {
+            brave_html_search(query, sub).await
+        } else { Ok(Vec::new()) }
+    };
+    let duckduckgo_future = async {
+        if engine_allowed(engines_filter, "duckduckgo") {
+            duckduckgo_html_search(query, sub).await
+        } else { Ok(Vec::new()) }
+    };
+    let bing_future = async {
+        if engine_allowed(engines_filter, "bing") {
+            bing_rss_search(query, sub).await
+        } else { Ok(Vec::new()) }
+    };
+    let ahmia_future = async {
+        if engine_allowed(engines_filter, "ahmia") {
+            ahmia_search(&state.transport, query, sub, use_tor).await
+        } else { Ok(Vec::new()) }
+    };
+
+    let (searx, wiki, news, brave, duckduckgo, bing, ahmia) = tokio::join!(
         searx_future,
         wiki_future,
         news_future,
         brave_future,
         duckduckgo_future,
-        bing_future
+        bing_future,
+        ahmia_future
     );
 
     vec![
@@ -758,6 +1113,7 @@ async fn collect_provider_reports(
         as_provider_report("brave-html", brave),
         as_provider_report("duckduckgo", duckduckgo),
         as_provider_report("bing-rss", bing),
+        as_provider_report("ahmia", ahmia),
     ]
 }
 
@@ -1214,6 +1570,7 @@ async fn web_search_single_endpoint(
                             source_domain: source_domain(&item.url),
                             score: None,
                             endpoint: Some(endpoint.to_string()),
+                            via: "searx".to_string(),
                         })
                         .collect::<Vec<_>>();
                     return Ok(mapped);
@@ -1324,6 +1681,7 @@ async fn wikipedia_search(query: &str, max_results: usize) -> Result<Vec<SearchR
                 source_domain: source_domain(&page_url),
                 score: None,
                 endpoint: None,
+                via: "clearnet".to_string(),
             }
         })
         .collect::<Vec<_>>();
@@ -1424,6 +1782,7 @@ async fn google_news_region_search(
             source_domain: source_domain(&item.link),
             score: None,
             endpoint: None,
+            via: "clearnet".to_string(),
         })
         .collect::<Vec<_>>())
 }
@@ -1506,6 +1865,7 @@ async fn brave_html_search(query: &str, max_results: usize) -> Result<Vec<Search
             source_domain: source_domain(href),
             score: None,
             endpoint: Some("https://search.brave.com/search".to_string()),
+            via: "clearnet".to_string(),
         });
     }
 
@@ -1576,6 +1936,7 @@ async fn duckduckgo_html_search(query: &str, max_results: usize) -> Result<Vec<S
                     source_domain: source_domain(&url),
                     score: None,
                     endpoint: Some("https://lite.duckduckgo.com/lite".to_string()),
+                    via: "clearnet".to_string(),
                 });
             }
             if !results.is_empty() {
@@ -1647,6 +2008,7 @@ async fn duckduckgo_html_search(query: &str, max_results: usize) -> Result<Vec<S
             source_domain: source_domain(&url),
             score: None,
             endpoint: Some("https://duckduckgo.com/html".to_string()),
+            via: "clearnet".to_string(),
         });
     }
 
@@ -1673,6 +2035,7 @@ async fn duckduckgo_html_search(query: &str, max_results: usize) -> Result<Vec<S
                 source_domain: source_domain(&url),
                 score: None,
                 endpoint: Some("https://duckduckgo.com/html".to_string()),
+                via: "clearnet".to_string(),
             });
         }
     }
@@ -1751,10 +2114,74 @@ async fn bing_rss_search(query: &str, max_results: usize) -> Result<Vec<SearchRe
             source_domain: source_domain(&item.link),
             score: None,
             endpoint: Some("https://www.bing.com/search?format=rss".to_string()),
+            via: "clearnet".to_string(),
         })
         .collect::<Vec<_>>())
 }
 
+async fn ahmia_search(
+    transport: &TransportLayer,
+    query: &str,
+    max_results: usize,
+    use_tor: bool,
+) -> Result<Vec<SearchResult>> {
+    let (client, via, base_url) = transport.client_for_ahmia(use_tor);
+    let response = client.get(base_url).query(&[("q", query)]).send().await?;
+    if !response.status().is_success() {
+        return Err(anyhow!("ahmia provider failed: HTTP {}", response.status()));
+    }
+    let html = response.text().await?;
+    let document = Html::parse_document(&html);
+    let result_selector = Selector::parse(".result, .search-result, li")
+        .map_err(|e| anyhow!("Invalid Ahmia result selector: {}", e))?;
+    let link_selector =
+        Selector::parse("a").map_err(|e| anyhow!("Invalid Ahmia link selector: {}", e))?;
+
+    let mut output = Vec::new();
+    for result in document.select(&result_selector) {
+        if output.len() >= max_results.min(MAX_PROVIDER_RESULTS) {
+            break;
+        }
+        for link in result.select(&link_selector) {
+            let href = link.value().attr("href").unwrap_or("").trim();
+            let title = link.text().collect::<String>().trim().to_string();
+            if href.is_empty() || title.len() < 3 {
+                continue;
+            }
+            let resolved = if href.starts_with("http://") || href.starts_with("https://") {
+                href.to_string()
+            } else {
+                format!("{}{}", base_url.trim_end_matches('/'), href)
+            };
+            let content = result
+                .text()
+                .collect::<Vec<_>>()
+                .join(" ")
+                .split_whitespace()
+                .collect::<Vec<_>>()
+                .join(" ");
+            output.push(SearchResult {
+                title,
+                url: resolved.clone(),
+                content: truncate_text(&content, 400),
+                engine: "ahmia".to_string(),
+                provider: "ahmia".to_string(),
+                source_domain: source_domain(&resolved),
+                score: None,
+                endpoint: Some(base_url.to_string()),
+                via: via.to_string(),
+            });
+            break;
+        }
+    }
+
+    if output.is_empty() {
+        Err(anyhow!("ahmia provider returned no results"))
+    } else {
+        Ok(output)
+    }
+}
+
 async fn direct_domain_probe(query: &str, max_results: usize) -> Result<Vec<SearchResult>> {
     let mut tokens = query
         .split_whitespace()
@@ -1813,6 +2240,7 @@ async fn direct_domain_probe(query: &str, max_results: usize) -> Result<Vec<Sear
             source_domain: source_domain(&url),
             score: None,
             endpoint: None,
+            via: "clearnet".to_string(),
         });
     }
 
@@ -2107,11 +2535,8 @@ fn now_unix_secs() -> u64 {
         .unwrap_or(0)
 }
 
-async fn web_fetch_page(url: &str) -> Result<String> {
-    let client = reqwest::Client::builder()
-        .user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36")
-        .timeout(Duration::from_secs(DEFAULT_FETCH_TIMEOUT_SECS))
-        .build()?;
+async fn web_fetch_page(url: &str, transport: &TransportLayer) -> Result<String> {
+    let (client, _via) = transport.client_for(url);
     let response = client.get(url).send().await?;
     if !response.status().is_success() {
         return Err(anyhow!("fetch failed: HTTP {}", response.status()));
@@ -2165,10 +2590,13 @@ async fn web_fetch_page(url: &str) -> Result<String> {
 }
 
 async fn torrent_search(
+    state: &DiscoveryState,
     endpoint_override: Option<&str>,
     query: &str,
     max_results: usize,
     with_magnets: bool,
+    providers: &[String],
+    use_tor: bool,
 ) -> Result<Vec<TorrentSearchResult>> {
     let normalized_query = normalize_torrent_query(query);
     let direct_query = query.trim();
@@ -2178,21 +2606,96 @@ async fn torrent_search(
         direct_query
     };
     let endpoints = resolve_searx_endpoints(endpoint_override);
+    let allowed = normalize_provider_filter(providers);
+    let wants_all = allowed.is_empty();
+
+    let mirror_future = async {
+        if wants_all || has_provider(&allowed, "mirror") {
+            search_mirror_catalog(
+                &state.torrent_providers,
+                &state.transport,
+                &state.doh,
+                Some(&state.cache),
+                direct_query,
+                max_results * 2,
+                if allowed.is_empty() {
+                    None
+                } else {
+                    Some(&allowed)
+                },
+                use_tor,
+            )
+            .await
+        } else {
+            Ok(Vec::new())
+        }
+    };
+    let searx_future = async {
+        if wants_all || has_provider(&allowed, "searx") {
+            searx_search_broker(
+                &endpoints,
+                &normalized_query,
+                (max_results * 2).min(MAX_PROVIDER_RESULTS),
+                MAX_SEARX_PARALLEL_ENDPOINTS,
+            )
+            .await
+        } else {
+            Ok(Vec::new())
+        }
+    };
+    let nyaa_future = async {
+        if wants_all || has_provider(&allowed, "nyaa") {
+            nyaa_torrent_search(direct_query, max_results * 2).await
+        } else {
+            Ok(Vec::new())
+        }
+    };
+    let archive_future = async {
+        if wants_all || has_provider(&allowed, "archive") {
+            archive_torrent_search(direct_query, max_results * 2).await
+        } else {
+            Ok(Vec::new())
+        }
+    };
 
-    let searx_future = searx_search_broker(
-        &endpoints,
-        &normalized_query,
-        (max_results * 2).min(MAX_PROVIDER_RESULTS),
-        MAX_SEARX_PARALLEL_ENDPOINTS,
-    );
-    let nyaa_future = nyaa_torrent_search(direct_query, max_results * 2);
-    let archive_future = archive_torrent_search(direct_query, max_results * 2);
-
-    let (searx, nyaa, archive) = tokio::join!(searx_future, nyaa_future, archive_future);
+    let (mirror, searx, nyaa, archive) =
+        tokio::join!(mirror_future, searx_future, nyaa_future, archive_future);
 
     let mut output = Vec::new();
     let mut provider_errors = Vec::new();
 
+    match mirror {
+        Ok(items) => {
+            for item in items {
+                output.push(TorrentSearchResult {
+                    title: item.title,
+                    url: item.url,
+                    content: item.content,
+                    engine: "mirror-catalog".to_string(),
+                    magnet_links: if with_magnets {
+                        item.magnet_links.clone()
+                    } else {
+                        Vec::new()
+                    },
+                    provider: item.provider,
+                    via: item.via,
+                    source_url: item.source_url,
+                    info_hash: item
+                        .magnet_links
+                        .iter()
+                        .find_map(|magnet| extract_info_hash(magnet)),
+                    size_bytes: item.size.as_deref().and_then(parse_size_bytes),
+                    size: item.size,
+                    seeds: item.seeds,
+                    leeches: item.leeches,
+                    uploader: item.uploader,
+                    category: item.category,
+                });
+            }
+        }
+        Err(e) => provider_errors.push(format!("mirror: {}", e)),
+    }
+
     match nyaa {
         Ok(mut items) => output.append(&mut items),
         Err(e) => provider_errors.push(format!("nyaa: {}", e)),
@@ -2218,6 +2721,20 @@ async fn torrent_search(
                         result.engine
                     },
                     magnet_links,
+                    provider: "searx".to_string(),
+                    via: if result.via.trim().is_empty() {
+                        "searx".to_string()
+                    } else {
+                        result.via
+                    },
+                    source_url: result.endpoint.unwrap_or_else(|| "searx".to_string()),
+                    info_hash: None,
+                    size: None,
+                    size_bytes: None,
+                    seeds: None,
+                    leeches: None,
+                    uploader: None,
+                    category: None,
                 });
             }
         }
@@ -2254,7 +2771,15 @@ fn dedup_torrent_results(
     let mut deduped = Vec::with_capacity(results.len());
 
     for mut item in results.drain(..) {
-        let key = normalize_url_key(&item.url);
+        let key = item
+            .info_hash
+            .clone()
+            .or_else(|| {
+                item.magnet_links
+                    .first()
+                    .map(|value| sanitize_magnet(value))
+            })
+            .unwrap_or_else(|| normalize_url_key(&item.url));
         if key.is_empty() || !seen.insert(key) {
             continue;
         }
@@ -2276,7 +2801,9 @@ fn dedup_torrent_results(
 
 fn torrent_engine_rank(engine: &str) -> u8 {
     let lowered = engine.to_lowercase();
-    if lowered.contains("nyaa") {
+    if lowered.contains("mirror") {
+        5
+    } else if lowered.contains("nyaa") {
         4
     } else if lowered.contains("searx") {
         3
@@ -2360,6 +2887,16 @@ async fn nyaa_torrent_search(query: &str, max_results: usize) -> Result<Vec<Torr
             },
             engine: "nyaa-html".to_string(),
             magnet_links: magnets,
+            provider: "nyaa".to_string(),
+            via: "clearnet".to_string(),
+            source_url: url.clone(),
+            info_hash: None,
+            size: None,
+            size_bytes: None,
+            seeds: None,
+            leeches: None,
+            uploader: None,
+            category: None,
         });
     }
 
@@ -2419,6 +2956,16 @@ async fn archive_torrent_search(
             },
             engine: "archive-search".to_string(),
             magnet_links: Vec::new(),
+            provider: "archive".to_string(),
+            via: "clearnet".to_string(),
+            source_url: url.clone(),
+            info_hash: None,
+            size: None,
+            size_bytes: None,
+            seeds: None,
+            leeches: None,
+            uploader: None,
+            category: None,
         });
     }
 
@@ -2445,6 +2992,102 @@ fn archive_description_to_string(value: Option<serde_json::Value>) -> String {
     }
 }
 
+fn normalize_provider_filter(providers: &[String]) -> HashSet<String> {
+    providers
+        .iter()
+        .map(|value| value.trim().to_lowercase())
+        .filter(|value| !value.is_empty())
+        .collect()
+}
+
+fn has_provider(providers: &HashSet<String>, provider: &str) -> bool {
+    providers.contains(provider) || providers.contains(&provider.to_lowercase())
+}
+
+async fn search_status(state: &DiscoveryState) -> SearchStatusSnapshot {
+    let endpoint_map = endpoint_health_store().lock().await.clone();
+    let mut searxng_endpoints = resolve_searx_endpoints(None)
+        .into_iter()
+        .map(|endpoint| {
+            let stats = endpoint_map.get(&endpoint).cloned().unwrap_or_default();
+            SearxEndpointStatus {
+                url: endpoint,
+                successes: stats.successes,
+                failures: stats.failures,
+                cooldown_until: stats.cooldown_until,
+                last_error: stats.last_error,
+            }
+        })
+        .collect::<Vec<_>>();
+    searxng_endpoints.sort_by(|a, b| a.url.cmp(&b.url));
+
+    SearchStatusSnapshot {
+        plugin_version: MODULE_VERSION.to_string(),
+        tor: TorStatus {
+            available: state.transport.tor_available(),
+            port: state.transport.tor_port(),
+            mode: "auto-fallback".to_string(),
+        },
+        doh: DohStatus {
+            primary: "cloudflare-dns.com".to_string(),
+            fallback: "dns.google".to_string(),
+        },
+        magnet_cache: MagnetCacheStatus {
+            entries: state.cache.size().await,
+            ttl_hours: MagnetCache::ttl_hours(),
+        },
+        searxng_endpoints,
+        torrent_providers: provider_statuses(&state.torrent_providers),
+    }
+}
+
+fn default_data_dir() -> Option<PathBuf> {
+    if let Ok(value) = std::env::var("MASIX_DATA_DIR") {
+        let trimmed = value.trim();
+        if !trimmed.is_empty() {
+            return Some(PathBuf::from(trimmed));
+        }
+    }
+    dirs::home_dir().map(|home| home.join(".masix"))
+}
+
+fn parse_size_bytes(value: &str) -> Option<u64> {
+    let normalized = value.trim().replace(',', ".");
+    let mut number = String::new();
+    let mut unit = String::new();
+    for ch in normalized.chars() {
+        if ch.is_ascii_digit() || ch == '.' {
+            number.push(ch);
+        } else if !ch.is_whitespace() {
+            unit.push(ch);
+        }
+    }
+    let base = number.parse::<f64>().ok()?;
+    let multiplier = match unit.to_lowercase().as_str() {
+        "kb" | "kib" => 1024_f64,
+        "mb" | "mib" => 1024_f64.powi(2),
+        "gb" | "gib" => 1024_f64.powi(3),
+        "tb" | "tib" => 1024_f64.powi(4),
+        "" => 1_f64,
+        _ => return None,
+    };
+    Some((base * multiplier) as u64)
+}
+
+fn extract_info_hash(magnet: &str) -> Option<String> {
+    let lower = magnet.to_lowercase();
+    let needle = "xt=urn:btih:";
+    let start = lower.find(needle)?;
+    let suffix = &magnet[start + needle.len()..];
+    let end = suffix.find('&').unwrap_or(suffix.len());
+    let hash = suffix[..end].trim();
+    if hash.is_empty() {
+        None
+    } else {
+        Some(hash.to_string())
+    }
+}
+
 fn truncate_text(value: &str, max_chars: usize) -> String {
     if value.chars().count() <= max_chars {
         return value.to_string();
@@ -2568,3 +3211,80 @@ fn url_encode(value: &str) -> String {
     }
     out
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn normalize_provider_filter_is_case_insensitive() {
+        let items = normalize_provider_filter(&[
+            "Nyaa".to_string(),
+            "  SeArX ".to_string(),
+            "mirror".to_string(),
+        ]);
+        assert!(items.contains("nyaa"));
+        assert!(items.contains("searx"));
+        assert!(items.contains("mirror"));
+    }
+
+    #[test]
+    fn parse_size_bytes_supports_common_units() {
+        assert_eq!(parse_size_bytes("1.5 GB"), Some(1610612736));
+        assert_eq!(parse_size_bytes("700MB"), Some(734003200));
+        assert_eq!(parse_size_bytes("42"), Some(42));
+        assert_eq!(parse_size_bytes("unknown"), None);
+    }
+
+    #[test]
+    fn extract_info_hash_reads_btih() {
+        let magnet = "magnet:?xt=urn:btih:ABCDEF1234567890&dn=test";
+        assert_eq!(
+            extract_info_hash(magnet).as_deref(),
+            Some("ABCDEF1234567890")
+        );
+    }
+
+    #[test]
+    fn dedup_torrent_results_prefers_info_hash_key() {
+        let results = vec![
+            TorrentSearchResult {
+                title: "A".to_string(),
+                url: "https://example.com/a".to_string(),
+                content: String::new(),
+                engine: "mirror-catalog".to_string(),
+                magnet_links: vec!["magnet:?xt=urn:btih:HASH1".to_string()],
+                provider: "mirror".to_string(),
+                via: "clearnet".to_string(),
+                source_url: "https://example.com/search".to_string(),
+                info_hash: Some("HASH1".to_string()),
+                size: None,
+                size_bytes: None,
+                seeds: Some(10),
+                leeches: None,
+                uploader: None,
+                category: None,
+            },
+            TorrentSearchResult {
+                title: "B".to_string(),
+                url: "https://example.com/b".to_string(),
+                content: String::new(),
+                engine: "mirror-catalog".to_string(),
+                magnet_links: vec!["magnet:?xt=urn:btih:HASH1".to_string()],
+                provider: "mirror".to_string(),
+                via: "proxy".to_string(),
+                source_url: "https://example.com/search".to_string(),
+                info_hash: Some("HASH1".to_string()),
+                size: None,
+                size_bytes: None,
+                seeds: Some(8),
+                leeches: None,
+                uploader: None,
+                category: None,
+            },
+        ];
+
+        let deduped = dedup_torrent_results(results, true);
+        assert_eq!(deduped.len(), 1);
+    }
+}