@mmmbuto/masix 0.4.0 → 0.4.2

package/packages/plugin-base/discovery/source/src/main.rs

@@ -0,0 +1,2570 @@
+use anyhow::{anyhow, Result};
+use clap::{Parser, Subcommand};
+use scraper::{Html, Selector};
+use serde::{Deserialize, Serialize};
+use std::cmp::Ordering;
+use std::collections::{HashMap, HashSet};
+use std::io::{self, BufRead, Write};
+use std::sync::OnceLock;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use tokio::sync::Mutex;
+
+const DEFAULT_SEARXNG_URLS: &[&str] = &[
+    "https://search.inetol.net",
+    "https://searx.work",
+    "https://search.privacyredirect.com",
+];
+const MAX_WEB_CONTENT: usize = 15_000;
+const SEARX_RETRIES: usize = 3;
+const MAX_SEARX_PARALLEL_ENDPOINTS: usize = 3;
+const MAX_ENDPOINTS_FROM_CONFIG: usize = 8;
+const DEFAULT_SEARCH_TIMEOUT_SECS: u64 = 15;
+const DEFAULT_FETCH_TIMEOUT_SECS: u64 = 20;
+const MAX_PROVIDER_RESULTS: usize = 40;
+const MODULE_VERSION: &str = env!("CARGO_PKG_VERSION");
+const BROWSER_USER_AGENT: &str =
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0 Safari/537.36";
+const NEWS_REGIONS: &[(&str, &str, &str)] = &[
+    ("US", "en-US", "US:en"),
+    ("GB", "en-GB", "GB:en"),
+    ("IT", "it-IT", "IT:it"),
+    ("DE", "de-DE", "DE:de"),
+    ("ES", "es-ES", "ES:es"),
+    ("JP", "ja", "JP:ja"),
+];
+
+#[derive(Parser)]
+#[command(name = "masix-plugin-discovery")]
+#[command(about = "External discovery module for MasiX (web search + fetch)")]
+struct Cli {
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+    /// Search the web using autonomous multi-engine broker
+    WebSearch {
+        query: String,
+        #[arg(short, long, default_value_t = 5)]
+        max_results: usize,
+        #[arg(long)]
+        endpoint: Option<String>,
+        #[arg(short, long)]
+        json: bool,
+    },
+    /// Fetch and extract text content from a web page
+    WebFetch { url: String },
+    /// Search torrent metadata pages (lawful content only)
+    TorrentSearch {
+        query: String,
+        #[arg(short, long, default_value_t = 5)]
+        max_results: usize,
+        #[arg(long)]
+        endpoint: Option<String>,
+        #[arg(long, default_value_t = true)]
+        with_magnets: bool,
+        #[arg(short, long)]
+        json: bool,
+    },
+    /// Extract magnet links from a web page URL
+    TorrentExtract {
+        url: String,
+        #[arg(short, long, default_value_t = 5)]
+        max_links: usize,
+        #[arg(short, long)]
+        json: bool,
+    },
+    /// Print plugin metadata (draft)
+    Manifest,
+    /// Run MCP server over stdio (JSON-RPC)
+    ServeMcp,
+}
+
+#[derive(Debug, Deserialize)]
+struct SearxResponse {
+    #[serde(default)]
+    results: Vec<SearxResultRaw>,
+}
+
+#[derive(Debug, Deserialize)]
+struct SearxResultRaw {
+    #[serde(default)]
+    title: String,
+    #[serde(default)]
+    url: String,
+    #[serde(default, alias = "content")]
+    content: String,
+    #[serde(default)]
+    engine: String,
+}
+
+#[derive(Debug, Serialize, Clone)]
+struct SearchResult {
+    title: String,
+    url: String,
+    content: String,
+    engine: String,
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    provider: String,
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    source_domain: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    score: Option<f64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    endpoint: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+struct TorrentSearchResult {
+    title: String,
+    url: String,
+    content: String,
+    engine: String,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    magnet_links: Vec<String>,
+}
+
+#[derive(Debug, Deserialize)]
+struct JsonRpcRequest {
+    #[allow(dead_code)]
+    jsonrpc: String,
+    id: Option<serde_json::Value>,
+    method: String,
+    #[serde(default)]
+    params: serde_json::Value,
+}
+
+#[derive(Debug, Serialize)]
+struct JsonRpcResponse {
+    jsonrpc: String,
+    id: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    result: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    error: Option<JsonRpcError>,
+}
+
+#[derive(Debug, Serialize)]
+struct JsonRpcError {
+    code: i32,
+    message: String,
+}
+
+#[derive(Debug, Serialize)]
+struct ToolDefinition {
+    name: String,
+    description: String,
+    input_schema: serde_json::Value,
+}
+
+#[derive(Debug, Serialize)]
+struct ToolResult {
+    content: Vec<ToolContent>,
+    #[serde(skip_serializing_if = "is_false")]
+    is_error: bool,
+}
+
+fn is_false(v: &bool) -> bool {
+    !v
+}
+
+#[derive(Debug, Serialize)]
+struct ToolContent {
+    #[serde(rename = "type")]
+    content_type: String,
+    text: String,
+}
+
+#[derive(Debug, Clone, Default)]
+struct EndpointHealth {
+    successes: u32,
+    failures: u32,
+    cooldown_until: u64,
+    last_error: Option<String>,
+}
+
+#[derive(Debug, Clone)]
+struct SearchProviderReport {
+    provider: &'static str,
+    items: Vec<SearchResult>,
+    error: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+struct WikipediaResponse {
+    #[serde(default)]
+    query: Option<WikipediaQuery>,
+}
+
+#[derive(Debug, Deserialize)]
+struct WikipediaQuery {
+    #[serde(default)]
+    search: Vec<WikipediaEntry>,
+}
+
+#[derive(Debug, Deserialize)]
+struct WikipediaEntry {
+    #[serde(default)]
+    title: String,
+    #[serde(default)]
+    snippet: String,
+    #[serde(default)]
+    pageid: u64,
+}
+
+#[derive(Debug, Deserialize)]
+struct ArchiveResponse {
+    #[serde(default)]
+    response: ArchiveInnerResponse,
+}
+
+#[derive(Debug, Deserialize, Default)]
+struct ArchiveInnerResponse {
+    #[serde(default)]
+    docs: Vec<ArchiveDoc>,
+}
+
+#[derive(Debug, Deserialize, Default)]
+struct ArchiveDoc {
+    #[serde(default)]
+    identifier: String,
+    #[serde(default)]
+    title: Option<String>,
+    #[serde(default)]
+    description: Option<serde_json::Value>,
+}
+
+static ENDPOINT_HEALTH: OnceLock<Mutex<HashMap<String, EndpointHealth>>> = OnceLock::new();
+
+fn endpoint_health_store() -> &'static Mutex<HashMap<String, EndpointHealth>> {
+    ENDPOINT_HEALTH.get_or_init(|| Mutex::new(HashMap::new()))
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let cli = Cli::parse();
+    match cli.command {
+        Commands::WebSearch {
+            query,
+            max_results,
+            endpoint,
+            json,
+        } => {
+            let results = broker_web_search(endpoint.as_deref(), &query, max_results).await?;
+            if json {
+                println!("{}", serde_json::to_string_pretty(&results)?);
+            } else if results.is_empty() {
+                println!("No results found.");
+            } else {
+                for (i, item) in results.iter().enumerate() {
+                    println!(
+                        "{}. {} [{} | {}]\n   {}\n   {}\n",
+                        i + 1,
+                        if item.title.trim().is_empty() {
+                            "(untitled)"
+                        } else {
+                            item.title.trim()
+                        },
+                        if item.engine.trim().is_empty() {
+                            "unknown"
+                        } else {
+                            item.engine.trim()
+                        },
+                        if item.provider.trim().is_empty() {
+                            "broker"
+                        } else {
+                            item.provider.trim()
+                        },
+                        item.url.trim(),
+                        item.content.trim()
+                    );
+                }
+            }
+        }
+        Commands::WebFetch { url } => {
+            println!("{}", web_fetch_page(&url).await?);
+        }
+        Commands::TorrentSearch {
+            query,
+            max_results,
+            endpoint,
+            with_magnets,
+            json,
+        } => {
+            let results = torrent_search(
+                endpoint.as_deref(),
+                &query,
+                max_results.min(20).max(1),
+                with_magnets,
+            )
+            .await?;
+            if json {
+                println!("{}", serde_json::to_string_pretty(&results)?);
+            } else if results.is_empty() {
+                println!("No torrent results found.");
+            } else {
+                for (idx, item) in results.iter().enumerate() {
+                    println!(
+                        "{}. {} [{}]\n   {}\n   {}\n",
+                        idx + 1,
+                        if item.title.trim().is_empty() {
+                            "(untitled)"
+                        } else {
+                            item.title.trim()
+                        },
+                        if item.engine.trim().is_empty() {
+                            "unknown"
+                        } else {
+                            item.engine.trim()
+                        },
+                        item.url.trim(),
+                        item.content.trim()
+                    );
+                    if !item.magnet_links.is_empty() {
+                        for magnet in &item.magnet_links {
+                            println!("   magnet: {}", magnet);
+                        }
+                        println!();
+                    }
+                }
+            }
+        }
+        Commands::TorrentExtract {
+            url,
+            max_links,
+            json,
+        } => {
+            let links = extract_magnet_links(&url, max_links.min(20).max(1)).await?;
+            if json {
+                println!("{}", serde_json::to_string_pretty(&links)?);
+            } else if links.is_empty() {
+                println!("No magnet links found.");
+            } else {
+                for (idx, link) in links.iter().enumerate() {
+                    println!("{}. {}", idx + 1, link);
+                }
+            }
+        }
+        Commands::Manifest => {
+            println!("{}", include_str!("../plugin.manifest.json"));
+        }
+        Commands::ServeMcp => {
+            run_mcp_server().await?;
+        }
+    }
+    Ok(())
+}
+
+async fn run_mcp_server() -> Result<()> {
+    let stdin = io::stdin();
+    let mut stdout = io::stdout();
+
+    for line in stdin.lock().lines() {
+        let line = line?;
+        let line = line.trim();
+        if line.is_empty() {
+            continue;
+        }
+
+        let request: JsonRpcRequest = match serde_json::from_str(line) {
+            Ok(req) => req,
+            Err(e) => {
+                let response = JsonRpcResponse {
+                    jsonrpc: "2.0".to_string(),
+                    id: None,
+                    result: None,
+                    error: Some(JsonRpcError {
+                        code: -32700,
+                        message: format!("Parse error: {}", e),
+                    }),
+                };
+                writeln!(stdout, "{}", serde_json::to_string(&response)?)?;
+                stdout.flush()?;
+                continue;
+            }
+        };
+
+        let response = handle_mcp_request(&request).await;
+        writeln!(stdout, "{}", serde_json::to_string(&response)?)?;
+        stdout.flush()?;
+    }
+
+    Ok(())
+}
+
+async fn handle_mcp_request(request: &JsonRpcRequest) -> JsonRpcResponse {
+    match request.method.as_str() {
+        "initialize" => JsonRpcResponse {
+            jsonrpc: "2.0".to_string(),
+            id: request.id.clone(),
+            result: Some(serde_json::json!({
+                "protocolVersion": "2024-11-05",
+                "capabilities": {
+                    "tools": {}
+                },
+                "serverInfo": {
+                    "name": "masix-discovery",
+                    "version": env!("CARGO_PKG_VERSION")
+                }
+            })),
+            error: None,
+        },
+        "notifications/initialized" => JsonRpcResponse {
+            jsonrpc: "2.0".to_string(),
+            id: None,
+            result: None,
+            error: None,
+        },
+        "tools/list" => {
+            let tools = get_tool_definitions();
+            JsonRpcResponse {
+                jsonrpc: "2.0".to_string(),
+                id: request.id.clone(),
+                result: Some(serde_json::json!({ "tools": tools })),
+                error: None,
+            }
+        }
+        "tools/call" => {
+            let params = &request.params;
+            let tool_name = params.get("name").and_then(|v| v.as_str()).unwrap_or("");
+
+            let arguments = params
+                .get("arguments")
+                .cloned()
+                .unwrap_or(serde_json::json!({}));
+
+            match handle_tool_call(tool_name, arguments).await {
+                Ok(result) => JsonRpcResponse {
+                    jsonrpc: "2.0".to_string(),
+                    id: request.id.clone(),
+                    result: Some(serde_json::to_value(result).unwrap_or(serde_json::json!({}))),
+                    error: None,
+                },
+                Err(e) => JsonRpcResponse {
+                    jsonrpc: "2.0".to_string(),
+                    id: request.id.clone(),
+                    result: Some(
+                        serde_json::to_value(ToolResult {
+                            content: vec![ToolContent {
+                                content_type: "text".to_string(),
+                                text: format!("Error: {}", e),
+                            }],
+                            is_error: true,
+                        })
+                        .unwrap_or(serde_json::json!({})),
+                    ),
+                    error: None,
+                },
+            }
+        }
+        _ => JsonRpcResponse {
+            jsonrpc: "2.0".to_string(),
+            id: request.id.clone(),
+            result: None,
+            error: Some(JsonRpcError {
+                code: -32601,
+                message: format!("Method not found: {}", request.method),
+            }),
+        },
+    }
+}
+
+fn get_tool_definitions() -> Vec<ToolDefinition> {
+    vec![
+        ToolDefinition {
+            name: "web_search".to_string(),
+            description:
+                "Search the web using autonomous multi-engine broker (SearXNG + direct sources)."
+                    .to_string(),
+            input_schema: serde_json::json!({
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "Search query"
+                    },
+                    "max_results": {
+                        "type": "integer",
+                        "description": "Maximum number of results (default: 5, max: 20)",
+                        "default": 5
+                    }
+                },
+                "required": ["query"]
+            }),
+        },
+        ToolDefinition {
+            name: "web_fetch".to_string(),
+            description: "Fetch and extract text content from a web page".to_string(),
+            input_schema: serde_json::json!({
+                "type": "object",
+                "properties": {
+                    "url": {
+                        "type": "string",
+                        "description": "URL to fetch"
+                    }
+                },
+                "required": ["url"]
+            }),
+        },
+        ToolDefinition {
+            name: "torrent_search".to_string(),
+            description:
+                "Search torrent metadata pages (lawful use only), with optional magnet extraction"
+                    .to_string(),
+            input_schema: serde_json::json!({
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "Search query for torrent metadata"
+                    },
+                    "max_results": {
+                        "type": "integer",
+                        "description": "Maximum number of results (default: 5, max: 20)",
+                        "default": 5
+                    },
+                    "with_magnets": {
+                        "type": "boolean",
+                        "description": "Try to extract magnet links from each result URL (default: true)",
+                        "default": true
+                    }
+                },
+                "required": ["query"]
+            }),
+        },
+        ToolDefinition {
+            name: "torrent_extract".to_string(),
+            description: "Extract magnet links from a page URL".to_string(),
+            input_schema: serde_json::json!({
+                "type": "object",
+                "properties": {
+                    "url": {
+                        "type": "string",
+                        "description": "Page URL or magnet URL"
+                    },
+                    "max_links": {
+                        "type": "integer",
+                        "description": "Maximum links to return (default: 5, max: 20)",
+                        "default": 5
+                    }
+                },
+                "required": ["url"]
+            }),
+        },
+    ]
+}
+
+async fn handle_tool_call(name: &str, arguments: serde_json::Value) -> Result<ToolResult> {
+    match name {
+        "web_search" => {
+            let query = arguments
+                .get("query")
+                .and_then(|v| v.as_str())
+                .ok_or_else(|| anyhow!("Missing 'query' parameter"))?;
+            let max_results = arguments
+                .get("max_results")
+                .and_then(|v| v.as_u64())
+                .unwrap_or(5) as usize;
+
+            let results = broker_web_search(None, query, max_results.min(20).max(1)).await?;
+            let text = serde_json::to_string_pretty(&results)?;
+
+            Ok(ToolResult {
+                content: vec![ToolContent {
+                    content_type: "text".to_string(),
+                    text,
+                }],
+                is_error: false,
+            })
+        }
+        "web_fetch" => {
+            let url = arguments
+                .get("url")
+                .and_then(|v| v.as_str())
+                .ok_or_else(|| anyhow!("Missing 'url' parameter"))?;
+
+            let content = web_fetch_page(url).await?;
+
+            Ok(ToolResult {
+                content: vec![ToolContent {
+                    content_type: "text".to_string(),
+                    text: content,
+                }],
+                is_error: false,
+            })
+        }
+        "torrent_search" => {
+            let query = arguments
+                .get("query")
+                .and_then(|v| v.as_str())
+                .ok_or_else(|| anyhow!("Missing 'query' parameter"))?;
+            let max_results = arguments
+                .get("max_results")
+                .and_then(|v| v.as_u64())
+                .unwrap_or(5) as usize;
+            let with_magnets = arguments
+                .get("with_magnets")
+                .and_then(|v| v.as_bool())
+                .unwrap_or(true);
+
+            let results =
+                torrent_search(None, query, max_results.min(20).max(1), with_magnets).await?;
+            let text = serde_json::to_string_pretty(&results)?;
+
+            Ok(ToolResult {
+                content: vec![ToolContent {
+                    content_type: "text".to_string(),
+                    text,
+                }],
+                is_error: false,
+            })
+        }
+        "torrent_extract" => {
+            let url = arguments
+                .get("url")
+                .and_then(|v| v.as_str())
+                .ok_or_else(|| anyhow!("Missing 'url' parameter"))?;
+            let max_links = arguments
+                .get("max_links")
+                .and_then(|v| v.as_u64())
+                .unwrap_or(5) as usize;
+
+            let links = extract_magnet_links(url, max_links.min(20).max(1)).await?;
+            let text = serde_json::to_string_pretty(&links)?;
+
+            Ok(ToolResult {
+                content: vec![ToolContent {
+                    content_type: "text".to_string(),
+                    text,
+                }],
+                is_error: false,
+            })
+        }
+        _ => Err(anyhow!("Unknown tool: {}", name)),
+    }
+}
+
+async fn broker_web_search(
+    endpoint_override: Option<&str>,
+    query: &str,
+    max_results: usize,
+) -> Result<Vec<SearchResult>> {
+    let max_results = max_results.min(20).max(1);
+    let endpoints = resolve_searx_endpoints(endpoint_override);
+    let mut reports = collect_provider_reports(&endpoints, query, max_results).await;
+    let mut merged = Vec::new();
+    for report in &reports {
+        merged.extend(report.items.clone());
+    }
+    let ranked = rank_and_dedup(merged, max_results, query);
+    if !ranked.is_empty() {
+        return Ok(ranked);
+    }
+
+    // Retry once with a simplified keyword-only query, useful when LLM sends
+    // long instruction-style prompts instead of search terms.
+    if let Some(relaxed_query) = relax_search_query(query) {
+        if relaxed_query != query {
+            let retry_reports =
+                collect_provider_reports(&endpoints, &relaxed_query, max_results).await;
+            let mut retry_merged = Vec::new();
+            for report in &retry_reports {
+                retry_merged.extend(report.items.clone());
+            }
+            let retry_ranked = rank_and_dedup(retry_merged, max_results, &relaxed_query);
+            if !retry_ranked.is_empty() {
+                return Ok(retry_ranked);
+            }
+            reports.extend(retry_reports);
+        }
+    }
+
+    // Second retry using topic-focused keywords for geopolitical/news prompts.
+    if let Some(topic_query) = topic_focus_query(query) {
+        if topic_query != query {
+            let topic_reports =
+                collect_provider_reports(&endpoints, &topic_query, max_results).await;
+            let mut topic_merged = Vec::new();
+            for report in &topic_reports {
+                topic_merged.extend(report.items.clone());
+            }
+            let topic_ranked = rank_and_dedup(topic_merged, max_results, &topic_query);
+            if !topic_ranked.is_empty() {
+                return Ok(topic_ranked);
+            }
+            reports.extend(topic_reports);
+        }
+    }
+
+    if let Ok(probe_results) = direct_domain_probe(query, max_results).await {
+        if !probe_results.is_empty() {
+            return Ok(probe_results);
+        }
+    }
+
+    let mut provider_errors = Vec::new();
+    for report in reports {
+        if let Some(err) = report.error {
+            provider_errors.push(format!("{}: {}", report.provider, err));
+        }
+    }
+
+    if provider_errors.is_empty() {
+        Err(anyhow!(
+            "multi-engine search returned no results for query '{}'; providers returned empty sets",
+            query
+        ))
+    } else {
+        Err(anyhow!(
+            "multi-engine search failed for query '{}': {}",
+            query,
+            provider_errors.join(" | ")
+        ))
+    }
+}
+
+async fn collect_provider_reports(
+    endpoints: &[String],
+    query: &str,
+    max_results: usize,
+) -> Vec<SearchProviderReport> {
+    let searx_future = searx_search_broker(
+        endpoints,
+        query,
+        (max_results * 3).min(MAX_PROVIDER_RESULTS),
+        MAX_SEARX_PARALLEL_ENDPOINTS,
+    );
+    let wiki_future = wikipedia_search(query, (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS));
+    let news_future = google_news_search(query, (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS));
+    let brave_future = brave_html_search(query, (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS));
+    let duckduckgo_future =
+        duckduckgo_html_search(query, (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS));
+    let bing_future = bing_rss_search(query, (max_results / 2).max(3).min(MAX_PROVIDER_RESULTS));
+
+    let (searx, wiki, news, brave, duckduckgo, bing) = tokio::join!(
+        searx_future,
+        wiki_future,
+        news_future,
+        brave_future,
+        duckduckgo_future,
+        bing_future
+    );
+
+    vec![
+        as_provider_report("searx", searx),
+        as_provider_report("wikipedia", wiki),
+        as_provider_report("news-rss", news),
+        as_provider_report("brave-html", brave),
+        as_provider_report("duckduckgo", duckduckgo),
+        as_provider_report("bing-rss", bing),
+    ]
+}
+
+fn relax_search_query(query: &str) -> Option<String> {
+    let stopwords: HashSet<&'static str> = [
+        "fai",
+        "ricerca",
+        "ricercare",
+        "cerca",
+        "cercare",
+        "correlate",
+        "correlato",
+        "analizza",
+        "analisi",
+        "eventuali",
+        "possibili",
+        "breve",
+        "riassunto",
+        "sintesi",
+        "su",
+        "sul",
+        "sulla",
+        "sulle",
+        "con",
+        "per",
+        "tra",
+        "fra",
+        "del",
+        "della",
+        "delle",
+        "degli",
+        "dei",
+        "e",
+        "ed",
+        "in",
+        "di",
+        "da",
+        "a",
+        "il",
+        "lo",
+        "la",
+        "gli",
+        "le",
+        "the",
+        "and",
+        "for",
+        "with",
+        "from",
+        "into",
+    ]
+    .into_iter()
+    .collect();
+
+    let mut keywords = Vec::new();
+    for raw in query.split_whitespace() {
+        let cleaned = raw
+            .trim_matches(|c: char| !c.is_alphanumeric() && c != '_' && c != '-')
+            .to_lowercase();
+        if cleaned.len() < 3 {
+            continue;
+        }
+        if stopwords.contains(cleaned.as_str()) {
+            continue;
+        }
+        keywords.push(cleaned);
+        if keywords.len() >= 10 {
+            break;
+        }
+    }
+
+    if keywords.len() < 3 {
+        let fallback = query
+            .split_whitespace()
+            .map(|s| s.trim_matches(|c: char| !c.is_alphanumeric() && c != '_' && c != '-'))
+            .filter(|s| s.len() >= 3)
+            .take(8)
+            .collect::<Vec<_>>();
+        if fallback.is_empty() {
+            return None;
+        }
+        return Some(fallback.join(" "));
+    }
+
+    Some(keywords.join(" "))
+}
+
+fn topic_focus_query(query: &str) -> Option<String> {
+    let priority = [
+        "israele",
+        "israel",
+        "gaza",
+        "hamas",
+        "guerra",
+        "war",
+        "iran",
+        "libano",
+        "hezbollah",
+        "blocco",
+        "blocchi",
+        "navale",
+        "navali",
+        "mar",
+        "mare",
+        "italia",
+        "italy",
+        "europa",
+        "europe",
+    ];
+
+    let tokens = query
+        .split_whitespace()
+        .map(|raw| {
+            raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '_' && c != '-')
+                .to_lowercase()
+        })
+        .filter(|t| t.len() >= 3)
+        .collect::<Vec<_>>();
+
+    if tokens.is_empty() {
+        return None;
+    }
+
+    let mut selected = Vec::new();
+    for p in priority {
+        if tokens.iter().any(|t| t == p) {
+            selected.push(p.to_string());
+        }
+        if selected.len() >= 7 {
+            break;
+        }
+    }
+
+    if selected.len() < 3 {
+        for t in tokens {
+            if !selected.iter().any(|x| x == &t) {
+                selected.push(t);
+            }
+            if selected.len() >= 7 {
+                break;
+            }
+        }
+    }
+
+    if selected.is_empty() {
+        None
+    } else {
+        Some(selected.join(" "))
+    }
+}
+
+fn as_provider_report(
+    provider: &'static str,
+    result: Result<Vec<SearchResult>>,
+) -> SearchProviderReport {
+    match result {
+        Ok(items) => SearchProviderReport {
+            provider,
+            items,
+            error: None,
+        },
+        Err(e) => SearchProviderReport {
+            provider,
+            items: Vec::new(),
+            error: Some(e.to_string()),
+        },
+    }
+}
+
+fn rank_and_dedup(
+    results: Vec<SearchResult>,
+    max_results: usize,
+    query: &str,
+) -> Vec<SearchResult> {
+    let query_tokens = query_tokens(query);
+    let mut deduped = Vec::with_capacity(results.len());
+    let mut seen_urls = HashSet::new();
+
+    for mut item in results {
+        if item.url.trim().is_empty() {
+            continue;
+        }
+        let key = normalize_url_key(&item.url);
+        if key.is_empty() || !seen_urls.insert(key) {
+            continue;
+        }
+        if item.source_domain.trim().is_empty() {
+            item.source_domain = source_domain(&item.url);
+        }
+        deduped.push(item);
+    }
+
+    let mut domain_counts: HashMap<String, u32> = HashMap::new();
+    for item in &mut deduped {
+        let domain = if item.source_domain.trim().is_empty() {
+            "unknown".to_string()
+        } else {
+            item.source_domain.clone()
+        };
+
+        let count = *domain_counts.get(&domain).unwrap_or(&0);
+        let mut score = 1.0;
+
+        score += match item.provider.as_str() {
+            p if p.starts_with("searx") => 0.9,
+            p if p.starts_with("brave-html") => 0.82,
+            "wikipedia" => 0.75,
+            p if p.starts_with("news-rss") => 0.7,
+            p if p.starts_with("duckduckgo") => 0.65,
+            p if p.starts_with("bing-rss") => 0.55,
+            p if p.starts_with("direct-probe") => 0.62,
+            _ => 0.5,
+        };
+
+        if !item.engine.trim().is_empty() {
+            score += 0.15;
+        }
+        if item.content.len() > 80 {
+            score += 0.15;
+        }
+        let combined = format!("{} {} {}", item.title, item.content, item.url);
+        let relevance = query_overlap_score(&combined, &query_tokens);
+        score += relevance;
+        if relevance <= 0.01 {
+            score -= 0.3;
+        }
+        score -= 0.18 * count as f64;
+        if domain == "unknown" {
+            score -= 0.1;
+        }
+
+        item.score = Some((score * 100.0).round() / 100.0);
+        domain_counts.insert(domain, count + 1);
+    }
+
+    deduped.sort_by(|a, b| {
+        let sb = b.score.unwrap_or(0.0);
+        let sa = a.score.unwrap_or(0.0);
+        sb.partial_cmp(&sa)
+            .unwrap_or(Ordering::Equal)
+            .then_with(|| a.source_domain.cmp(&b.source_domain))
+            .then_with(|| a.title.cmp(&b.title))
+    });
+
+    if !query_tokens.is_empty() {
+        deduped.retain(|item| {
+            let combined = format!("{} {} {}", item.title, item.content, item.url);
+            query_overlap_score(&combined, &query_tokens) > 0.01
+        });
+    }
+
+    deduped.truncate(max_results);
+    deduped
+}
+
+fn query_tokens(query: &str) -> Vec<String> {
+    let mut out = Vec::new();
+    for raw in query.split_whitespace() {
+        let token = raw
+            .trim_matches(|c: char| !c.is_alphanumeric() && c != '_' && c != '-')
+            .to_lowercase();
+        if token.len() < 3 {
+            continue;
+        }
+        if !out.iter().any(|v| v == &token) {
+            out.push(token);
+        }
+        if out.len() >= 12 {
+            break;
+        }
+    }
+    out
+}
+
+fn query_overlap_score(text: &str, query_tokens: &[String]) -> f64 {
+    if query_tokens.is_empty() {
+        return 0.0;
+    }
+    let lower = text.to_lowercase();
+    let mut matches = 0usize;
+    for token in query_tokens {
+        if lower.contains(token) {
+            matches += 1;
+        }
+    }
+    (matches as f64 / query_tokens.len() as f64) * 0.9
+}
+
+async fn searx_search_broker(
+    endpoints: &[String],
+    query: &str,
+    max_results_per_endpoint: usize,
+    max_parallel_endpoints: usize,
+) -> Result<Vec<SearchResult>> {
+    let ordered = rank_endpoints(endpoints).await;
+    let cap = max_parallel_endpoints.max(1).min(ordered.len().max(1));
+    let selected: Vec<String> = ordered.iter().take(cap).cloned().collect();
+
+    let mut tasks = Vec::with_capacity(selected.len());
+    for endpoint in selected {
+        let q = query.to_string();
+        tasks.push(tokio::spawn(async move {
+            let out = web_search_single_endpoint(&endpoint, &q, max_results_per_endpoint).await;
+            (endpoint, out)
+        }));
+    }
+
+    let mut merged = Vec::new();
+    let mut errors = Vec::new();
+
+    for task in tasks {
+        match task.await {
+            Ok((endpoint, Ok(results))) => {
+                mark_endpoint_success(&endpoint).await;
+                merged.extend(results);
+            }
+            Ok((endpoint, Err(err))) => {
+                mark_endpoint_failure(&endpoint, &err.to_string()).await;
+                errors.push(format!("{} -> {}", endpoint, err));
+            }
+            Err(err) => {
+                errors.push(format!("join error: {}", err));
+            }
+        }
+    }
+
+    if merged.is_empty() {
+        // Fallback pass: if caller provided more endpoints than current parallel cap,
+        // probe the remaining ones sequentially before failing.
+        for endpoint in ordered.iter().skip(cap).cloned() {
+            match web_search_single_endpoint(&endpoint, query, max_results_per_endpoint).await {
+                Ok(results) => {
+                    mark_endpoint_success(&endpoint).await;
+                    merged.extend(results);
+                    if !merged.is_empty() {
+                        break;
+                    }
+                }
+                Err(err) => {
+                    mark_endpoint_failure(&endpoint, &err.to_string()).await;
+                    errors.push(format!("{} -> {}", endpoint, err));
+                }
+            }
+        }
+    }
+
+    if merged.is_empty() {
+        let details = if errors.is_empty() {
+            "unknown failure".to_string()
+        } else {
+            errors.join(" | ")
+        };
+        return Err(anyhow!("all searx endpoints failed: {}", details));
+    }
+
+    Ok(merged)
+}
+
+async fn rank_endpoints(endpoints: &[String]) -> Vec<String> {
+    let mut unique = Vec::new();
+    let mut seen = HashSet::new();
+    for raw in endpoints {
+        let normalized = raw.trim().trim_end_matches('/').to_string();
+        if normalized.is_empty() {
+            continue;
+        }
+        if seen.insert(normalized.clone()) {
+            unique.push(normalized);
+        }
+    }
+
+    let now = now_unix_secs();
+    let map = endpoint_health_store().lock().await;
+
+    let mut ranked: Vec<(String, i64)> = unique
+        .into_iter()
+        .map(|endpoint| {
+            let stat = map.get(&endpoint).cloned().unwrap_or_default();
+            let cooldown_penalty = if stat.cooldown_until > now { 20 } else { 0 };
+            let score = (stat.successes as i64 * 2) - (stat.failures as i64) - cooldown_penalty;
+            (endpoint, score)
+        })
+        .collect();
+
+    ranked.sort_by(|a, b| b.1.cmp(&a.1));
+    ranked.into_iter().map(|x| x.0).collect()
+}
+
+async fn mark_endpoint_success(endpoint: &str) {
+    let mut map = endpoint_health_store().lock().await;
+    let entry = map.entry(endpoint.to_string()).or_default();
+    entry.successes = entry.successes.saturating_add(1);
+    entry.cooldown_until = 0;
+    entry.last_error = None;
+}
+
+async fn mark_endpoint_failure(endpoint: &str, error: &str) {
+    let mut map = endpoint_health_store().lock().await;
+    let entry = map.entry(endpoint.to_string()).or_default();
+    entry.failures = entry.failures.saturating_add(1);
+    let lower = error.to_lowercase();
+    let cooldown = if lower.contains("429") { 120 } else { 30 };
+    entry.cooldown_until = now_unix_secs().saturating_add(cooldown);
+    entry.last_error = Some(error.to_string());
+}
+
+async fn web_search_single_endpoint(
+    endpoint: &str,
+    query: &str,
+    max_results: usize,
+) -> Result<Vec<SearchResult>> {
+    let base = endpoint.trim_end_matches('/');
+    let url = format!(
+        "{}/search?q={}&format=json&language=en-US",
+        base,
+        url_encode(query)
+    );
+    let client = reqwest::Client::builder()
+        .user_agent(format!("MasiXDiscovery/{} searx", MODULE_VERSION))
+        .timeout(Duration::from_secs(DEFAULT_SEARCH_TIMEOUT_SECS))
+        .build()?;
+
+    let mut last_error: Option<String> = None;
+
+    for attempt in 1..=SEARX_RETRIES {
+        match client
+            .get(&url)
+            .header("Accept", "application/json")
+            .header("Accept-Language", "en-US,en;q=0.8")
+            .send()
+            .await
+        {
+            Ok(response) => {
+                let status = response.status();
+                if status.is_success() {
+                    let mut parsed: SearxResponse = response
+                        .json()
+                        .await
+                        .map_err(|e| anyhow!("Invalid SearXNG JSON from {}: {}", endpoint, e))?;
+                    parsed.results.retain(|r| !r.url.trim().is_empty());
+                    parsed
+                        .results
+                        .truncate(max_results.min(MAX_PROVIDER_RESULTS));
+
+                    let provider = format!("searx:{}", endpoint_host(endpoint));
+                    let mapped = parsed
+                        .results
+                        .into_iter()
+                        .map(|item| SearchResult {
+                            title: item.title,
+                            url: item.url.clone(),
+                            content: item.content,
+                            engine: item.engine,
+                            provider: provider.clone(),
+                            source_domain: source_domain(&item.url),
+                            score: None,
+                            endpoint: Some(endpoint.to_string()),
+                        })
+                        .collect::<Vec<_>>();
+                    return Ok(mapped);
+                }
+
+                let body = response.text().await.unwrap_or_default();
+                let snippet = summarize_for_error(&body);
+                let err = format!(
+                    "SearX endpoint {} HTTP {} (attempt {}/{}): {}",
+                    endpoint, status, attempt, SEARX_RETRIES, snippet
+                );
+
+                if should_retry_status(status) && attempt < SEARX_RETRIES {
+                    last_error = Some(err);
+                    tokio::time::sleep(backoff_for_attempt(attempt, status.as_u16())).await;
+                    continue;
+                }
+
+                return Err(anyhow!(err));
+            }
+            Err(e) => {
+                let err = format!(
+                    "SearX endpoint {} request error (attempt {}/{}): {}",
+                    endpoint, attempt, SEARX_RETRIES, e
+                );
+                if attempt < SEARX_RETRIES {
+                    last_error = Some(err);
+                    tokio::time::sleep(backoff_for_attempt(attempt, 0)).await;
+                    continue;
+                }
+                return Err(anyhow!(err));
+            }
+        }
+    }
+
+    Err(anyhow!(
+        "SearX endpoint {} failed: {}",
+        endpoint,
+        last_error.unwrap_or_else(|| "unknown error".to_string())
+    ))
+}
+
+fn should_retry_status(status: reqwest::StatusCode) -> bool {
+    matches!(
+        status.as_u16(),
+        429 | 500 | 502 | 503 | 504 | 520 | 521 | 522 | 523 | 524
+    )
+}
+
+fn backoff_for_attempt(attempt: usize, status: u16) -> Duration {
+    let base_ms = match status {
+        429 => 1200,
+        500 | 502 | 503 | 504 => 700,
+        _ => 500,
+    };
+    let factor = (attempt.saturating_sub(1)) as u32;
+    Duration::from_millis(base_ms * (2_u64.pow(factor)).min(6))
+}
+
+fn endpoint_host(endpoint: &str) -> String {
+    reqwest::Url::parse(endpoint)
+        .ok()
+        .and_then(|u| u.host_str().map(|s| s.to_string()))
+        .unwrap_or_else(|| "unknown".to_string())
+}
+
+async fn wikipedia_search(query: &str, max_results: usize) -> Result<Vec<SearchResult>> {
+    let client = reqwest::Client::builder()
+        .user_agent(format!("MasiXDiscovery/{} wikipedia", MODULE_VERSION))
+        .timeout(Duration::from_secs(DEFAULT_SEARCH_TIMEOUT_SECS))
+        .build()?;
+
+    let url = format!(
+        "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={}&utf8=&format=json&srlimit={}",
+        url_encode(query),
+        max_results.min(20).max(1)
+    );
+
+    let response = client
+        .get(&url)
+        .header("Accept", "application/json")
+        .send()
+        .await?;
+
+    if !response.status().is_success() {
+        return Err(anyhow!(
+            "wikipedia provider failed: HTTP {}",
+            response.status()
+        ));
+    }
+
+    let parsed: WikipediaResponse = response.json().await?;
+    let entries = parsed
+        .query
+        .map(|q| q.search)
+        .unwrap_or_default()
+        .into_iter()
+        .filter(|item| item.pageid > 0)
+        .take(max_results.min(MAX_PROVIDER_RESULTS))
+        .map(|item| {
+            let page_url = format!("https://en.wikipedia.org/?curid={}", item.pageid);
+            SearchResult {
+                title: item.title,
+                url: page_url.clone(),
+                content: strip_html_tags(&decode_xml_entities(&item.snippet)),
+                engine: "wikipedia-api".to_string(),
+                provider: "wikipedia".to_string(),
+                source_domain: source_domain(&page_url),
+                score: None,
+                endpoint: None,
+            }
+        })
+        .collect::<Vec<_>>();
+
+    Ok(entries)
+}
+
+async fn google_news_search(query: &str, max_results: usize) -> Result<Vec<SearchResult>> {
+    let per_region = ((max_results.saturating_mul(2)) / NEWS_REGIONS.len())
+        .max(2)
+        .min(MAX_PROVIDER_RESULTS);
+
+    let mut tasks = Vec::with_capacity(NEWS_REGIONS.len());
+    for (country, hl, ceid) in NEWS_REGIONS {
+        let q = query.to_string();
+        let country = *country;
+        let hl = *hl;
+        let ceid = *ceid;
+        tasks.push(tokio::spawn(async move {
+            let out = google_news_region_search(&q, country, hl, ceid, per_region).await;
+            (country, out)
+        }));
+    }
+
+    let mut merged = Vec::new();
+    let mut errors = Vec::new();
+    for task in tasks {
+        match task.await {
+            Ok((_, Ok(items))) => merged.extend(items),
+            Ok((country, Err(err))) => errors.push(format!("{}: {}", country, err)),
+            Err(err) => errors.push(format!("join error: {}", err)),
+        }
+    }
+
+    if merged.is_empty() {
+        if errors.is_empty() {
+            Err(anyhow!("news-rss provider returned no results"))
+        } else {
+            Err(anyhow!(
+                "news-rss providers failed for query '{}': {}",
+                query,
+                errors.join(" | ")
+            ))
+        }
+    } else {
+        Ok(merged)
+    }
+}
+
+async fn google_news_region_search(
+    query: &str,
+    country: &str,
+    hl: &str,
+    ceid: &str,
+    max_results: usize,
+) -> Result<Vec<SearchResult>> {
+    let client = reqwest::Client::builder()
+        .user_agent(format!("MasiXDiscovery/{} news-rss", MODULE_VERSION))
+        .timeout(Duration::from_secs(DEFAULT_SEARCH_TIMEOUT_SECS))
+        .build()?;
+
+    let url = format!(
+        "https://news.google.com/rss/search?q={}&hl={}&gl={}&ceid={}",
+        url_encode(query),
+        url_encode(hl),
+        url_encode(country),
+        url_encode(ceid)
+    );
+
+    let response = client.get(&url).send().await?;
+    if !response.status().is_success() {
+        return Err(anyhow!(
+            "news-rss provider {} failed: HTTP {}",
+            country,
+            response.status()
+        ));
+    }
+
+    let body = response.text().await?;
+    let limit = max_results.min(MAX_PROVIDER_RESULTS).max(1);
+    let mut items = parse_rss_items(&body, limit);
+
+    if items.is_empty() {
+        items = parse_atom_entries(&body, limit);
+    }
+
+    let engine = format!("google-news-rss-{}", country.to_lowercase());
+    let provider = format!("news-rss:{}", country);
+
+    Ok(items
+        .into_iter()
+        .map(|item| SearchResult {
+            title: item.title,
+            url: item.link.clone(),
+            content: item.description,
+            engine: engine.clone(),
+            provider: provider.clone(),
+            source_domain: source_domain(&item.link),
+            score: None,
+            endpoint: None,
+        })
+        .collect::<Vec<_>>())
+}
+
+async fn brave_html_search(query: &str, max_results: usize) -> Result<Vec<SearchResult>> {
+    let client = reqwest::Client::builder()
+        .user_agent(BROWSER_USER_AGENT)
+        .timeout(Duration::from_secs(DEFAULT_SEARCH_TIMEOUT_SECS))
+        .build()?;
+
+    let url = format!(
+        "https://search.brave.com/search?q={}&source=web",
+        url_encode(query)
+    );
+    let response = client
+        .get(&url)
+        .header("Accept-Language", "en-US,en;q=0.8")
+        .header("Accept-Encoding", "gzip, deflate")
+        .send()
+        .await?;
+    if !response.status().is_success() {
+        return Err(anyhow!(
+            "brave-html provider failed: HTTP {}",
+            response.status()
+        ));
+    }
+
+    let body = response.text().await?;
+    let doc = Html::parse_document(&body);
+    let snippet_selector = Selector::parse("div.snippet[data-type=\"web\"]")
+        .map_err(|e| anyhow!("brave snippet selector error: {}", e))?;
+    let link_selector =
+        Selector::parse("a.l1[href]").map_err(|e| anyhow!("brave link selector error: {}", e))?;
+    let title_selector =
+        Selector::parse("div.title").map_err(|e| anyhow!("brave title selector error: {}", e))?;
+    let desc_selector = Selector::parse("div.snippet-description")
+        .map_err(|e| anyhow!("brave description selector error: {}", e))?;
+
+    let mut results = Vec::new();
+    let limit = max_results.min(MAX_PROVIDER_RESULTS).max(1);
+    for snippet in doc.select(&snippet_selector) {
+        if results.len() >= limit {
+            break;
+        }
+        let Some(link) = snippet.select(&link_selector).next() else {
+            continue;
+        };
+        let href = link.value().attr("href").unwrap_or("").trim();
+        if !(href.starts_with("http://") || href.starts_with("https://")) {
+            continue;
+        }
+
+        let title = snippet
+            .select(&title_selector)
+            .next()
+            .map(|n| n.text().collect::<Vec<_>>().join(" ").trim().to_string())
+            .filter(|v| !v.is_empty())
+            .or_else(|| {
+                let fallback = link.text().collect::<Vec<_>>().join(" ").trim().to_string();
+                if fallback.is_empty() {
+                    None
+                } else {
+                    Some(fallback)
+                }
+            })
+            .unwrap_or_else(|| "Result from Brave Search".to_string());
+
+        let content = snippet
+            .select(&desc_selector)
+            .next()
+            .map(|n| n.text().collect::<Vec<_>>().join(" ").trim().to_string())
+            .unwrap_or_default();
+
+        results.push(SearchResult {
+            title,
+            url: href.to_string(),
+            content,
+            engine: "brave-html".to_string(),
+            provider: "brave-html".to_string(),
+            source_domain: source_domain(href),
+            score: None,
+            endpoint: Some("https://search.brave.com/search".to_string()),
+        });
+    }
+
+    if results.is_empty() {
+        Err(anyhow!("brave-html provider returned no results"))
+    } else {
+        Ok(results)
+    }
+}
+
+async fn duckduckgo_html_search(query: &str, max_results: usize) -> Result<Vec<SearchResult>> {
+    let client = reqwest::Client::builder()
+        .user_agent(format!("MasiXDiscovery/{} duckduckgo", MODULE_VERSION))
+        .timeout(Duration::from_secs(DEFAULT_SEARCH_TIMEOUT_SECS))
+        .build()?;
+    let target = max_results.min(MAX_PROVIDER_RESULTS).max(1);
+
+    // Preferred path: DuckDuckGo lite endpoint is more stable than /html in headless contexts.
+    let lite_url = format!(
+        "https://lite.duckduckgo.com/lite/?q={}&kl=wt-wt",
+        url_encode(query)
+    );
+    if let Ok(response) = client
+        .get(&lite_url)
+        .header("User-Agent", BROWSER_USER_AGENT)
+        .header("Accept-Language", "en-US,en;q=0.8")
+        .send()
+        .await
+    {
+        if response.status().is_success() {
+            let body = response.text().await.unwrap_or_default();
+            let doc = Html::parse_document(&body);
+            let title_selector = Selector::parse("a.result-link")
+                .map_err(|e| anyhow!("duckduckgo lite selector error: {}", e))?;
+            let snippet_selector = Selector::parse("td.result-snippet")
+                .map_err(|e| anyhow!("duckduckgo lite snippet selector error: {}", e))?;
+            let snippets = doc
+                .select(&snippet_selector)
+                .map(|n| n.text().collect::<Vec<_>>().join(" ").trim().to_string())
+                .collect::<Vec<_>>();
+
+            let mut results = Vec::new();
+            for (idx, element) in doc.select(&title_selector).enumerate() {
+                if results.len() >= target {
+                    break;
+                }
+                let raw_href = element.value().attr("href").unwrap_or("").trim();
+                let url = normalize_duckduckgo_href(raw_href);
+                if url.is_empty() {
+                    continue;
+                }
+                let title = element
+                    .text()
+                    .collect::<Vec<_>>()
+                    .join(" ")
+                    .trim()
+                    .to_string();
+                if title.is_empty() {
+                    continue;
+                }
+                let content = snippets.get(idx).cloned().unwrap_or_default();
+                results.push(SearchResult {
+                    title,
+                    url: url.clone(),
+                    content,
+                    engine: "duckduckgo-lite".to_string(),
+                    provider: "duckduckgo".to_string(),
+                    source_domain: source_domain(&url),
+                    score: None,
+                    endpoint: Some("https://lite.duckduckgo.com/lite".to_string()),
+                });
+            }
+            if !results.is_empty() {
+                return Ok(results);
+            }
+        }
+    }
+
+    let url = format!(
+        "https://duckduckgo.com/html/?q={}&kl=wt-wt",
+        url_encode(query)
+    );
+    let response = client
+        .get(&url)
+        .header("User-Agent", BROWSER_USER_AGENT)
+        .header("Accept-Language", "en-US,en;q=0.8")
+        .send()
+        .await?;
+    if !response.status().is_success() {
+        return Err(anyhow!(
+            "duckduckgo provider failed: HTTP {}",
+            response.status()
+        ));
+    }
+
+    let body = response.text().await?;
+    let doc = Html::parse_document(&body);
+    let title_selector =
+        Selector::parse("a.result__a").map_err(|e| anyhow!("duckduckgo selector error: {}", e))?;
+    let snippet_selector = Selector::parse(".result__snippet")
+        .map_err(|e| anyhow!("duckduckgo snippet selector error: {}", e))?;
+    let fallback_link_selector =
+        Selector::parse("a[href]").map_err(|e| anyhow!("duckduckgo fallback selector: {}", e))?;
+
+    let mut results = Vec::new();
+
+    for element in doc.select(&title_selector) {
+        if results.len() >= target {
+            break;
+        }
+        let raw_href = element.value().attr("href").unwrap_or("").trim();
+        let url = normalize_duckduckgo_href(raw_href);
+        if url.is_empty() {
+            continue;
+        }
+        let title = element
+            .text()
+            .collect::<Vec<_>>()
+            .join(" ")
+            .trim()
+            .to_string();
+        if title.is_empty() {
+            continue;
+        }
+
+        let content = element
+            .parent()
+            .and_then(scraper::ElementRef::wrap)
+            .and_then(|parent| parent.select(&snippet_selector).next())
+            .map(|n| n.text().collect::<Vec<_>>().join(" ").trim().to_string())
+            .unwrap_or_default();
+
+        results.push(SearchResult {
+            title,
+            url: url.clone(),
+            content,
+            engine: "duckduckgo-html".to_string(),
+            provider: "duckduckgo".to_string(),
+            source_domain: source_domain(&url),
+            score: None,
+            endpoint: Some("https://duckduckgo.com/html".to_string()),
+        });
+    }
+
+    if results.is_empty() {
+        for link in doc.select(&fallback_link_selector) {
+            if results.len() >= target {
+                break;
+            }
+            let raw_href = link.value().attr("href").unwrap_or("").trim();
+            let url = normalize_duckduckgo_href(raw_href);
+            if url.is_empty() {
+                continue;
+            }
+            let title = link.text().collect::<Vec<_>>().join(" ").trim().to_string();
+            if title.len() < 12 {
+                continue;
+            }
+            results.push(SearchResult {
+                title,
+                url: url.clone(),
+                content: String::new(),
+                engine: "duckduckgo-html-fallback".to_string(),
+                provider: "duckduckgo".to_string(),
+                source_domain: source_domain(&url),
+                score: None,
+                endpoint: Some("https://duckduckgo.com/html".to_string()),
+            });
+        }
+    }
+
+    if results.is_empty() {
+        Err(anyhow!("duckduckgo provider returned no results"))
+    } else {
+        Ok(results)
+    }
+}
+
+fn normalize_duckduckgo_href(raw_href: &str) -> String {
+    let href = raw_href.trim();
+    if href.is_empty() {
+        return String::new();
+    }
+    if href.starts_with("http://") || href.starts_with("https://") {
+        return href.to_string();
+    }
+    if href.starts_with("/l/?") || href.starts_with("//duckduckgo.com/l/?") {
+        let canonical = if let Some(stripped) = href.strip_prefix("//duckduckgo.com") {
+            format!("https://duckduckgo.com{}", stripped)
+        } else {
+            format!("https://duckduckgo.com{}", href)
+        };
+        if let Ok(url) = reqwest::Url::parse(&canonical) {
+            for (k, v) in url.query_pairs() {
+                if k == "uddg" {
+                    let out = v.to_string();
+                    if out.starts_with("http://") || out.starts_with("https://") {
+                        return out;
+                    }
+                }
+            }
+        }
+    }
+    String::new()
+}
+
+async fn bing_rss_search(query: &str, max_results: usize) -> Result<Vec<SearchResult>> {
+    let client = reqwest::Client::builder()
+        .user_agent(format!("MasiXDiscovery/{} bing-rss", MODULE_VERSION))
+        .timeout(Duration::from_secs(DEFAULT_SEARCH_TIMEOUT_SECS))
+        .build()?;
+    let url = format!(
+        "https://www.bing.com/search?q={}&format=rss&setlang=en-US&cc=us",
+        url_encode(query)
+    );
+
+    let response = client.get(&url).send().await?;
+    if !response.status().is_success() {
+        return Err(anyhow!(
+            "bing-rss provider failed: HTTP {}",
+            response.status()
+        ));
+    }
+
+    let body = response.text().await?;
+    let limit = max_results.min(MAX_PROVIDER_RESULTS).max(1);
+    let mut items = parse_rss_items(&body, limit);
+    if items.is_empty() {
+        items = parse_atom_entries(&body, limit);
+    }
+    if items.is_empty() {
+        return Err(anyhow!("bing-rss provider returned no results"));
+    }
+
+    Ok(items
+        .into_iter()
+        .map(|item| SearchResult {
+            title: item.title,
+            url: item.link.clone(),
+            content: item.description,
+            engine: "bing-rss".to_string(),
+            provider: "bing-rss".to_string(),
+            source_domain: source_domain(&item.link),
+            score: None,
+            endpoint: Some("https://www.bing.com/search?format=rss".to_string()),
+        })
+        .collect::<Vec<_>>())
+}
+
+async fn direct_domain_probe(query: &str, max_results: usize) -> Result<Vec<SearchResult>> {
+    let mut tokens = query
+        .split_whitespace()
+        .map(|raw| {
+            raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '-' && c != '_')
+                .to_lowercase()
+        })
+        .filter(|t| t.len() >= 3 && t.chars().all(|c| c.is_ascii_alphanumeric() || c == '-'))
+        .collect::<Vec<_>>();
+    tokens.dedup();
+    if tokens.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let mut stems = Vec::new();
+    stems.push(tokens[0].clone());
+    if tokens.len() >= 2 {
+        stems.push(format!("{}{}", tokens[0], tokens[1]));
+    }
+    stems.truncate(2);
+
+    let tlds = ["com", "org", "net", "io", "it"];
+    let mut candidates = Vec::new();
+    for stem in stems {
+        for tld in tlds {
+            candidates.push(format!("https://{}.{}", stem, tld));
+        }
+    }
+
+    let client = reqwest::Client::builder()
+        .user_agent(BROWSER_USER_AGENT)
+        .timeout(Duration::from_secs(8))
+        .build()?;
+
+    let mut out = Vec::new();
+    for url in candidates {
+        if out.len() >= max_results.min(3) {
+            break;
+        }
+        let response = match client.get(&url).send().await {
+            Ok(r) => r,
+            Err(_) => continue,
+        };
+        if !response.status().is_success() {
+            continue;
+        }
+        let body = response.text().await.unwrap_or_default();
+        let title = extract_html_title(&body).unwrap_or_else(|| url.clone());
+        let snippet = extract_text_snippet_from_html(&body, 260);
+        out.push(SearchResult {
+            title,
+            url: url.clone(),
+            content: snippet,
+            engine: "direct-domain-probe".to_string(),
+            provider: "direct-probe".to_string(),
+            source_domain: source_domain(&url),
+            score: None,
+            endpoint: None,
+        });
+    }
+
+    Ok(out)
+}
+
+fn extract_html_title(html: &str) -> Option<String> {
+    let lower = html.to_lowercase();
+    let start = lower.find("<title>")?;
+    let end = lower[start + 7..].find("</title>")?;
+    let raw = &html[start + 7..start + 7 + end];
+    let title = decode_xml_entities(raw).trim().to_string();
+    if title.is_empty() {
+        None
+    } else {
+        Some(title)
+    }
+}
+
+fn extract_text_snippet_from_html(html: &str, max_chars: usize) -> String {
+    if html.trim().is_empty() {
+        return String::new();
+    }
+    let document = Html::parse_document(html);
+    let selectors = ["article", "main", ".content", "#content", "body"];
+    let mut best = String::new();
+    for selector_str in selectors {
+        let Ok(selector) = Selector::parse(selector_str) else {
+            continue;
+        };
+        for element in document.select(&selector) {
+            let text = element
+                .text()
+                .collect::<Vec<_>>()
+                .join(" ")
+                .split_whitespace()
+                .collect::<Vec<_>>()
+                .join(" ");
+            if text.len() > best.len() {
+                best = text;
+            }
+        }
+        if !best.is_empty() {
+            break;
+        }
+    }
+    truncate_text(best.trim(), max_chars)
+}
+
+#[derive(Debug)]
+struct FeedItem {
+    title: String,
+    link: String,
+    description: String,
+}
+
+fn parse_rss_items(xml: &str, limit: usize) -> Vec<FeedItem> {
+    let mut items = Vec::new();
+    let mut rest = xml;
+
+    while let Some(start_idx) = rest.find("<item") {
+        let after_start = &rest[start_idx..];
+        let Some(open_end) = after_start.find('>') else {
+            break;
+        };
+        let content_start = start_idx + open_end + 1;
+        let Some(close_rel) = rest[content_start..].find("</item>") else {
+            break;
+        };
+        let content_end = content_start + close_rel;
+        let chunk = &rest[content_start..content_end];
+
+        let title = extract_xml_tag(chunk, "title").unwrap_or_default();
+        let link = extract_xml_tag(chunk, "link").unwrap_or_default();
+        let description = extract_xml_tag(chunk, "description")
+            .or_else(|| extract_xml_tag(chunk, "content:encoded"))
+            .unwrap_or_default();
+
+        if !link.trim().is_empty() {
+            items.push(FeedItem {
+                title: clean_feed_text(&title),
+                link: clean_feed_text(&link),
+                description: clean_feed_text(&description),
+            });
+        }
+
+        rest = &rest[content_end + "</item>".len()..];
+        if items.len() >= limit {
+            break;
+        }
+    }
+
+    items
+}
+
+fn parse_atom_entries(xml: &str, limit: usize) -> Vec<FeedItem> {
+    let mut entries = Vec::new();
+    let mut rest = xml;
+
+    while let Some(start_idx) = rest.find("<entry") {
+        let after_start = &rest[start_idx..];
+        let Some(open_end) = after_start.find('>') else {
+            break;
+        };
+        let content_start = start_idx + open_end + 1;
+        let Some(close_rel) = rest[content_start..].find("</entry>") else {
+            break;
+        };
+        let content_end = content_start + close_rel;
+        let chunk = &rest[content_start..content_end];
+
+        let title = extract_xml_tag(chunk, "title").unwrap_or_default();
+        let description = extract_xml_tag(chunk, "summary")
+            .or_else(|| extract_xml_tag(chunk, "content"))
+            .unwrap_or_default();
+
+        let link = extract_atom_link(chunk).unwrap_or_default();
+
+        if !link.trim().is_empty() {
+            entries.push(FeedItem {
+                title: clean_feed_text(&title),
+                link: clean_feed_text(&link),
+                description: clean_feed_text(&description),
+            });
+        }
+
+        rest = &rest[content_end + "</entry>".len()..];
+        if entries.len() >= limit {
+            break;
+        }
+    }
+
+    entries
+}
+
+fn extract_atom_link(chunk: &str) -> Option<String> {
+    let mut cursor = chunk;
+    while let Some(idx) = cursor.find("<link") {
+        let tail = &cursor[idx..];
+        let end = tail.find('>')?;
+        let tag = &tail[..=end];
+        if let Some(href) = extract_attr(tag, "href") {
+            if !href.trim().is_empty() {
+                return Some(href);
+            }
+        }
+        cursor = &tail[end + 1..];
+    }
+    None
+}
+
+fn extract_attr(tag: &str, attr: &str) -> Option<String> {
+    let needle = format!("{}=\"", attr);
+    let start = tag.find(&needle)? + needle.len();
+    let end = tag[start..].find('"')?;
+    Some(tag[start..start + end].to_string())
+}
+
+fn extract_xml_tag(chunk: &str, tag: &str) -> Option<String> {
+    let open = format!("<{}>", tag);
+    let close = format!("</{}>", tag);
+
+    if let Some(start) = chunk.find(&open) {
+        let value_start = start + open.len();
+        if let Some(end_rel) = chunk[value_start..].find(&close) {
+            return Some(chunk[value_start..value_start + end_rel].to_string());
+        }
+    }
+
+    None
+}
+
+fn clean_feed_text(value: &str) -> String {
+    let mut out = value.trim().to_string();
+    out = out.replace("<![CDATA[", "").replace("]]>", "");
+    out = decode_xml_entities(&out);
+    out = strip_html_tags(&out);
+    out.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+fn decode_xml_entities(input: &str) -> String {
+    input
+        .replace("&amp;", "&")
+        .replace("&lt;", "<")
+        .replace("&gt;", ">")
+        .replace("&quot;", "\"")
+        .replace("&#39;", "'")
+        .replace("&apos;", "'")
+}
+
+fn strip_html_tags(input: &str) -> String {
+    let mut out = String::with_capacity(input.len());
+    let mut in_tag = false;
+    for ch in input.chars() {
+        match ch {
+            '<' => in_tag = true,
+            '>' => in_tag = false,
+            _ if !in_tag => out.push(ch),
+            _ => {}
+        }
+    }
+    out
+}
+
+fn resolve_searx_endpoints(endpoint_override: Option<&str>) -> Vec<String> {
+    if let Some(single) = endpoint_override.map(str::trim).filter(|s| !s.is_empty()) {
+        return vec![single.trim_end_matches('/').to_string()];
+    }
+
+    if let Ok(list) = std::env::var("MASIX_DISCOVERY_SEARXNG_URLS") {
+        let parsed = parse_endpoint_list(&list);
+        if !parsed.is_empty() {
+            return parsed;
+        }
+    }
+
+    if let Ok(single) = std::env::var("MASIX_DISCOVERY_SEARXNG_URL") {
+        let parsed = parse_endpoint_list(&single);
+        if !parsed.is_empty() {
+            return parsed;
+        }
+    }
+
+    DEFAULT_SEARXNG_URLS
+        .iter()
+        .map(|v| v.trim_end_matches('/').to_string())
+        .collect()
+}
+
+fn parse_endpoint_list(raw: &str) -> Vec<String> {
+    let mut out = Vec::new();
+    let mut seen = HashSet::new();
+
+    for token in raw.split(',') {
+        let value = token.trim().trim_end_matches('/').to_string();
+        if value.is_empty() {
+            continue;
+        }
+        if !(value.starts_with("https://") || value.starts_with("http://")) {
+            continue;
+        }
+        if seen.insert(value.clone()) {
+            out.push(value);
+        }
+        if out.len() >= MAX_ENDPOINTS_FROM_CONFIG {
+            break;
+        }
+    }
+
+    out
+}
+
+fn summarize_for_error(body: &str) -> String {
+    let compact = body.split_whitespace().collect::<Vec<_>>().join(" ");
+    let mut text = compact;
+    if text.len() > 180 {
+        text.truncate(180);
+        text.push_str("...");
+    }
+    if text.is_empty() {
+        "(empty body)".to_string()
+    } else {
+        text
+    }
+}
+
+fn normalize_url_key(url: &str) -> String {
+    match reqwest::Url::parse(url) {
+        Ok(mut parsed) => {
+            parsed.set_fragment(None);
+            parsed
+                .as_str()
+                .trim_end_matches('/')
+                .to_lowercase()
+                .replace("http://", "https://")
+        }
+        Err(_) => url.trim().to_lowercase(),
+    }
+}
+
+fn source_domain(url: &str) -> String {
+    reqwest::Url::parse(url)
+        .ok()
+        .and_then(|parsed| parsed.host_str().map(|s| s.to_string()))
+        .unwrap_or_else(|| "unknown".to_string())
+}
+
+fn now_unix_secs() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .unwrap_or(0)
+}
+
+async fn web_fetch_page(url: &str) -> Result<String> {
+    let client = reqwest::Client::builder()
+        .user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36")
+        .timeout(Duration::from_secs(DEFAULT_FETCH_TIMEOUT_SECS))
+        .build()?;
+    let response = client.get(url).send().await?;
+    if !response.status().is_success() {
+        return Err(anyhow!("fetch failed: HTTP {}", response.status()));
+    }
+    let html = response.text().await?;
+    let document = Html::parse_document(&html);
+
+    let selectors = [
+        "article", "main", ".content", "#content", ".post", ".article", "body",
+    ];
+    let mut text_content = String::new();
+    for selector_str in selectors {
+        if let Ok(selector) = Selector::parse(selector_str) {
+            for element in document.select(&selector) {
+                let text = element
+                    .text()
+                    .collect::<String>()
+                    .split_whitespace()
+                    .collect::<Vec<_>>()
+                    .join(" ");
+                if text.len() > text_content.len() {
+                    text_content = text;
+                }
+            }
+            if !text_content.is_empty() {
+                break;
+            }
+        }
+    }
+
+    if text_content.is_empty() {
+        text_content = "(no readable text extracted)".to_string();
+    }
+    if text_content.len() > MAX_WEB_CONTENT {
+        text_content.truncate(MAX_WEB_CONTENT);
+        text_content.push_str("... [truncated]");
+    }
+
+    let title = Selector::parse("title")
+        .ok()
+        .and_then(|sel| document.select(&sel).next())
+        .map(|e| e.text().collect::<String>())
+        .unwrap_or_else(|| "N/A".to_string());
+
+    Ok(format!(
+        "Title: {}\nURL: {}\n\n{}",
+        title.trim(),
+        url,
+        text_content
+    ))
+}
+
+async fn torrent_search(
+    endpoint_override: Option<&str>,
+    query: &str,
+    max_results: usize,
+    with_magnets: bool,
+) -> Result<Vec<TorrentSearchResult>> {
+    let normalized_query = normalize_torrent_query(query);
+    let direct_query = query.trim();
+    let direct_query = if direct_query.is_empty() {
+        normalized_query.as_str()
+    } else {
+        direct_query
+    };
+    let endpoints = resolve_searx_endpoints(endpoint_override);
+
+    let searx_future = searx_search_broker(
+        &endpoints,
+        &normalized_query,
+        (max_results * 2).min(MAX_PROVIDER_RESULTS),
+        MAX_SEARX_PARALLEL_ENDPOINTS,
+    );
+    let nyaa_future = nyaa_torrent_search(direct_query, max_results * 2);
+    let archive_future = archive_torrent_search(direct_query, max_results * 2);
+
+    let (searx, nyaa, archive) = tokio::join!(searx_future, nyaa_future, archive_future);
+
+    let mut output = Vec::new();
+    let mut provider_errors = Vec::new();
+
+    match nyaa {
+        Ok(mut items) => output.append(&mut items),
+        Err(e) => provider_errors.push(format!("nyaa: {}", e)),
+    }
+
+    match searx {
+        Ok(items) => {
+            for result in items {
+                let magnet_links = if with_magnets {
+                    extract_magnet_links(&result.url, 3)
+                        .await
+                        .unwrap_or_default()
+                } else {
+                    Vec::new()
+                };
+                output.push(TorrentSearchResult {
+                    title: result.title,
+                    url: result.url,
+                    content: result.content,
+                    engine: if result.engine.trim().is_empty() {
+                        "searx".to_string()
+                    } else {
+                        result.engine
+                    },
+                    magnet_links,
+                });
+            }
+        }
+        Err(e) => provider_errors.push(format!("searx: {}", e)),
+    }
+
+    match archive {
+        Ok(mut items) => output.append(&mut items),
+        Err(e) => provider_errors.push(format!("archive: {}", e)),
+    }
+
+    output = dedup_torrent_results(output, with_magnets);
+    output.truncate(max_results.min(20).max(1));
+
+    if output.is_empty() {
+        if provider_errors.is_empty() {
+            Err(anyhow!("torrent search returned no results"))
+        } else {
+            Err(anyhow!(
+                "torrent search failed across providers: {}",
+                provider_errors.join(" | ")
+            ))
+        }
+    } else {
+        Ok(output)
+    }
+}
+
+fn dedup_torrent_results(
+    mut results: Vec<TorrentSearchResult>,
+    with_magnets: bool,
+) -> Vec<TorrentSearchResult> {
+    let mut seen = HashSet::new();
+    let mut deduped = Vec::with_capacity(results.len());
+
+    for mut item in results.drain(..) {
+        let key = normalize_url_key(&item.url);
+        if key.is_empty() || !seen.insert(key) {
+            continue;
+        }
+        if !with_magnets {
+            item.magnet_links.clear();
+        }
+        deduped.push(item);
+    }
+
+    deduped.sort_by(|a, b| {
+        torrent_engine_rank(&b.engine)
+            .cmp(&torrent_engine_rank(&a.engine))
+            .then_with(|| b.magnet_links.len().cmp(&a.magnet_links.len()))
+            .then_with(|| a.title.cmp(&b.title))
+    });
+
+    deduped
+}
+
+fn torrent_engine_rank(engine: &str) -> u8 {
+    let lowered = engine.to_lowercase();
+    if lowered.contains("nyaa") {
+        4
+    } else if lowered.contains("searx") {
+        3
+    } else if lowered.contains("archive") {
+        2
+    } else {
+        1
+    }
+}
+
+async fn nyaa_torrent_search(query: &str, max_results: usize) -> Result<Vec<TorrentSearchResult>> {
+    let url = format!("https://nyaa.si/?f=0&c=0_0&q={}", url_encode(query));
+    let client = reqwest::Client::builder()
+        .user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36")
+        .timeout(Duration::from_secs(DEFAULT_SEARCH_TIMEOUT_SECS))
+        .build()?;
+    let response = client.get(&url).send().await?;
+    if !response.status().is_success() {
+        return Err(anyhow!("Nyaa provider failed: HTTP {}", response.status()));
+    }
+    let html = response.text().await?;
+    let document = Html::parse_document(&html);
+
+    let row_selector = Selector::parse("table tbody tr")
+        .map_err(|e| anyhow!("Invalid Nyaa row selector: {}", e))?;
+    let title_selector = Selector::parse("td[colspan=\"2\"] a[href^=\"/view/\"]")
+        .map_err(|e| anyhow!("Invalid Nyaa title selector: {}", e))?;
+    let magnet_selector = Selector::parse("a[href^=\"magnet:?\"]")
+        .map_err(|e| anyhow!("Invalid Nyaa magnet selector: {}", e))?;
+    let cell_selector =
+        Selector::parse("td").map_err(|e| anyhow!("Invalid Nyaa cell selector: {}", e))?;
+
+    let mut output = Vec::new();
+    for row in document.select(&row_selector) {
+        if output.len() >= max_results.min(MAX_PROVIDER_RESULTS) {
+            break;
+        }
+
+        let mut title = String::new();
+        let mut detail_url = String::new();
+        for link in row.select(&title_selector) {
+            if let Some(href) = link.value().attr("href") {
+                title = link.text().collect::<String>().trim().to_string();
+                detail_url = format!("https://nyaa.si{}", href);
+                break;
+            }
+        }
+        if detail_url.is_empty() {
+            continue;
+        }
+
+        let mut magnets = Vec::new();
+        for a in row.select(&magnet_selector) {
+            if let Some(href) = a.value().attr("href") {
+                let cleaned = sanitize_magnet(href);
+                if cleaned.starts_with("magnet:?") {
+                    magnets.push(cleaned);
+                }
+            }
+        }
+
+        let mut columns_text = row
+            .select(&cell_selector)
+            .map(|td| td.text().collect::<String>())
+            .collect::<Vec<_>>();
+        columns_text.retain(|v| !v.trim().is_empty());
+        let summary = columns_text.join(" | ");
+        let summary = summary.split_whitespace().collect::<Vec<_>>().join(" ");
+
+        output.push(TorrentSearchResult {
+            title: if title.is_empty() {
+                "Nyaa result".to_string()
+            } else {
+                title
+            },
+            url: detail_url,
+            content: if summary.is_empty() {
+                "Result from Nyaa index".to_string()
+            } else {
+                truncate_text(&summary, 700)
+            },
+            engine: "nyaa-html".to_string(),
+            magnet_links: magnets,
+        });
+    }
+
+    if output.is_empty() {
+        return Err(anyhow!("Nyaa provider returned no results"));
+    }
+
+    Ok(output)
+}
+
+async fn archive_torrent_search(
+    query: &str,
+    max_results: usize,
+) -> Result<Vec<TorrentSearchResult>> {
+    let url = format!(
+        "https://archive.org/advancedsearch.php?q={}&fl[]=identifier,title,description&rows={}&page=1&output=json",
+        url_encode(query),
+        max_results.min(MAX_PROVIDER_RESULTS).max(1)
+    );
+    let client = reqwest::Client::builder()
+        .user_agent(format!("MasiXDiscovery/{} archive", MODULE_VERSION))
+        .timeout(Duration::from_secs(DEFAULT_SEARCH_TIMEOUT_SECS))
+        .build()?;
+    let response = client.get(&url).send().await?;
+    if !response.status().is_success() {
+        return Err(anyhow!(
+            "Archive provider failed: HTTP {}",
+            response.status()
+        ));
+    }
+
+    let payload: ArchiveResponse = response.json().await?;
+    let mut output = Vec::new();
+    for doc in payload
+        .response
+        .docs
+        .into_iter()
+        .take(max_results.min(MAX_PROVIDER_RESULTS))
+    {
+        if doc.identifier.trim().is_empty() {
+            continue;
+        }
+
+        let title = doc
+            .title
+            .unwrap_or_else(|| format!("archive.org item {}", doc.identifier));
+        let description = truncate_text(&archive_description_to_string(doc.description), 700);
+        let item_url = format!("https://archive.org/details/{}", doc.identifier);
+
+        output.push(TorrentSearchResult {
+            title,
+            url: item_url,
+            content: if description.is_empty() {
+                "Result from archive.org".to_string()
+            } else {
+                description
+            },
+            engine: "archive-search".to_string(),
+            magnet_links: Vec::new(),
+        });
+    }
+
+    if output.is_empty() {
+        return Err(anyhow!("Archive provider returned no results"));
+    }
+
+    Ok(output)
+}
+
+fn archive_description_to_string(value: Option<serde_json::Value>) -> String {
+    let Some(value) = value else {
+        return String::new();
+    };
+
+    match value {
+        serde_json::Value::String(v) => v,
+        serde_json::Value::Array(values) => values
+            .into_iter()
+            .filter_map(|v| v.as_str().map(|s| s.to_string()))
+            .collect::<Vec<_>>()
+            .join(" "),
+        _ => String::new(),
+    }
+}
+
+fn truncate_text(value: &str, max_chars: usize) -> String {
+    if value.chars().count() <= max_chars {
+        return value.to_string();
+    }
+    let mut out = String::new();
+    for (idx, ch) in value.chars().enumerate() {
+        if idx >= max_chars {
+            break;
+        }
+        out.push(ch);
+    }
+    out.push_str("...");
+    out
+}
+
+fn normalize_torrent_query(query: &str) -> String {
+    let trimmed = query.trim();
+    if trimmed.is_empty() {
+        "torrent".to_string()
+    } else if trimmed.to_lowercase().contains("torrent") {
+        trimmed.to_string()
+    } else {
+        format!("{} torrent", trimmed)
+    }
+}
+
+async fn extract_magnet_links(url: &str, max_links: usize) -> Result<Vec<String>> {
+    if url.trim_start().starts_with("magnet:?") {
+        return Ok(vec![sanitize_magnet(url)]);
+    }
+
+    let client = reqwest::Client::builder()
+        .user_agent(format!("MasiXDiscovery/{} torrent", MODULE_VERSION))
+        .timeout(Duration::from_secs(DEFAULT_SEARCH_TIMEOUT_SECS))
+        .build()?;
+    let response = client.get(url).send().await?;
+    if !response.status().is_success() {
+        return Ok(Vec::new());
+    }
+    let html = response.text().await.unwrap_or_default();
+    let document = Html::parse_document(&html);
+    let selector = Selector::parse("a[href^=\"magnet:?\"]").ok();
+
+    let mut seen = HashSet::new();
+    let mut output = Vec::new();
+
+    if let Some(sel) = selector {
+        for element in document.select(&sel) {
+            if let Some(link) = element.value().attr("href") {
+                let cleaned = sanitize_magnet(link);
+                if cleaned.starts_with("magnet:?")
+                    && seen.insert(cleaned.clone())
+                    && output.len() < max_links
+                {
+                    output.push(cleaned);
+                }
+            }
+            if output.len() >= max_links {
+                break;
+            }
+        }
+    }
+
+    if output.len() < max_links {
+        for candidate in extract_magnet_links_from_text(&html) {
+            if seen.insert(candidate.clone()) {
+                output.push(candidate);
+            }
+            if output.len() >= max_links {
+                break;
+            }
+        }
+    }
+
+    Ok(output)
+}
+
+fn extract_magnet_links_from_text(text: &str) -> Vec<String> {
+    let mut links = Vec::new();
+    let mut idx = 0usize;
+    while idx < text.len() {
+        let Some(found) = text[idx..].find("magnet:?") else {
+            break;
+        };
+        let start = idx + found;
+        let remainder = &text[start..];
+        let end_rel = remainder
+            .find(|c: char| c.is_whitespace() || matches!(c, '"' | '\'' | '<' | '>'))
+            .unwrap_or(remainder.len());
+        let raw = &remainder[..end_rel];
+        let candidate = sanitize_magnet(raw);
+        if candidate.starts_with("magnet:?") {
+            links.push(candidate);
+        }
+        idx = start + end_rel;
+    }
+    links
+}
+
+fn sanitize_magnet(value: &str) -> String {
+    value
+        .trim()
+        .trim_matches('"')
+        .trim_matches('\'')
+        .replace("&amp;", "&")
+}
+
+fn url_encode(value: &str) -> String {
+    let mut out = String::new();
+    for b in value.as_bytes() {
+        let is_unreserved = matches!(
+            *b,
+            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~'
+        );
+        if is_unreserved {
+            out.push(*b as char);
+        } else {
+            use std::fmt::Write as _;
+            let _ = write!(&mut out, "%{:02X}", b);
+        }
+    }
+    out
+}