@hraness/wordcell 0.23.0 → 0.24.1

package/src/clip/metadata-search-tool/src/duckduckgo.rs

@@ -0,0 +1,534 @@
+//! Bounded parsing of the fixed DuckDuckGo HTML endpoint. An unrecognized page
+//! is unavailable evidence, never proof that a query has no results.
+use std::time::Duration;
+
+use metadata_search_engine_rs::models::SearchResult;
+use reqwest::{Client, header};
+use scraper::{Html, Selector};
+use url::Url;
+
+const ENDPOINT: &str = "https://html.duckduckgo.com/html/";
+const MAX_BODY_BYTES: usize = 2 * 1024 * 1024;
+
+type Outcome = Result<Vec<SearchResult>, &'static str>;
+
+pub async fn search(client: &Client, query: &str, limit: usize, timeout: Duration) -> Outcome {
+    search_at(client, ENDPOINT, query, limit, timeout).await
+}
+
+async fn search_at(
+    client: &Client,
+    endpoint: &str,
+    query: &str,
+    limit: usize,
+    timeout: Duration,
+) -> Outcome {
+    tokio::time::timeout(timeout, async {
+        let mut response = client
+            .get(endpoint)
+            .query(&[("q", query)])
+            .send()
+            .await
+            .map_err(|_| "http")?;
+        validate_response(
+            response.status(),
+            response.headers(),
+            response.content_length(),
+        )?;
+        let mut body = Vec::new();
+        while let Some(chunk) = response.chunk().await.map_err(|_| "http-body")? {
+            append_body(&mut body, &chunk)?;
+        }
+        let html = std::str::from_utf8(&body).map_err(|_| "body-encoding")?;
+        parse(html, limit)
+    })
+    .await
+    .map_err(|_| "timeout")?
+}
+
+fn validate_response(
+    status: reqwest::StatusCode,
+    headers: &header::HeaderMap,
+    length: Option<u64>,
+) -> Result<(), &'static str> {
+    if !status.is_success() {
+        return Err("http-status");
+    }
+    let content_type = headers
+        .get(header::CONTENT_TYPE)
+        .and_then(|value| value.to_str().ok())
+        .unwrap_or("")
+        .split(';')
+        .next()
+        .unwrap_or("")
+        .trim();
+    if !content_type.eq_ignore_ascii_case("text/html")
+        && !content_type.eq_ignore_ascii_case("application/xhtml+xml")
+    {
+        return Err("content-type");
+    }
+    // reqwest removes this header after a supported decoding operation.
+    if headers.get(header::CONTENT_ENCODING).is_some() {
+        return Err("content-encoding");
+    }
+    if length.is_some_and(|length| length > MAX_BODY_BYTES as u64) {
+        return Err("body-limit");
+    }
+    Ok(())
+}
+
+fn append_body(body: &mut Vec<u8>, chunk: &[u8]) -> Result<(), &'static str> {
+    if chunk.len() > MAX_BODY_BYTES.saturating_sub(body.len()) {
+        return Err("body-limit");
+    }
+    body.extend_from_slice(chunk);
+    Ok(())
+}
+
+fn selector(value: &str) -> Selector {
+    Selector::parse(value).expect("static selector")
+}
+
+fn destination(href: &str) -> Option<String> {
+    let endpoint = Url::parse(ENDPOINT).ok()?;
+    let link = endpoint.join(href).ok()?;
+    let is_wrapper = matches!(
+        link.host_str(),
+        Some("duckduckgo.com" | "www.duckduckgo.com" | "html.duckduckgo.com")
+    ) && link.path() == "/l/";
+    let destination = if is_wrapper {
+        let values: Vec<_> = link
+            .query_pairs()
+            .filter(|(name, _)| name == "uddg")
+            .collect();
+        if values.len() != 1 {
+            return None;
+        }
+        Url::parse(&values[0].1).ok()?
+    } else {
+        // Relative links to navigation are not organic destinations.
+        if !href.starts_with("https://") && !href.starts_with("http://") {
+            return None;
+        }
+        link
+    };
+    if !matches!(destination.scheme(), "http" | "https")
+        || destination.host_str().is_none()
+        || !destination.username().is_empty()
+        || destination.password().is_some()
+    {
+        return None;
+    }
+    Some(destination.into())
+}
+
+fn parse(html: &str, limit: usize) -> Outcome {
+    if html.len() > MAX_BODY_BYTES {
+        return Err("body-limit");
+    }
+    if limit == 0 || limit > 20 {
+        return Err("result-limit");
+    }
+    let document = Html::parse_document(html);
+    if document
+        .select(&selector(
+            "#challenge-form, #anomaly-modal, .anomaly-modal, form[action*='anomaly.js']",
+        ))
+        .next()
+        .is_some()
+    {
+        return Err("challenge");
+    }
+    let result_selector = selector("div.result");
+    let title_selector = selector("a.result__a");
+    let snippet_selector = selector(".result__snippet");
+    let mut results = Vec::new();
+    let mut result_containers = 0;
+    for element in document.select(&result_selector) {
+        if element
+            .value()
+            .classes()
+            .any(|class| class == "result--no-result")
+        {
+            continue;
+        }
+        result_containers += 1;
+        let Some(title_element) = element.select(&title_selector).next() else {
+            continue;
+        };
+        let title = title_element
+            .text()
+            .collect::<String>()
+            .split_whitespace()
+            .collect::<Vec<_>>()
+            .join(" ");
+        let Some(url) = title_element.value().attr("href").and_then(destination) else {
+            continue;
+        };
+        if title.is_empty() {
+            continue;
+        }
+        let snippet = element
+            .select(&snippet_selector)
+            .next()
+            .map(|element| {
+                element
+                    .text()
+                    .collect::<String>()
+                    .split_whitespace()
+                    .collect::<Vec<_>>()
+                    .join(" ")
+            })
+            .filter(|text| !text.is_empty());
+        results.push(SearchResult {
+            title,
+            url,
+            snippet,
+            source_engine: "duckduckgo".to_string(),
+        });
+        if results.len() == limit {
+            break;
+        }
+    }
+    if !results.is_empty() {
+        return Ok(results);
+    }
+    // The known no-results message is accepted only if there are no competing
+    // organic result containers. Generic page prose is not a no-results marker.
+    let no_results = document
+        .select(&selector(".no-results, .result--no-result .result__body"))
+        .any(|element| {
+            element
+                .text()
+                .collect::<String>()
+                .split_whitespace()
+                .collect::<Vec<_>>()
+                .join(" ")
+                .starts_with("No results found for")
+        });
+    if result_containers == 0 && no_results {
+        Ok(Vec::new())
+    } else {
+        Err("unrecognized-html")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn result(href: &str) -> String {
+        format!(
+            r#"<div class="result"><a class="result__a" href="{href}">Python <b>Tasks</b></a><a class="result__snippet">Read tasks.</a></div>"#
+        )
+    }
+
+    #[test]
+    fn parses_direct_and_redirect_results() {
+        for href in [
+            "https://docs.python.org/3/library/asyncio-task.html",
+            "/l/?uddg=https%3A%2F%2Fdocs.python.org%2F3%2Flibrary%2Fasyncio-task.html",
+            "//duckduckgo.com/l/?uddg=https%3A%2F%2Fdocs.python.org%2F3%2Flibrary%2Fasyncio-task.html",
+            "https://duckduckgo.com/l/?uddg=https%3A%2F%2Fdocs.python.org%2F3%2Flibrary%2Fasyncio-task.html",
+        ] {
+            let results = parse(&result(href), 5).unwrap();
+            assert_eq!(results.len(), 1);
+            assert_eq!(results[0].title, "Python Tasks");
+            assert_eq!(
+                results[0].url,
+                "https://docs.python.org/3/library/asyncio-task.html"
+            );
+            assert_eq!(results[0].snippet.as_deref(), Some("Read tasks."));
+        }
+    }
+
+    #[test]
+    fn rejects_challenge_even_with_injected_results() {
+        for marker in [
+            "<form id='challenge-form'></form>",
+            "<div class='anomaly-modal'></div>",
+            "<form action='//duckduckgo.com/anomaly.js'></form>",
+        ] {
+            assert_eq!(
+                parse(&format!("{marker}{}", result("https://example.org")), 5).unwrap_err(),
+                "challenge"
+            );
+        }
+    }
+
+    #[test]
+    fn only_explicit_no_results_is_empty_success() {
+        assert!(
+            parse(
+                "<div class='no-results'>No results found for <b>query</b></div>",
+                5
+            )
+            .unwrap()
+            .is_empty()
+        );
+        for html in [
+            "",
+            "<html>Consent required</html>",
+            "<p>No results found for query</p>",
+            "<div class='no-results'>changed markup</div>",
+            "<div class='result'><h2>unrecognized</h2></div>",
+            "<div class='no-results'>No results found for query</div><div class='result'></div>",
+        ] {
+            assert_eq!(parse(html, 5).unwrap_err(), "unrecognized-html");
+        }
+    }
+
+    #[test]
+    fn rejects_invalid_destinations() {
+        for href in [
+            "javascript:alert(1)",
+            "https://user:secret@example.org",
+            "/settings",
+            "/l/?uddg=javascript%3Aalert(1)",
+            "/l/?uddg=https%3A%2F%2Fa.org&uddg=https%3A%2F%2Fb.org",
+        ] {
+            assert!(destination(href).is_none(), "{href}");
+            assert_eq!(parse(&result(href), 5).unwrap_err(), "unrecognized-html");
+        }
+    }
+
+    #[test]
+    fn enforces_result_and_body_limits() {
+        assert_eq!(
+            parse(&result("https://example.org").repeat(5), 2)
+                .unwrap()
+                .len(),
+            2
+        );
+        assert_eq!(parse("", 0).unwrap_err(), "result-limit");
+        assert_eq!(parse("", 21).unwrap_err(), "result-limit");
+        let mut bytes = vec![0; MAX_BODY_BYTES - 1];
+        append_body(&mut bytes, &[0]).unwrap();
+        assert_eq!(append_body(&mut bytes, &[1]).unwrap_err(), "body-limit");
+        assert_eq!(bytes.len(), MAX_BODY_BYTES);
+        assert_eq!(
+            parse(&"x".repeat(MAX_BODY_BYTES + 1), 1).unwrap_err(),
+            "body-limit"
+        );
+    }
+    #[test]
+    fn validates_http_metadata_before_reading() {
+        let mut headers = header::HeaderMap::new();
+        headers.insert(
+            header::CONTENT_TYPE,
+            header::HeaderValue::from_static("text/html; charset=utf-8"),
+        );
+        assert!(validate_response(reqwest::StatusCode::OK, &headers, None).is_ok());
+        for status in [302, 403, 429, 500] {
+            assert_eq!(
+                validate_response(
+                    reqwest::StatusCode::from_u16(status).unwrap(),
+                    &headers,
+                    None
+                ),
+                Err("http-status")
+            );
+        }
+        assert_eq!(
+            validate_response(
+                reqwest::StatusCode::OK,
+                &headers,
+                Some(MAX_BODY_BYTES as u64 + 1)
+            ),
+            Err("body-limit")
+        );
+        headers.insert(
+            header::CONTENT_ENCODING,
+            header::HeaderValue::from_static("deflate"),
+        );
+        assert_eq!(
+            validate_response(reqwest::StatusCode::OK, &headers, None),
+            Err("content-encoding")
+        );
+        headers.remove(header::CONTENT_ENCODING);
+        headers.insert(
+            header::CONTENT_TYPE,
+            header::HeaderValue::from_static("application/json"),
+        );
+        assert_eq!(
+            validate_response(reqwest::StatusCode::OK, &headers, None),
+            Err("content-type")
+        );
+    }
+
+    #[test]
+    fn result_limits_preserve_prefix_and_destination_roundtrips() {
+        for count in 1..=25 {
+            let urls: Vec<_> = (0..count)
+                .map(|index| format!("https://example.org/path/{index}?a=one%20two&b=%26"))
+                .collect();
+            let html = urls
+                .iter()
+                .map(|url| {
+                    let mut wrapper = Url::parse("https://duckduckgo.com/l/").unwrap();
+                    wrapper.query_pairs_mut().append_pair("uddg", url);
+                    result(wrapper.as_str())
+                })
+                .collect::<String>();
+            for limit in 1..=20 {
+                let parsed = parse(&html, limit).unwrap();
+                assert_eq!(parsed.len(), count.min(limit));
+                assert_eq!(
+                    parsed.iter().map(|item| &item.url).collect::<Vec<_>>(),
+                    urls.iter().take(limit).collect::<Vec<_>>()
+                );
+            }
+        }
+    }
+    fn fixture(
+        status: &str,
+        headers: &str,
+        body: Vec<u8>,
+        body_delay: Duration,
+    ) -> (String, std::thread::JoinHandle<String>) {
+        use std::io::{Read, Write};
+        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+        listener.set_nonblocking(true).unwrap();
+        let url = format!("http://{}/html/", listener.local_addr().unwrap());
+        let response = format!(
+            "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n{headers}\r\n",
+            body.len()
+        );
+        let handle = std::thread::spawn(move || {
+            let deadline = std::time::Instant::now() + Duration::from_secs(3);
+            let mut stream = loop {
+                match listener.accept() {
+                    Ok((stream, _)) => break stream,
+                    Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
+                        assert!(
+                            std::time::Instant::now() < deadline,
+                            "fixture accept timeout"
+                        );
+                        std::thread::sleep(Duration::from_millis(2));
+                    }
+                    Err(error) => panic!("fixture accept: {error}"),
+                }
+            };
+            stream
+                .set_read_timeout(Some(Duration::from_secs(2)))
+                .unwrap();
+            stream
+                .set_write_timeout(Some(Duration::from_secs(2)))
+                .unwrap();
+            let mut request = Vec::new();
+            let mut byte = [0];
+            while !request.ends_with(b"\r\n\r\n") {
+                assert!(request.len() < 16 * 1024);
+                if stream.read(&mut byte).unwrap() == 0 {
+                    break;
+                }
+                request.push(byte[0]);
+            }
+            let _ = stream.write_all(response.as_bytes());
+            std::thread::sleep(body_delay);
+            let _ = stream.write_all(&body);
+            String::from_utf8(request).unwrap()
+        });
+        (url, handle)
+    }
+
+    fn client() -> Client {
+        // Exercise the actual production header/decoder setup. The fixture
+        // address is explicit; no production DNS/private-address policy changes.
+        super::super::build_pinned_http_client(&[], Duration::from_secs(2)).unwrap()
+    }
+
+    #[tokio::test]
+    async fn transport_uses_supported_encodings_and_parses_html() {
+        let (url, server) = fixture(
+            "200 OK",
+            "Content-Type: text/html\r\n",
+            result("https://example.org/docs").into_bytes(),
+            Duration::ZERO,
+        );
+        let outcome = search_at(
+            &client(),
+            &url,
+            "two words + plus",
+            5,
+            Duration::from_secs(2),
+        )
+        .await;
+        let request = server.join().unwrap().to_ascii_lowercase();
+        assert_eq!(outcome.unwrap().len(), 1);
+        let encoding = request
+            .lines()
+            .find(|line| line.starts_with("accept-encoding:"))
+            .unwrap();
+        assert!(encoding.contains("gzip") && encoding.contains("br"));
+        assert!(!encoding.contains("deflate"));
+        assert!(request.starts_with("get /html/?q=two+words+%2b+plus "));
+    }
+
+    #[tokio::test]
+    async fn transport_rejects_status_type_encoding_and_slow_body() {
+        for (status, headers, delay, expected) in [
+            (
+                "403 Forbidden",
+                "Content-Type: text/html\r\n",
+                Duration::ZERO,
+                "http-status",
+            ),
+            (
+                "302 Found",
+                "Content-Type: text/html\r\nLocation: http://127.0.0.1:1/\r\n",
+                Duration::ZERO,
+                "http-status",
+            ),
+            (
+                "200 OK",
+                "Content-Type: application/json\r\n",
+                Duration::ZERO,
+                "content-type",
+            ),
+            (
+                "200 OK",
+                "Content-Type: text/html\r\nContent-Encoding: deflate\r\n",
+                Duration::ZERO,
+                "content-encoding",
+            ),
+            (
+                "200 OK",
+                "Content-Type: text/html\r\n",
+                Duration::from_millis(150),
+                "timeout",
+            ),
+        ] {
+            let (url, server) = fixture(status, headers, b"some body".to_vec(), delay);
+            let timeout = if delay.is_zero() {
+                Duration::from_secs(2)
+            } else {
+                Duration::from_millis(50)
+            };
+            let outcome = search_at(&client(), &url, "public fixture", 5, timeout).await;
+            server.join().unwrap();
+            assert_eq!(outcome.unwrap_err(), expected);
+        }
+    }
+
+    #[tokio::test]
+    async fn transport_bounds_decompressed_body() {
+        // Deterministic gzip of MAX_BODY_BYTES + 1 ASCII x bytes, generated
+        // offline. The encoded body is small; the decoded size must be bounded.
+        let encoded = "1f8b08000000000002ffedc13101000000c2a0da8b6f0c1fa000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ce0641d1a7a101002000";
+        let bytes: Vec<u8> = encoded
+            .as_bytes()
+            .chunks_exact(2)
+            .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap())
+            .collect();
+        let (url, server) = fixture(
+            "200 OK",
+            "Content-Type: text/html\r\nContent-Encoding: gzip\r\n",
+            bytes,
+            Duration::ZERO,
+        );
+        let outcome = search_at(&client(), &url, "fixture", 5, Duration::from_secs(2)).await;
+        server.join().unwrap();
+        assert_eq!(outcome.unwrap_err(), "body-limit");
+    }
+}

package/src/clip/metadata-search-tool/src/main.rs

@@ -1,3 +1,5 @@
+mod duckduckgo;
+
 use std::alloc::{GlobalAlloc, Layout, System};
 use std::io::{self, Read};
 use std::net::{IpAddr, SocketAddr};
@@ -8,9 +10,7 @@ use std::time::Duration;
 
 use metadata_search_engine_rs::{
     aggregator::aggregate,
-    engines::{
-        BraveEngine, DuckDuckGoEngine, SearchEngine, StartpageEngine, YahooEngine,
-    },
+    engines::{BraveEngine, SearchEngine, StartpageEngine, YahooEngine},
     models::SearchResponse,
 };
 use reqwest::{
@@ -225,8 +225,7 @@ fn is_private_or_reserved(address: IpAddr) -> bool {
         IpAddr::V6(address) => {
             let groups = address.segments();
             let ipv4_compatible = groups[..6].iter().all(|group| *group == 0);
-            let ipv4_mapped = groups[..5].iter().all(|group| *group == 0)
-                && groups[5] == 0xffff;
+            let ipv4_mapped = groups[..5].iter().all(|group| *group == 0) && groups[5] == 0xffff;
             if ipv4_compatible || ipv4_mapped {
                 let high = groups[6];
                 let low = groups[7];
@@ -291,18 +290,12 @@ fn build_pinned_http_client(
     );
     headers.insert(
         header::ACCEPT,
-        HeaderValue::from_static(
-            "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-        ),
+        HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
     );
     headers.insert(
         header::ACCEPT_LANGUAGE,
         HeaderValue::from_static("en-US,en;q=0.9"),
     );
-    headers.insert(
-        header::ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br"),
-    );
     headers.insert("DNT", HeaderValue::from_static("1"));
     headers.insert("Sec-Fetch-Dest", HeaderValue::from_static("document"));
     headers.insert("Sec-Fetch-Mode", HeaderValue::from_static("navigate"));
@@ -341,21 +334,27 @@ async fn run() {
     };
     let timeout = Duration::from_millis(request.timeout_ms / ENGINE_HOSTS.len() as u64);
     let client = Arc::new(
-        build_pinned_http_client(&request.engine_hosts, timeout)
-            .unwrap_or_else(|code| fail(code)),
+        build_pinned_http_client(&request.engine_hosts, timeout).unwrap_or_else(|code| fail(code)),
     );
     let engines: Vec<Arc<dyn SearchEngine>> = vec![
-        Arc::new(DuckDuckGoEngine::with_timeout(Arc::clone(&client), timeout)),
         Arc::new(BraveEngine::with_timeout(Arc::clone(&client), timeout)),
         Arc::new(StartpageEngine::with_timeout(Arc::clone(&client), timeout)),
         Arc::new(YahooEngine::with_timeout(Arc::clone(&client), timeout)),
     ];
-    let engines_queried = engines
+    let mut engines_queried = engines
         .iter()
         .map(|engine| engine.name().to_string())
         .collect::<Vec<_>>();
+    engines_queried.insert(0, "duckduckgo".to_string());
     let mut successes = Vec::new();
     let mut engines_failed = Vec::new();
+    match duckduckgo::search(&client, request.query.trim(), request.max_results, timeout).await {
+        Ok(results) => successes.push(("duckduckgo".to_string(), results)),
+        Err(code) => {
+            eprintln!("metadata search engine duckduckgo: {code}");
+            engines_failed.push("duckduckgo".to_string());
+        }
+    }
     // Run one upstream engine at a time so its unbounded internal `text()`
     // allocation is contained by the process memory ceiling rather than
     // multiplied across four concurrent responses.
@@ -365,8 +364,22 @@ async fn run() {
             .search(request.query.trim(), request.max_results)
             .await
         {
-            Ok(results) => successes.push((name, results)),
-            Err(_) => engines_failed.push(name),
+            Ok(results) if !results.is_empty() => successes.push((name, results)),
+            Ok(_) => {
+                eprintln!("metadata search engine {name}: unverified-empty");
+                engines_failed.push(name);
+            }
+            Err(error) => {
+                use metadata_search_engine_rs::error::EngineError;
+                let code = match error {
+                    EngineError::Timeout { .. } => "timeout",
+                    EngineError::Http { .. } => "http",
+                    EngineError::BadStatus { .. } => "http-status",
+                    EngineError::ParseFailed { .. } => "parse",
+                };
+                eprintln!("metadata search engine {name}: {code}");
+                engines_failed.push(name);
+            }
         }
     }
     let mut results = aggregate(successes, request.max_results);

package/src/clip/metadata-search.ts

@@ -495,7 +495,7 @@ async function resolveEngineHosts(
       allowPrivateNetwork: false,
       timeoutMs,
     });
-    if (addresses.length === 0 || addresses.length > MAX_ENGINE_ADDRESSES) {
+    if (addresses.length === 0) {
       throw new Error("metadata search engine DNS answer count is invalid");
     }
     const unique = new Map<string, ResolvedNetworkAddress>();
@@ -511,8 +511,10 @@ async function resolveEngineHosts(
     if (unique.size !== addresses.length) throw new Error("metadata search engine DNS answer is duplicated");
     return Object.freeze({
       hostname,
+      // Validate every DNS answer before selecting the bounded transport candidates.
       addresses: Object.freeze([...unique.values()].sort((left, right) =>
-        left.family - right.family || (left.address < right.address ? -1 : left.address > right.address ? 1 : 0))),
+        left.family - right.family || (left.address < right.address ? -1 : left.address > right.address ? 1 : 0))
+        .slice(0, MAX_ENGINE_ADDRESSES)),
     });
   }));
   return Object.freeze(resolved);

package/src/clip/url-metadata.ts

@@ -1,4 +1,5 @@
 import { execFileSync } from "node:child_process";
+import { syncsDirectories } from "../directory-sync.js";
 import { randomUUID } from "node:crypto";
 import {
   closeSync,
@@ -1371,7 +1372,7 @@ export function writeUrlMetadataDocument(
       throw new Error(`URL metadata sidecar failed post-install verification: ${saved.sidecarPath}`);
     }
     assertHeldUrlMetadataWriteLockCurrent(heldLock);
-    fsyncSync(directoryDescriptor);
+    if (syncsDirectories()) fsyncSync(directoryDescriptor);
     return { changed: true, path: saved.sidecarPath };
   } catch (error) {
     operationError = error;

package/src/directory-sync.ts

@@ -0,0 +1,9 @@
+/**
+ * Whether this platform can fsync a directory handle to persist a rename or
+ * link. POSIX filesystems need it. Windows refuses (fsync on a directory
+ * handle fails with EPERM), and NTFS journals the directory metadata itself,
+ * so callers skip the directory sync there and keep every file fsync.
+ */
+export function syncsDirectories(platform: NodeJS.Platform = process.platform): boolean {
+  return platform !== "win32";
+}