valid_email_checker 0.1.0

data/ext/valid_email_checker/src/lib.rs

@@ -0,0 +1,371 @@
+//! Native extension backing the `valid_email_checker` gem.
+//!
+//! This is a thin bridge onto the embedded `check-if-email-exists` crate under
+//! `vendor/`. Everything crossing the boundary is JSON: Ruby hands down an
+//! options object, and gets back the upstream `CheckEmailOutput` serialized
+//! verbatim. Keeping the seam at JSON means the gem exposes the upstream
+//! result shape without this file having to mirror every upstream struct.
+
+mod nogvl;
+
+use std::collections::HashMap;
+use std::time::Duration;
+
+use check_if_email_exists::smtp::verif_method::{
+    EverythingElseVerifMethod, GmailVerifMethod, HotmailB2BVerifMethod, HotmailB2CVerifMethod,
+    MimecastVerifMethod, ProofpointVerifMethod, VerifMethod, VerifMethodSmtpConfig,
+    YahooVerifMethod, DEFAULT_PROXY_ID,
+};
+use check_if_email_exists::syntax::{check_syntax, get_similar_mail_provider};
+use check_if_email_exists::{check_email, CheckEmailInput, CheckEmailInputProxy};
+use futures::stream::{self, StreamExt};
+use magnus::{function, prelude::*, Error, Ruby};
+use serde::Deserialize;
+
+use nogvl::nogvl;
+
+const UPSTREAM_JSON: &str = include_str!("../vendor/UPSTREAM.json");
+
+/// Upstream's own defaults, restated here so the gem behaves identically to
+/// `check-if-email-exists` when the caller configures nothing.
+const DEFAULT_FROM_EMAIL: &str = "reacher@gmail.com";
+const DEFAULT_HELLO_NAME: &str = "gmail.com";
+const DEFAULT_WEBDRIVER_ADDR: &str = "http://localhost:9515";
+const DEFAULT_BACKEND_NAME: &str = "valid_email_checker";
+const DEFAULT_SMTP_PORT: u16 = 25;
+const DEFAULT_RETRIES: usize = 1;
+
+/// How to verify addresses at providers that offer more than one strategy.
+#[derive(Debug, Clone, Copy, PartialEq, Deserialize)]
+#[serde(rename_all = "lowercase")]
+enum ProviderMethod {
+    /// Talk to the provider's SMTP server.
+    Smtp,
+    /// Use the provider's own HTTP endpoint (Yahoo only).
+    Api,
+    /// Drive the provider's password-recovery page through a WebDriver.
+    Headless,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+struct ProxyOptions {
+    host: String,
+    port: u16,
+    #[serde(default)]
+    username: Option<String>,
+    #[serde(default)]
+    password: Option<String>,
+    #[serde(default)]
+    timeout_ms: Option<u64>,
+}
+
+impl From<ProxyOptions> for CheckEmailInputProxy {
+    fn from(p: ProxyOptions) -> Self {
+        CheckEmailInputProxy {
+            host: p.host,
+            port: p.port,
+            username: p.username,
+            password: p.password,
+            timeout_ms: p.timeout_ms,
+        }
+    }
+}
+
+/// Every field is optional so that Ruby can pass `nil` for anything it has no
+/// opinion about; defaults are resolved here rather than in serde, which keeps
+/// the "unset" and "explicitly nil" cases identical.
+#[derive(Debug, Clone, Default, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+struct Options {
+    from_email: Option<String>,
+    hello_name: Option<String>,
+    smtp_port: Option<u16>,
+    smtp_timeout_ms: Option<u64>,
+    retries: Option<usize>,
+    check_gravatar: Option<bool>,
+    haveibeenpwned_api_key: Option<String>,
+    proxy: Option<ProxyOptions>,
+    webdriver_addr: Option<String>,
+    backend_name: Option<String>,
+    timeout_ms: Option<u64>,
+    yahoo: Option<ProviderMethod>,
+    hotmail_b2c: Option<ProviderMethod>,
+}
+
+impl Options {
+    /// Build the upstream input for one address.
+    ///
+    /// Note the provider defaults: unlike upstream, Yahoo defaults to its HTTP
+    /// API and Hotmail B2C to SMTP, so that a freshly installed gem works
+    /// without a WebDriver running. Pass `yahoo: :headless` /
+    /// `hotmail_b2c: :headless` to opt back into upstream's behaviour.
+    fn to_input(&self, email: &str) -> Result<CheckEmailInput, String> {
+        let mut proxies = HashMap::new();
+        let proxy_id = match &self.proxy {
+            Some(proxy) => {
+                proxies.insert(DEFAULT_PROXY_ID.to_string(), proxy.clone().into());
+                Some(DEFAULT_PROXY_ID.to_string())
+            }
+            None => None,
+        };
+
+        let smtp_config = VerifMethodSmtpConfig {
+            from_email: self
+                .from_email
+                .clone()
+                .unwrap_or_else(|| DEFAULT_FROM_EMAIL.to_string()),
+            hello_name: self
+                .hello_name
+                .clone()
+                .unwrap_or_else(|| DEFAULT_HELLO_NAME.to_string()),
+            proxy: proxy_id,
+            smtp_port: self.smtp_port.unwrap_or(DEFAULT_SMTP_PORT),
+            smtp_timeout: self.smtp_timeout_ms.map(Duration::from_millis),
+            retries: self.retries.unwrap_or(DEFAULT_RETRIES),
+        };
+
+        let yahoo = match self.yahoo.unwrap_or(ProviderMethod::Api) {
+            ProviderMethod::Api => YahooVerifMethod::Api,
+            ProviderMethod::Headless => YahooVerifMethod::Headless,
+            ProviderMethod::Smtp => YahooVerifMethod::Smtp(smtp_config.clone()),
+        };
+
+        let hotmailb2c = match self.hotmail_b2c.unwrap_or(ProviderMethod::Smtp) {
+            ProviderMethod::Smtp => HotmailB2CVerifMethod::Smtp(smtp_config.clone()),
+            ProviderMethod::Headless => HotmailB2CVerifMethod::Headless,
+            ProviderMethod::Api => {
+                return Err("hotmail_b2c does not support :api; use :smtp or :headless".to_string())
+            }
+        };
+
+        let verif_method = VerifMethod {
+            proxies,
+            gmail: GmailVerifMethod::Smtp(smtp_config.clone()),
+            hotmailb2b: HotmailB2BVerifMethod::Smtp(smtp_config.clone()),
+            hotmailb2c,
+            mimecast: MimecastVerifMethod::Smtp(smtp_config.clone()),
+            proofpoint: ProofpointVerifMethod::Smtp(smtp_config.clone()),
+            yahoo,
+            everything_else: EverythingElseVerifMethod::Smtp(smtp_config),
+        };
+
+        verif_method
+            .validate_proxies()
+            .map_err(|e| format!("invalid proxy configuration: {e}"))?;
+
+        Ok(CheckEmailInput {
+            to_email: email.to_string(),
+            verif_method,
+            check_gravatar: self.check_gravatar.unwrap_or(false),
+            haveibeenpwned_api_key: self.haveibeenpwned_api_key.clone(),
+            webdriver_addr: self
+                .webdriver_addr
+                .clone()
+                .unwrap_or_else(|| DEFAULT_WEBDRIVER_ADDR.to_string()),
+            webdriver_config: Default::default(),
+            backend_name: self
+                .backend_name
+                .clone()
+                .unwrap_or_else(|| DEFAULT_BACKEND_NAME.to_string()),
+            sentry_dsn: None,
+        })
+    }
+}
+
+/// A tokio runtime is built per call rather than cached in a static.
+///
+/// Runtime creation costs well under a millisecond against verifications that
+/// take seconds, and building it on demand means there is no cached runtime to
+/// be left with dead worker threads if the process forks.
+///
+/// Note that this does not make verification fork-safe on its own: the system
+/// network stack has its own inherited state, which the Ruby side guards
+/// against. See `ValidEmailChecker::ForkAfterVerificationError`.
+fn runtime() -> Result<tokio::runtime::Runtime, String> {
+    tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(2)
+        .enable_all()
+        .build()
+        .map_err(|e| format!("could not start the async runtime: {e}"))
+}
+
+/// Serialize one verification, substituting a synthetic record when the
+/// wall-clock budget runs out.
+///
+/// The timeout result keeps the same top-level shape as a real one — an
+/// `input` and an `is_reachable` of `unknown` — so Ruby can wrap every entry
+/// in the same `Result` object and callers do not need a separate code path.
+async fn check_one_to_json(input: CheckEmailInput, timeout: Option<Duration>) -> String {
+    let email = input.to_email.clone();
+
+    let output = match timeout {
+        Some(duration) => match tokio::time::timeout(duration, check_email(&input)).await {
+            Ok(output) => output,
+            Err(_) => {
+                return serde_json::json!({
+                    "input": email,
+                    "is_reachable": "unknown",
+                    "error": {
+                        "type": "Timeout",
+                        "message": format!("verification timed out after {}ms", duration.as_millis()),
+                    },
+                })
+                .to_string()
+            }
+        },
+        None => check_email(&input).await,
+    };
+
+    serde_json::to_string(&output)
+        .unwrap_or_else(|e| panic!("upstream result for {email} could not be serialized: {e}"))
+}
+
+fn parse_options(ruby: &Ruby, options_json: String) -> Result<Options, Error> {
+    serde_json::from_str::<Options>(&options_json)
+        .map_err(|e| Error::new(ruby.exception_arg_error(), format!("invalid options: {e}")))
+}
+
+fn config_error(ruby: &Ruby, message: String) -> Error {
+    Error::new(ruby.exception_arg_error(), message)
+}
+
+/// Re-raise a panic from the verification thread as a Ruby exception, now that
+/// we hold the GVL again.
+fn panic_to_error(ruby: &Ruby, payload: Box<dyn std::any::Any + Send>) -> Error {
+    let detail = if let Some(s) = payload.downcast_ref::<&str>() {
+        (*s).to_string()
+    } else if let Some(s) = payload.downcast_ref::<String>() {
+        s.clone()
+    } else {
+        "unknown panic".to_string()
+    };
+
+    Error::new(
+        ruby.exception_runtime_error(),
+        format!("valid_email_checker: the native verifier panicked: {detail}"),
+    )
+}
+
+/// Syntax-only check. Pure CPU work, no network, so the GVL stays held.
+///
+/// Upstream fills in the typo `suggestion` from a separate call, which
+/// `check_email` only reaches on its failure paths. We always populate it,
+/// since a caller asking about syntax alone is exactly who wants "did you mean
+/// gmail.com?".
+fn rb_check_syntax(ruby: &Ruby, email: String) -> Result<String, Error> {
+    let mut details = check_syntax(&email);
+    get_similar_mail_provider(&mut details);
+
+    // That helper matches any provider within a levenshtein distance of 3,
+    // which includes a distance of 0 -- so a correct domain "suggests" itself.
+    // Upstream only calls it on failure paths, where that is harmless noise.
+    // We call it on every syntax check, so drop anything that is not actually
+    // a correction.
+    let suggests_same_domain = details
+        .suggestion
+        .as_deref()
+        .and_then(|suggestion| suggestion.rsplit_once('@'))
+        .map(|(_, domain)| domain == details.domain)
+        .unwrap_or(false);
+
+    if suggests_same_domain {
+        details.suggestion = None;
+    }
+
+    serde_json::to_string(&details).map_err(|e| {
+        Error::new(
+            ruby.exception_runtime_error(),
+            format!("could not serialize syntax details: {e}"),
+        )
+    })
+}
+
+/// Verify a single address. Returns the upstream result as a JSON string.
+fn rb_check(ruby: &Ruby, email: String, options_json: String) -> Result<String, Error> {
+    let options = parse_options(ruby, options_json)?;
+    let timeout = options.timeout_ms.map(Duration::from_millis);
+    let input = options
+        .to_input(&email)
+        .map_err(|e| config_error(ruby, e))?;
+
+    let work = move || {
+        let runtime = runtime()?;
+        Ok(runtime.block_on(check_one_to_json(input, timeout)))
+    };
+
+    nogvl(work)
+        .map_err(|payload| panic_to_error(ruby, payload))?
+        .map_err(|e| config_error(ruby, e))
+}
+
+/// Verify several addresses concurrently on a single runtime.
+///
+/// This is the main reason to embed the crate rather than shell out: one
+/// GVL release and one runtime cover the whole batch, and `concurrency`
+/// verifications proceed at once. Results come back in the order the emails
+/// were given, regardless of which finished first.
+fn rb_check_many(
+    ruby: &Ruby,
+    emails: Vec<String>,
+    options_json: String,
+    concurrency: usize,
+) -> Result<String, Error> {
+    let options = parse_options(ruby, options_json)?;
+    let timeout = options.timeout_ms.map(Duration::from_millis);
+    let concurrency = concurrency.max(1);
+
+    let inputs = emails
+        .iter()
+        .map(|email| options.to_input(email))
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|e| config_error(ruby, e))?;
+
+    let work = move || {
+        let runtime = runtime()?;
+
+        let results = runtime.block_on(async move {
+            let mut results: Vec<(usize, String)> = stream::iter(inputs.into_iter().enumerate())
+                .map(|(index, input)| async move {
+                    (index, check_one_to_json(input, timeout).await)
+                })
+                .buffer_unordered(concurrency)
+                .collect()
+                .await;
+
+            results.sort_by_key(|(index, _)| *index);
+            results
+        });
+
+        // Each element is already a JSON document, so splice them into an
+        // array rather than round-tripping through serde_json::Value.
+        let joined = results
+            .into_iter()
+            .map(|(_, json)| json)
+            .collect::<Vec<_>>()
+            .join(",");
+
+        Ok(format!("[{joined}]"))
+    };
+
+    nogvl(work)
+        .map_err(|payload| panic_to_error(ruby, payload))?
+        .map_err(|e| config_error(ruby, e))
+}
+
+/// Provenance of the embedded crate, as JSON.
+fn rb_upstream() -> &'static str {
+    UPSTREAM_JSON
+}
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let parent = ruby.define_module("ValidEmailChecker")?;
+    let module = parent.define_module("Native")?;
+
+    module.define_singleton_method("check_syntax", function!(rb_check_syntax, 1))?;
+    module.define_singleton_method("check", function!(rb_check, 2))?;
+    module.define_singleton_method("check_many", function!(rb_check_many, 3))?;
+    module.define_singleton_method("upstream", function!(rb_upstream, 0))?;
+
+    Ok(())
+}

data/ext/valid_email_checker/src/nogvl.rs

@@ -0,0 +1,64 @@
+//! Releasing the GVL around blocking verification work.
+//!
+//! Verifying an email address is network-bound: it does DNS lookups and then
+//! holds an SMTP conversation, which routinely takes seconds. Holding Ruby's
+//! global VM lock for that whole time would freeze every other thread in the
+//! process, so we hand the work to `rb_thread_call_without_gvl`.
+//!
+//! magnus does not wrap that function (it is on its explicit "not yet
+//! wrapped" list), so we reach for the raw binding in `rb-sys`.
+
+use std::ffi::c_void;
+use std::panic::{catch_unwind, AssertUnwindSafe};
+
+/// Run `func` with the GVL released.
+///
+/// A panic inside `func` is caught rather than allowed to unwind across the
+/// `extern "C"` boundary, which would be undefined behaviour. It is returned
+/// to the caller so it can be re-raised once the GVL is held again.
+pub fn nogvl<F, R>(func: F) -> std::thread::Result<R>
+where
+    F: FnOnce() -> R,
+{
+    struct Args<F, R> {
+        func: Option<F>,
+        result: Option<std::thread::Result<R>>,
+    }
+
+    unsafe extern "C" fn call<F, R>(arg: *mut c_void) -> *mut c_void
+    where
+        F: FnOnce() -> R,
+    {
+        // Safety: `arg` is the `&mut Args` we pass below. Ruby invokes this
+        // callback exactly once, on this same thread, before
+        // `rb_thread_call_without_gvl` returns, so the reference is live and
+        // uniquely held for the duration.
+        let args = unsafe { &mut *(arg as *mut Args<F, R>) };
+        let func = args
+            .func
+            .take()
+            .expect("nogvl callback invoked more than once");
+        args.result = Some(catch_unwind(AssertUnwindSafe(func)));
+        std::ptr::null_mut()
+    }
+
+    let mut args: Args<F, R> = Args {
+        func: Some(func),
+        result: None,
+    };
+
+    // We deliberately pass no unblocking function. The alternative is to let
+    // Ruby interrupt the thread, but the actual work happens on tokio worker
+    // threads that would keep running, so there would be nothing to cancel.
+    // Instead, callers bound the blocking region with a wall-clock timeout.
+    unsafe {
+        rb_sys::rb_thread_call_without_gvl(
+            Some(call::<F, R>),
+            (&mut args) as *mut Args<F, R> as *mut c_void,
+            None,
+            std::ptr::null_mut(),
+        );
+    }
+
+    args.result.expect("nogvl callback never ran")
+}

data/ext/valid_email_checker/vendor/UPSTREAM.json

@@ -0,0 +1,9 @@
+{
+  "name": "check-if-email-exists",
+  "version": "0.11.7",
+  "commit": "81da93e8a419f601c05e1bcf4a197d25eb0e349d",
+  "repository": "https://github.com/reacherhq/check-if-email-exists",
+  "license": "AGPL-3.0-only OR Commercial",
+  "vendored_path": "ext/valid_email_checker/vendor/check-if-email-exists",
+  "notes": "Verbatim copy of the upstream `core` package (the check-if-email-exists library crate). Only Cargo.toml's `publish` and `readme` keys were changed; see the comment in that file."
+}

data/ext/valid_email_checker/vendor/check-if-email-exists/Cargo.toml

@@ -0,0 +1,52 @@
+[package]
+name = "check-if-email-exists"
+version = "0.11.7"
+authors = ["Amaury <amaury@reacher.email>"]
+categories = ["email"]
+description = "Check if an email address exists without sending any email"
+documentation = "http://docs.rs/check-if-email-exists"
+edition = "2018"
+keywords = ["email", "smtp", "mx", "verification"]
+license = "AGPL-3.0"
+# Vendored into the valid_email_checker gem. Two lines differ from upstream:
+# `publish` is false (this copy must never be pushed to crates.io) and `readme`
+# points at the copy of README.md alongside this file rather than `../`.
+publish = false
+readme = "README.md"
+repository = "https://github.com/reacherhq/check-if-email-exists"
+
+[dependencies]
+anyhow = "1.0"
+async-recursion = "1.0.5"
+async-smtp = { version = "0.9.1", features = ["runtime-tokio"] }
+chrono = { version = "0.4.31", features = ["serde"] }
+config = "0.14"
+derive_builder = "0.20"
+fast-socks5 = "0.9"
+fantoccini = { version = "0.21.2", default-features = false, features = [
+    "rustls-tls",
+] }
+futures = { version = "0.3.30" }
+hickory-proto = "0.24.0"
+hickory-resolver = "0.24.0"
+levenshtein = "1.0.5"
+log = "0.4.26"
+mailchecker = "6.0.11"
+md5 = "0.7.0"
+once_cell = "1.21.1"
+rand = { version = "0.8.5", features = ["small_rng"] }
+regex = "1.11.1"
+reqwest = { version = "0.12.15", default-features = false, features = [
+    "json",
+    "rustls-tls",
+] }
+rustls = { version = "0.23", features = ["ring"] }
+sentry = { version = "0.36", default-features = false, features = [
+    "reqwest",
+    "rustls",
+], optional = true }
+serde = { version = "1.0.214", features = ["derive"] }
+serde_json = "1.0.133"
+thiserror = "2.0"
+tokio = { version = "1.40.0", features = ["rt-multi-thread", "macros"] }
+tracing = "0.1.40"