nondisposable 0.4.0 → 0.6.0

data/lib/nondisposable/suggestion.rb

@@ -22,34 +22,85 @@ module Nondisposable
   # mistake. Under OSA it scores 1, alongside `gmai.com`, `gmail.co` and
   # `gmail.con`.
   #
+  # WHAT ONE EDIT CANNOT TELL APART, AND THE TWO FILTERS THAT FOLLOW
+  #
+  # Replaying "one edit from a provider" against the Tranco top 1M (2026-10-17,
+  # docs/evidence/) flagged 707 domains, and a lot of them are large real
+  # organisations: ge.com, mi.com and se.com are one edit from me.com; ea.com and
+  # aa.com from ya.com; bol.com and uol.com from aol.com; grail.com from
+  # gmail.com. Distance alone cannot see the difference, so two filters decide
+  # what is worth suggesting at all:
+  #
+  #   1. SHORT NAMES ONLY TAKE EDITS AFTER THE NAME. When a provider's name is
+  #      four characters or fewer (me, ya, qq, aol, msn, mail, live, zoho…),
+  #      one edit inside the name lands on somebody else's real domain far more
+  #      often than on a typo. `me.cmo` and `live.cm` are still suggested;
+  #      `ge.com`, `mil.ru` and `love.com` are not.
+  #   2. EVERY EDIT MUST LOOK LIKE A FINGER. A swap (`gmial`), a dropped letter
+  #      (`gmal`, `gmail.co`), a doubled letter (`gmmail`), an extra vowel
+  #      (`gamail`) or a neighbouring key on QWERTY (`gnail`, `gmail.con`) is
+  #      how people mistype. A letter from across the keyboard (`grail`,
+  #      `pmail`, `hive`) is how people spell a different word, and the same
+  #      holds after the name: `.au` is not a slip for `.ar`, nor `.dev` for
+  #      `.de`. Dropping the FIRST letter is not counted either — `cloud.com`
+  #      and `aver.com` are what that spells.
+  #
+  # Measured on the 70% tuning split of test/fixtures/lookalike_corpus.csv
+  # (precision over typo+real rows): distance alone 0.07, filter 1 alone 0.27,
+  # both 0.55, at 79% recall of reviewed typos. The corpus is the Tranco top 1M,
+  # so it is crowded with real popular domains — a signup form sees far more
+  # typos per lookalike than this does. The other 30% is a RETROSPECTIVE
+  # holdout, measured by test/nondisposable/lookalike_corpus_test.rb: the hash
+  # split was chosen before tuning but only committed with the tuned filters,
+  # so it is not a pre-registered holdout, and one early draft was seen
+  # against it.
+  #
+  # These filters only decide what to SUGGEST. They are a likelihood, not proof,
+  # and nothing in this file ever justifies refusing an address on its own —
+  # see Nondisposable::KnownTypos for the only domains the gem refuses outright,
+  # and ActiveModel::Validations::NondisposableValidator for how a suggestion
+  # becomes a confirmable error rather than a wall.
+  #
   # WHY THE DEFAULT THRESHOLD IS 1, AND WHY THIS IS OFF BY DEFAULT
   #
-  # Every suggestion is a guess about intent, and a wrong guess wired to
-  # `reject_lookalike_domains` stops a real person from signing up with their
-  # real address. One edit is the distance at which a guess is safe enough to
-  # act on; at two, legitimately distinct domains start colliding. The exact
-  # match check runs first and always wins, which is why the provider list has
-  # to be generous — see the header of data/email_providers.txt.
+  # One edit is the distance at which a guess is worth showing; at two,
+  # legitimately distinct domains start colliding. The exact match check runs
+  # first and always wins, which is why the provider list has to be generous —
+  # see the header of data/email_providers.txt.
   #
   # Suggesting is always available and never blocks:
   #
   #   Nondisposable.suggestion_for("someone@gmial.com") # => "someone@gmail.com"
   #   Nondisposable.suggestion_for("someone@gmail.com") # => nil
-  #
-  # Blocking on it is a separate, deliberate opt-in
-  # (`config.reject_lookalike_domains`), because "probably a typo" is a weaker
-  # claim than "this TLD does not exist" and deserves a weaker remedy.
   module Suggestion
     LIST_PATH = File.expand_path('../../data/email_providers.txt', __dir__)
 
+    # A provider name this short only takes edits after the name. See filter 1
+    # in the header for the evidence.
+    SHORT_NAME_LENGTH = 4
+
+    VOWELS = 'aeiou'
+
+    # Physical neighbours on a QWERTY keyboard, the layout of Spain, Latin
+    # America and the English-speaking world (and the default on phones there).
+    # `ñ` sits right of `l` on the Spanish layout. AZERTY and QWERTZ are left out
+    # on purpose: unioning them makes every key a neighbour of more keys, and
+    # QWERTZ's z/y swap alone would start suggesting zoho.com to zozo.com.
+    KEYBOARD_ROWS = %w[1234567890 qwertyuiop asdfghjklñ zxcvbnm].freeze
+
     class << self
       # The full corrected address, or nil when we have nothing useful to say.
       def for(email)
-        email = email.to_s.strip
+        email = email.to_s
+        # Bytes that are not UTF-8 have no keystrokes to correct.
+        email = email.dup.force_encoding(Encoding::UTF_8) unless email.encoding == Encoding::UTF_8
+        return nil unless email.valid_encoding?
+
+        email = email.strip
         local, _, domain = email.rpartition('@')
         return nil if local.empty? || domain.empty?
 
-        corrected = correct_domain(domain.downcase)
+        corrected = correct_domain(domain)
         return nil if corrected.nil?
 
         "#{local}@#{corrected}"
@@ -58,10 +109,32 @@ module Nondisposable
       # The corrected DOMAIN alone, or nil. Split out so a host can offer
       # "did you mean …?" next to a domain field, not just an email field.
       def correct_domain(domain)
-        domain = domain.to_s.strip.downcase.delete_suffix('.')
-        return nil if domain.empty?
+        typed = Domain.unicode(domain)
+        return nil if typed.empty?
         # A real provider is never a typo of another real provider.
-        return nil if providers.include?(domain)
+        return nil if providers.include?(Domain.normalize(typed))
+        # 0 switches the whole layer off, reviewed list included.
+        return nil if Nondisposable.configuration.lookalike_distance.to_i < 1
+
+        # A reviewed typo carries its own correction, and it wins over any
+        # guess: the list exists precisely because we checked it.
+        KnownTypos.correction_for(typed) || guess(typed)
+      end
+
+      # The ranker alone: distance plus the two filters, without the reviewed
+      # list. Public so the corpus test can measure the guess on its own —
+      # measuring it with the list in front would mostly measure the list.
+      def guess(domain)
+        typed = Domain.unicode(domain)
+        return nil if typed.empty?
+        return nil if providers.include?(Domain.normalize(typed))
+
+        # Punycode letters are an encoding, not keystrokes: `xn--gmal-xpa.com`
+        # is somebody typing `gmaíl.com`, and its distance to gmail.com in
+        # ASCII is meaningless. We judge the unicode form when we get it and
+        # stay silent on the encoded one.
+        return nil if typed.encoding == Encoding::BINARY # not UTF-8: nothing typed to measure
+        return nil if typed.split('.').any? { |label| label.start_with?('xn--') }
 
         threshold = Nondisposable.configuration.lookalike_distance.to_i
         return nil if threshold < 1
@@ -69,41 +142,44 @@ module Nondisposable
         best = nil
         best_distance = threshold + 1
 
-        providers.each do |candidate|
+        targets.each do |candidate|
           # Cheap rejection before the O(n*m) walk: an edit changes the length
           # by at most 1 per operation, so anything further apart than the
           # threshold in length alone cannot be within it.
-          next if (candidate.length - domain.length).abs > threshold
+          next if (candidate.length - typed.length).abs > threshold
 
-          distance = osa_distance(domain, candidate, best_distance)
+          distance = osa_distance(typed, candidate, best_distance)
           next if distance >= best_distance
+          next unless plausible?(typed, candidate, distance)
 
           best = candidate
           best_distance = distance
-          break if distance == 1 # nothing can beat one edit; stop early
+          break if distance == 1 # nothing plausible can beat one edit; stop early
         end
 
         best
       end
 
-      # Every domain we would suggest, as a frozen Set: the bundled list plus
-      # anything the host added. Memoised per configuration object so a host
-      # changing config in a test or console is picked up.
+      # Every provider, as a frozen Set of canonical (punycode) names: the
+      # bundled list plus anything the host added. This is the EXEMPTION set —
+      # an exact match here is never called a typo. Memoised per configuration
+      # so a host changing config in a test or console is picked up.
       def providers
-        config = Nondisposable.configuration
-        extra = Array(config.additional_email_providers).map { |d| d.to_s.strip.downcase }
-        cache_key = extra.hash
-
-        if @cache_key != cache_key || @providers.nil?
-          @providers = (bundled_providers + extra).to_set.freeze
-          @cache_key = cache_key
-        end
-
+        refresh_cache
         @providers
       end
 
+      # The same providers as a human would type them (unicode, not punycode):
+      # what a keystroke is measured against. `bücher.d` is one edit from
+      # `bücher.de` and eleven from `xn--bcher-kva.de`.
+      def targets
+        refresh_cache
+        @targets
+      end
+
       def reload!
         @providers = nil
+        @targets = nil
         @bundled_providers = nil
         @cache_key = nil
         self
@@ -111,6 +187,16 @@ module Nondisposable
 
       private
 
+      def refresh_cache
+        extra = Array(Nondisposable.configuration.additional_email_providers).map { |d| Domain.unicode(d) }.reject(&:empty?)
+        cache_key = extra.hash
+        return if @cache_key == cache_key && @providers && @targets
+
+        @targets = (bundled_providers + extra).uniq.freeze
+        @providers = @targets.map { |d| Domain.normalize(d) }.to_set.freeze
+        @cache_key = cache_key
+      end
+
       def bundled_providers
         @bundled_providers ||= File.readlines(LIST_PATH, chomp: true).filter_map do |line|
           value = line.strip.downcase
@@ -118,6 +204,90 @@ module Nondisposable
         end
       end
 
+      # Does the difference between what was typed and this provider look like
+      # a slip of the finger? The two filters from the header, in code.
+      def plausible?(typed, candidate, distance)
+        name = candidate.split('.').first
+        # The dot itself missed or swapped for a letter (`gmailmcom`,
+        # `gmailcom`): nobody owns a domain shaped like that, and the TLD path
+        # refuses it anyway, so it is always worth naming the correction.
+        return true if typed.count('.') < candidate.count('.') && typed.start_with?(name)
+
+        inside_name = typed.split('.').first != name
+        return false if inside_name && name.length <= SHORT_NAME_LENGTH
+        # The edit model below describes ONE edit. A host that widened
+        # lookalike_distance has opted into coarser guesses; filter 1 still
+        # applies, filter 2 has nothing to say.
+        return true if distance > 1
+
+        finger_slip?(typed, candidate)
+      end
+
+      # One edit, classified — in the name or after it, the same fingers type
+      # both (`gmail.cpm` is as plausible as `gmsil.com`, `yahoo.com.au` as
+      # implausible as `grail.com`). Callers have already established the
+      # distance is exactly 1, so exactly one of these shapes applies.
+      def finger_slip?(typed, candidate)
+        i = 0
+        i += 1 while i < typed.length && i < candidate.length && typed[i] == candidate[i]
+
+        case typed.length - candidate.length
+        when -1
+          # A dropped letter (`gmal`, `hotmal`, `gmail.co`) — but not the
+          # first one: `cloud.com`, `aver.com` and `mail.ch` are what
+          # dropping the first letter of icloud, naver and gmail.ch spells,
+          # and every one of them is somebody's real domain.
+          i.positive?
+        when 1
+          # An extra letter is a slip when it repeats, or sits next to, a
+          # letter it was typed beside (`gmmail`, `gmaiul` — not `mlive`), or
+          # when it is a vowel: spelling by ear adds vowels (`gamail`,
+          # `gimail`) far more often than consonants.
+          extra = typed[i]
+          return true if VOWELS.include?(extra)
+
+          beside = [i.positive? ? typed[i - 1] : nil, typed[i + 1]].compact
+          beside.any? { |near| near == extra || neighbours?(near, extra) }
+        else
+          # A swap (`gmial`) is the one mistake OSA counts as one edit…
+          return true if typed[i + 1] == candidate[i] && typed[i] == candidate[i + 1]
+
+          # …otherwise it is a substitution, which has to be a neighbouring
+          # key — or the same key with an accent (`gmaíl`: on a Spanish
+          # keyboard í is i after a dead key, not a different letter).
+          same_key?(typed[i], candidate[i]) || neighbours?(typed[i], candidate[i])
+        end
+      end
+
+      def same_key?(a, b)
+        a.unicode_normalize(:nfd)[0] == b.unicode_normalize(:nfd)[0]
+      end
+
+      def neighbours?(a, b)
+        keyboard.fetch(a, nil)&.include?(b) || false
+      end
+
+      # key => the keys around it, on a staggered QWERTY: the row's own left and
+      # right, and the two keys above and below that it physically touches.
+      def keyboard
+        @keyboard ||= begin
+          map = Hash.new { |h, k| h[k] = Set.new }
+          KEYBOARD_ROWS.each_with_index do |row, r|
+            row.each_char.with_index do |key, c|
+              touching = [[r, c - 1], [r, c + 1], [r - 1, c], [r - 1, c + 1], [r + 1, c - 1], [r + 1, c]]
+              touching.each do |rr, cc|
+                next if rr.negative? || cc.negative? || rr >= KEYBOARD_ROWS.size
+
+                other = KEYBOARD_ROWS[rr][cc]
+                map[key] << other if other
+              end
+            end
+          end
+          map.default_proc = nil
+          map.freeze
+        end
+      end
+
       # Optimal string alignment distance, with an early bail-out.
       #
       # Two rows instead of a full matrix (we only ever need the previous two),

data/lib/nondisposable/tld.rb

@@ -156,10 +156,15 @@ module Nondisposable
 
       private
 
-      # The domain half, trimmed, downcased, and without the trailing dot a
-      # fully-qualified name may carry (`gmail.com.` is the same host).
+      # The domain half in Nondisposable::Domain's canonical form, minus the
+      # punycode step: trimmed, downcased, no trailing root dot, fullwidth and
+      # ideographic dots read as dots (`foo．tk` ends in `tk`, exactly like
+      # `foo.tk`). Unicode TLDs stay unicode, so #valid? still declines to
+      # judge them.
       def normalize(email_or_domain)
-        email_or_domain.to_s.split('@').last.to_s.strip.downcase.delete_suffix('.')
+        value = email_or_domain.to_s
+        value = value.b unless value.valid_encoding? # unreadable bytes must not raise
+        Nondisposable::Domain.unicode(value.split('@').last.to_s)
       end
 
       # Skips IANA's comment header and any blank lines.

data/lib/nondisposable/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Nondisposable
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

data/lib/nondisposable.rb

@@ -2,11 +2,20 @@
 
 require_relative "nondisposable/version"
 require_relative "nondisposable/engine"
+require_relative "nondisposable/domain"
+require_relative "nondisposable/public_suffix"
+require_relative "nondisposable/precedence"
+require_relative "nondisposable/source"
+require_relative "nondisposable/source_merge"
+require_relative "nondisposable/source_fetcher"
 require_relative "nondisposable/tld"
+require_relative "nondisposable/known_typos"
 require_relative "nondisposable/suggestion"
+require_relative "nondisposable/confirmable_email"
 require_relative "nondisposable/email_validator"
 require_relative "nondisposable/domain_list_updater"
 require_relative "nondisposable/tld_list_updater"
+require_relative "nondisposable/public_suffix_list_updater"
 
 module Nondisposable
   class Error < StandardError; end
@@ -27,10 +36,12 @@ module Nondisposable
   end
 
   def self.disposable?(email)
-    return false if email.nil? || !email.include?('@')
-    domain = email.to_s.split('@').last
-    return false if domain.nil? || domain.empty?
-    DisposableDomain.disposable?(domain.downcase)
+    return false if email.nil? || !email.to_s.include?('@')
+    # Unicode form: DisposableDomain canonicalises it itself, and needs the
+    # unencoded spelling to match rows stored before 0.5.0.
+    domain = Domain.from_email(email, unicode: true)
+    return false if domain.empty?
+    DisposableDomain.disposable?(domain)
   end
 
   # Does this address end in a TLD that actually exists?
@@ -58,14 +69,35 @@ module Nondisposable
 
   class Configuration
     ON_CHECK_FAILURE_MODES = [:allow, :reject].freeze
+    LOOKALIKE_MODES = [false, true, :confirmable].freeze
 
     attr_accessor :error_message, :additional_domains, :excluded_domains
     # Whether to also match parent domains: with the default true, an email at
-    # x.tempmail.com is blocked when tempmail.com is on the list (checks up to
-    # DisposableDomain::PARENT_MATCH_DEPTH parent labels, never a bare TLD).
+    # x.tempmail.com is blocked when tempmail.com is on the list (checks
+    # every parent down to the registrable domain, never a public suffix).
     attr_accessor :check_parent_domains
     attr_reader :on_check_failure
 
+    # ---- Blocklist sources (Nondisposable::Source) --------------------------
+
+    # Where the list of disposable domains comes from: an ordered list of
+    # Nondisposable::Source objects (or Hashes of their attributes). The
+    # default is the single list this gem has always used, so upgrading changes
+    # nothing. Nondisposable::Sources::RECOMMENDED adds a second, independent
+    # primary. Needs the nondisposable_source_snapshots table
+    # (`rails g nondisposable:upgrade`); without it only the default is used.
+    attr_accessor :sources
+
+    # How many independent ORIGINS must list a domain before a :corroborating
+    # source's entry is blocked. Copies don't count: see Nondisposable::Source.
+    attr_accessor :corroboration_threshold
+
+    # Privacy relays (Apple Hide My Email, DuckDuckGo, Firefox Relay,
+    # SimpleLogin, addy.io — data/privacy_relays.txt) forward to a real,
+    # long-lived inbox. false (the default) ALLOWS them even when an upstream
+    # list carries them; true REFUSES them even when none does.
+    attr_accessor :block_privacy_relays
+
     # ---- TLD validity (Nondisposable::Tld) ----------------------------------
 
     # OFF by default, and it stays off: upgrading a gem must never silently
@@ -93,15 +125,21 @@ module Nondisposable
 
     # ---- Lookalike domains (Nondisposable::Suggestion) ----------------------
 
-    # Reject addresses one edit away from a well-known provider — `gmail.co`,
+    # Flag addresses one edit away from a well-known provider — `gmail.co`,
     # `gmial.com`, `hotmial.com` — with a message naming the correction.
     #
-    # OFF by default and worth leaving off unless you have thought about it: a
-    # suggestion is a guess about intent, and a wrong guess here stops a real
-    # person signing up with their real address. `Nondisposable.suggestion_for`
-    # is always available and blocks nothing, which is the gentler way to use
-    # this: show the hint, let the human decide.
-    attr_accessor :reject_lookalike_domains
+    #   false        (default) never flag; Nondisposable.suggestion_for still works
+    #   :confirmable flag them. Reviewed typos (data/known_typo_domains.txt) are
+    #                refused outright; every other lookalike is a guess, and the
+    #                user can confirm the address as typed and pass — see
+    #                Nondisposable::ConfirmableEmail.
+    #   true         the same as :confirmable. Before 0.5.0 it refused every
+    #                lookalike, including real organisations' domains (ge.com,
+    #                bol.com, one.com…) one edit from a provider.
+    #
+    # Worth leaving off unless your form can show the confirmation: without it a
+    # user at a domain that merely looks like a typo has no way through.
+    attr_reader :reject_lookalike_domains
 
     # How many edits still count as "a typo". 1 (the default) is the distance at
     # which acting on a guess is safe; at 2, genuinely different domains start
@@ -126,6 +164,10 @@ module Nondisposable
       @on_check_failure = :allow
       @check_parent_domains = true
 
+      @sources = Nondisposable::Sources::DEFAULT
+      @corroboration_threshold = 2
+      @block_privacy_relays = false
+
       @check_tld = false
       @additional_tlds = []
       @blocked_tlds = []
@@ -152,5 +194,18 @@ module Nondisposable
 
       @on_check_failure = mode
     end
+
+    # Strings are accepted for config loaded from YAML/ENV ("confirmable",
+    # "true"); anything else raises, so a typo in an initializer fails at boot
+    # instead of silently meaning "on".
+    def reject_lookalike_domains=(mode)
+      mode = false if mode.nil?
+      mode = { 'true' => true, 'false' => false }.fetch(mode, mode.to_sym) if mode.is_a?(String)
+      unless LOOKALIKE_MODES.include?(mode)
+        raise ArgumentError, "reject_lookalike_domains must be one of #{LOOKALIKE_MODES.map(&:inspect).join(', ')} (got #{mode.inspect})"
+      end
+
+      @reject_lookalike_domains = mode
+    end
   end
 end

data/lib/tasks/nondisposable.rake

@@ -35,3 +35,32 @@ namespace :nondisposable do
     end
   end
 end
+
+namespace :nondisposable do
+  namespace :psl do
+    desc "Refresh the vendored Public Suffix List snapshot (maintenance task — run in a gem checkout before a release)"
+    task :update do
+      # Same reasoning as tlds:update: no Rails here. The parser normalizes
+      # rules through Domain, so it comes along.
+      require_relative "../nondisposable/domain"
+      require_relative "../nondisposable/public_suffix"
+      require_relative "../nondisposable/public_suffix_list_updater"
+
+      before = begin
+        Nondisposable::PublicSuffix.rule_count
+      rescue StandardError
+        0
+      end
+
+      count = Nondisposable::PublicSuffixListUpdater.update
+
+      if count.nil?
+        abort "[nondisposable] Public Suffix List unchanged. See the error above."
+      else
+        puts "[nondisposable] PSL snapshot now holds #{count} rules (was #{before})."
+        puts "[nondisposable] PSL version: #{Nondisposable::PublicSuffix.version}"
+        puts "[nondisposable] Commit data/public_suffix_list.dat if it changed."
+      end
+    end
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: nondisposable
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - rameerez
 bindir: exe
 cert_chain: []
-date: 2026-08-27 00:00:00.000000000 Z
+date: 2026-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -41,21 +41,37 @@ files:
 - README.md
 - Rakefile
 - app/models/nondisposable/disposable_domain.rb
+- app/models/nondisposable/source_snapshot.rb
 - context7.json
 - data/disposable_email_blocklist.conf
 - data/email_providers.txt
 - data/iana_tlds.txt
+- data/known_typo_domains.txt
+- data/never_disposable.txt
+- data/privacy_relays.txt
+- data/public_suffix_list.dat
 - gemfiles/rails_7.2.gemfile
 - gemfiles/rails_8.0.gemfile
 - gemfiles/rails_8.1.gemfile
 - lib/generators/nondisposable/install_generator.rb
 - lib/generators/nondisposable/templates/create_nondisposable_disposable_domains.rb.erb
+- lib/generators/nondisposable/templates/create_nondisposable_source_snapshots.rb.erb
 - lib/generators/nondisposable/templates/disposable_email_domain_list_update_job.rb
 - lib/generators/nondisposable/templates/nondisposable.rb
+- lib/generators/nondisposable/upgrade_generator.rb
 - lib/nondisposable.rb
+- lib/nondisposable/confirmable_email.rb
+- lib/nondisposable/domain.rb
 - lib/nondisposable/domain_list_updater.rb
 - lib/nondisposable/email_validator.rb
 - lib/nondisposable/engine.rb
+- lib/nondisposable/known_typos.rb
+- lib/nondisposable/precedence.rb
+- lib/nondisposable/public_suffix.rb
+- lib/nondisposable/public_suffix_list_updater.rb
+- lib/nondisposable/source.rb
+- lib/nondisposable/source_fetcher.rb
+- lib/nondisposable/source_merge.rb
 - lib/nondisposable/suggestion.rb
 - lib/nondisposable/tld.rb
 - lib/nondisposable/tld_list_updater.rb