nondisposable 0.4.0 → 0.6.0

data/gemfiles/rails_7.2.gemfile

@@ -13,6 +13,7 @@ group :test do
   gem "webmock", "~> 3.19"
   gem "sqlite3", "~> 2.1"
   gem "simplecov", require: false
+  gem "csv"
 end
 
 gemspec path: "../"

data/gemfiles/rails_8.0.gemfile

@@ -13,6 +13,7 @@ group :test do
   gem "webmock", "~> 3.19"
   gem "sqlite3", "~> 2.1"
   gem "simplecov", require: false
+  gem "csv"
 end
 
 gemspec path: "../"

data/gemfiles/rails_8.1.gemfile

@@ -13,6 +13,7 @@ group :test do
   gem "webmock", "~> 3.19"
   gem "sqlite3", "~> 2.1"
   gem "simplecov", require: false
+  gem "csv"
 end
 
 gemspec path: "../"

data/lib/generators/nondisposable/install_generator.rb

@@ -16,6 +16,7 @@ module Nondisposable
 
       def create_migration_file
         migration_template 'create_nondisposable_disposable_domains.rb.erb', File.join(db_migrate_path, "create_nondisposable_disposable_domains.rb")
+        migration_template 'create_nondisposable_source_snapshots.rb.erb', File.join(db_migrate_path, "create_nondisposable_source_snapshots.rb")
       end
 
       def create_initializer

data/lib/generators/nondisposable/templates/create_nondisposable_source_snapshots.rb.erb

@@ -0,0 +1,42 @@
+class CreateNondisposableSourceSnapshots < ActiveRecord::Migration<%= migration_version %>
+  # One row per fetch of one blocklist source, so the merged
+  # nondisposable_disposable_domains table can always be rebuilt from each
+  # source's last good list (see Nondisposable::SourceSnapshot).
+  #
+  # Additive and safe to roll back: until this table exists, and again after
+  # dropping it, Nondisposable::DomainListUpdater uses the single default list
+  # exactly as before.
+  def change
+    primary_key_type = primary_key_type_setting
+
+    create_table :nondisposable_source_snapshots, id: primary_key_type do |t|
+      t.string :source, null: false
+      t.string :url
+      t.string :status, null: false
+      t.datetime :fetched_at, null: false
+      t.string :sha256
+      t.integer :domain_count
+      # Largest domain_count ever accepted for this source (the shrink guard's
+      # baseline), carried forward on each good snapshot so pruning never
+      # forgets it.
+      t.integer :high_water_count
+      # The largest list we know of is ~1.4 MB. The limit only matters on
+      # MySQL, where plain TEXT stops at 64 KB (this becomes LONGTEXT);
+      # PostgreSQL and SQLite ignore it.
+      t.text :domains, limit: 16.megabytes
+      t.text :error
+
+      t.timestamps
+    end
+
+    add_index :nondisposable_source_snapshots, [:source, :status, :fetched_at],
+              name: "index_nondisposable_source_snapshots_lookup"
+  end
+
+  private
+
+  def primary_key_type_setting
+    config = Rails.configuration.generators
+    config.options[config.orm][:primary_key_type] || :primary_key
+  end
+end

data/lib/generators/nondisposable/templates/nondisposable.rb

@@ -14,9 +14,29 @@ Nondisposable.configure do |config|
   # config.on_check_failure = :allow
   #
   # Also match parent domains: an email at x.tempmail.com is blocked when
-  # tempmail.com is on the list (checks up to 3 parent labels). Default: true
+  # tempmail.com is on the list (checks every parent down to the registrable domain). Default: true
   # config.check_parent_domains = true
 
+  # ---- Where the blocklist comes from ----
+  #
+  # Default: the single disposable-email-domains list this gem has always used.
+  # RECOMMENDED adds fakefilter, a second list that watches temp-mail sites
+  # directly (+3,275 domains on 2026-10-17, no known real provider among them).
+  # config.sources = Nondisposable::Sources::RECOMMENDED
+  #
+  # :corroborating sources only block a domain once this many INDEPENDENT
+  # lists carry it (a list that copies another doesn't count twice).
+  # config.corroboration_threshold = 2
+  #
+  # Privacy relays (Hide My Email, DuckDuckGo, Firefox Relay, SimpleLogin,
+  # addy.io) forward to a real inbox. Allowed by default, even when an upstream
+  # list carries them; set to true to refuse them.
+  # config.block_privacy_relays = false
+  #
+  # Who wins when lists disagree (top wins): excluded_domains, the gem's
+  # never-disposable list of real providers, additional_domains, the
+  # privacy-relay setting, the upstream sources.
+
   # ---- Catching typos ----
   #
   # A disposable address is someone hiding from you. A typo is someone who
@@ -39,15 +59,17 @@ Nondisposable.configure do |config|
   # customer you have.
   # config.allowed_tlds = nil
 
-  # Reject addresses one keystroke from a well-known provider (gmail.co,
+  # Flag addresses one keystroke from a well-known provider (gmail.co,
   # gmial.com). A TLD check CANNOT catch these — .co, .cm and .om are Colombia,
   # Cameroon and Oman, all real.
   #
-  # Think before switching this on: a suggestion is a guess about intent, and a
-  # wrong guess stops a real person signing up with their real address.
-  # Nondisposable.suggestion_for(email) is always available and blocks nothing,
-  # which is the gentler way to use this — show the hint, let the human decide.
-  # config.reject_lookalike_domains = true
+  # Reviewed typos (data/known_typo_domains.txt) are refused outright. Every
+  # other lookalike is a guess — ge.com is one edit from me.com and is General
+  # Electric — so it is a CONFIRMABLE error: the user ticks "yes, this is my
+  # address" and passes. Wire that checkbox first (see
+  # Nondisposable::ConfirmableEmail in the README), or a real user at a
+  # lookalike domain has no way through. `true` means the same as :confirmable.
+  # config.reject_lookalike_domains = :confirmable
   #
   # How many edits still count as a typo. 1 is safe; 2 starts colliding with
   # genuinely different domains. 0 disables suggestions entirely.

data/lib/generators/nondisposable/upgrade_generator.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'rails/generators/base'
+require 'rails/generators/active_record'
+
+module Nondisposable
+  module Generators
+    # For apps installed before 0.6.0: adds the source snapshots table that
+    # multi-source blocklists need. Purely additive — it touches no existing
+    # table, and the app keeps working the old way until it is migrated.
+    #
+    #   rails generate nondisposable:upgrade
+    #   rails db:migrate
+    class UpgradeGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path('templates', __dir__)
+
+      def self.next_migration_number(dir)
+        ActiveRecord::Generators::Base.next_migration_number(dir)
+      end
+
+      def create_source_snapshots_migration
+        migration_template 'create_nondisposable_source_snapshots.rb.erb',
+                           File.join(db_migrate_path, 'create_nondisposable_source_snapshots.rb')
+      end
+
+      def display_post_upgrade_message
+        say "\tAdded the nondisposable_source_snapshots migration.", :green
+        say "\nNext:"
+        say "  1. Run 'rails db:migrate'. Until you do, the gem keeps using the single default list, so deploying first is safe."
+        say "  2. Optionally choose your sources in config/initializers/nondisposable.rb, e.g. config.sources = Nondisposable::Sources::RECOMMENDED"
+        say "  3. Run 'Nondisposable::DomainListUpdater.update' (or wait for your scheduled job)."
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::STRING.to_f}]"
+      end
+    end
+  end
+end

data/lib/nondisposable/confirmable_email.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'active_support/concern'
+require 'active_model/type'
+
+module Nondisposable
+  # The way out of a lookalike error: "yes, this address is right".
+  #
+  # With `reject_lookalike_domains` on, `someone@ge.com` gets "looks like a
+  # typo. Did you mean someone@me.com?" — and General Electric's staff are not
+  # mistyping anything. Offline the gem cannot tell them from `gmial.com`, so
+  # instead of refusing, it asks. This concern gives the record somewhere to
+  # put the answer:
+  #
+  #   class User < ApplicationRecord
+  #     include Nondisposable::ConfirmableEmail
+  #     confirmable_email :email          # defines email_lookalike_confirmed
+  #
+  #     validates :email, nondisposable: true
+  #   end
+  #
+  #   # In the form, only when that error is showing:
+  #   <% if @user.errors.of_kind?(:email, :lookalike_domain) %>
+  #     <%= f.check_box :email_lookalike_confirmed %>
+  #     <%= f.label :email_lookalike_confirmed, "Sí, mi email es correcto" %>
+  #   <% end %>
+  #
+  #   # And permit it: params.require(:user).permit(:email, :email_lookalike_confirmed)
+  #
+  # A confirmed record passes the lookalike check and nothing else: a
+  # disposable provider, a TLD that does not exist, a blocked TLD and a
+  # reviewed typo (Nondisposable::KnownTypos) are still refused, because for
+  # those there is no address the user could be right about.
+  #
+  # The attribute is virtual — never persisted — and is reset after every
+  # save, so it answers for one save: a later lookalike is asked again.
+  #
+  # No concern needed if you would rather wire it yourself: the validator
+  # looks for a method named `<attribute>_lookalike_confirmed` returning true,
+  # however the record defines it.
+  module ConfirmableEmail
+    extend ActiveSupport::Concern
+
+    SUFFIX = '_lookalike_confirmed'
+
+    class_methods do
+      def confirmable_email(*attributes)
+        attributes.each do |attribute|
+          name = "#{attribute}#{SUFFIX}"
+          ivar = "@#{name}"
+
+          define_method(name) { instance_variable_defined?(ivar) ? instance_variable_get(ivar) : false }
+          # Form params arrive as "1", "0", "true", "" — cast the way Rails
+          # casts a boolean column, so a checkbox works without ceremony.
+          define_method("#{name}=") { |value| instance_variable_set(ivar, ActiveModel::Type::Boolean.new.cast(value) || false) }
+          alias_method "#{name}?", name
+        end
+
+        # Cleared after every save, so a record that was confirmed once and
+        # then given a different lookalike address is asked again.
+        return unless respond_to?(:after_save)
+
+        names = attributes.map { |attribute| "#{attribute}#{SUFFIX}" }
+        after_save { names.each { |name| public_send("#{name}=", false) } }
+      end
+    end
+
+    # Has the user confirmed this attribute's address as typed? Strictly
+    # `true`: a truthy string a host forgot to cast must not count as a "yes".
+    def self.confirmed?(record, attribute)
+      reader = "#{attribute}#{SUFFIX}"
+      record.respond_to?(reader) && record.public_send(reader) == true
+    end
+  end
+end

data/lib/nondisposable/domain.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+module Nondisposable
+  # One answer to "is this the same domain?", shared by every path that
+  # compares a domain against a list.
+  #
+  # Before this existed each path trimmed its input its own way: the suggestion
+  # layer stripped a trailing dot and the database check did not, so
+  # `user@gmial.com.` and `user@gmial.com` could get different verdicts from the
+  # same gem. Anything that matches a domain against a list — bundled, host
+  # configured or downloaded — should go through #normalize first, so that two
+  # spellings of one host can never be judged differently.
+  #
+  # WHAT IT DOES
+  #
+  #   Nondisposable::Domain.normalize("  GMail.COM. ")     # => "gmail.com"
+  #   Nondisposable::Domain.normalize("bücher.de")         # => "xn--bcher-kva.de"
+  #   Nondisposable::Domain.normalize("XN--BCHER-KVA.DE")  # => "xn--bcher-kva.de"
+  #   Nondisposable::Domain.from_email("First+tag@GMail.com") # => "gmail.com"
+  #
+  # Only the DOMAIN is ever touched. The local part is case-sensitive under
+  # RFC 5321 and `+tags` are the user's business; nothing here reads or
+  # rewrites them.
+  #
+  # WHY PUNYCODE, AND WHY NO DEPENDENCY
+  #
+  # A unicode domain and its punycode form are the same host (`bücher.de` ==
+  # `xn--bcher-kva.de`), and lists store the ASCII form, so matching has to
+  # happen in ASCII. The `simpleidn` / `addressable` gems would do it, but the
+  # encoding itself (RFC 3492) is forty lines of arithmetic and this gem ships
+  # with no runtime dependencies beyond Rails. What we deliberately do NOT
+  # implement is the full IDNA2008/UTS #46 validity table (which code points
+  # are disallowed): we map — NFKC, lowercase, the ideographic full stops, the
+  # invisible characters UTS #46 drops (nontransitionally: ZWJ/ZWNJ stay) — and
+  # encode, and leave "is this label legal" to the DNS. For matching that is
+  # the right trade: a domain we encode that the registry would have refused
+  # cannot be on anybody's list either.
+  module Domain
+    # The label separators UTS #46 treats as a full stop: ideographic,
+    # fullwidth and halfwidth ideographic. A user on a CJK keyboard types these.
+    DOTS = /[。．｡]/.freeze
+
+    # Every code point UTS #46 maps to nothing ("ignored" in
+    # IdnaMappingTable.txt, Unicode 2026-06-29): soft hyphen, combining
+    # grapheme joiner, Hangul and Khmer fillers, Mongolian variation selectors,
+    # zero-width space, word joiner and invisible operators, variation
+    # selectors 1–256, BOM, shorthand format controls, musical beam controls.
+    # Invisible in a form field, so `maili<SHY>nator.com` would otherwise be a
+    # different string from the listed `mailinator.com` — while a mail server
+    # applying IDNA would deliver it to mailinator.com.
+    #
+    # ZWNJ and ZWJ (U+200C/D) are deliberately NOT here. UTS #46 lists them as
+    # "deviation" characters, and nontransitional processing (IDNA2008, what
+    # registries and browsers use) keeps them: `نامه‌ای.com` with a ZWNJ is a
+    # different registered domain from the same letters without one.
+    IGNORED = /[­͏ᅟᅠ឴឵᠋-᠏​⁠-⁤-ㅤ︀-️ﾠ\u{1BCA0}-\u{1BCA3}\u{1D173}-\u{1D17A}\u{E0100}-\u{E01EF}]/.freeze
+
+    # DNS caps a name at 253 octets (RFC 1035 §2.3.4) and a label at 63, and
+    # the punycode encoder is quadratic in a label's distinct code points, so
+    # #normalize hands anything longer back unencoded: it cannot be a
+    # registered domain or match a stored row. The cap applies to the
+    # NORMALISED name — `e` + combining acute is two code points that NFKC
+    # composes into one `é`, so a valid 32-letter label can arrive as 64.
+    MAX_LENGTH = 253
+    MAX_LABEL_LENGTH = 63
+
+    # The bounds on what is fed to NFKC, which is superlinear on hostile input
+    # (a long run of combining marks to reorder). 1,024 code points is four
+    # per allowed character — room for any real decomposition (Hangul jamo
+    # take three) — and matches DisposableDomain::MAX_INPUT_LENGTH. A run of
+    # more than 30 combining marks is not stream-safe text (UAX #15 §13), so
+    # no registered label contains one and it is not worth normalising.
+    MAX_RAW_LENGTH = 1_024
+    UNSTREAMSAFE = /\p{M}{31,}/.freeze
+
+
+    # RFC 3492 §5: the bootstring parameters that make it punycode.
+    BASE = 36
+    TMIN = 1
+    TMAX = 26
+    SKEW = 38
+    DAMP = 700
+    INITIAL_BIAS = 72
+    INITIAL_N = 128
+    private_constant :BASE, :TMIN, :TMAX, :SKEW, :DAMP, :INITIAL_BIAS, :INITIAL_N
+
+    class << self
+      # The canonical, ASCII form of a domain: stripped, lowercased, without a
+      # trailing root dot, unicode labels punycode-encoded. "" for blank input,
+      # never nil, so callers can chain without guarding.
+      def normalize(domain)
+        value = unicode(domain)
+        return value if value.ascii_only? || value.encoding == Encoding::BINARY
+        # The punycode encoder is quadratic in the distinct code points of a
+        # label: a 4,000-character label would cost seconds. Over DNS's limits
+        # it cannot be a registered domain; hand it back unencoded.
+        return value if oversized?(value)
+
+        value.split('.', -1).map { |label| label.ascii_only? ? label : "xn--#{punycode(label)}" }.join('.')
+      end
+
+      # Longer than DNS allows once normalised — over 253 characters, a label
+      # over 63, or more than MAX_RAW_LENGTH code points before NFKC could
+      # even run? Such a name cannot be registered, so no address at it can
+      # receive mail, and no list can hold it: callers must treat it as
+      # invalid rather than as "not disposable". Judged on the punycode form,
+      # since a label that fits in unicode can still encode past 63.
+      def too_long?(domain)
+        oversized?(normalize(domain))
+      end
+
+      # The same canonicalisation WITHOUT the punycode step, for code that
+      # compares what a human typed (the suggestion layer: a keyboard slip is a
+      # distance between the characters someone pressed, and punycode letters
+      # have no keyboard meaning — `xn--gmal-zqa` is `gmaíl`). Never raises.
+      def unicode(domain)
+        value = utf8(domain.to_s)
+        # Bytes that are not UTF-8 cannot be a domain anybody typed: keep a
+        # stable, comparable binary form instead of raising inside a validator.
+        return value.strip.downcase.delete_suffix('.') if value.encoding == Encoding::BINARY
+
+        value = value.strip
+        return value.downcase.delete_suffix('.') if value.ascii_only?
+
+        # Linear passes first, so the raw bound below sees what is left once
+        # the invisible padding is gone (300 soft hyphens around
+        # mailinator.com must not buy an exemption). The DNS limits are
+        # enforced by #normalize on the result, after NFKC has composed it.
+        value = value.gsub(IGNORED, '').gsub(DOTS, '.')
+        return value.downcase.delete_suffix('.') if value.length > MAX_RAW_LENGTH || value.match?(UNSTREAMSAFE)
+
+        # NFKC, not NFC: it also folds compatibility forms, so fullwidth
+        # `ｇｍａｉｌ` is `gmail` — the mapping UTS #46 applies before lookup.
+        value.unicode_normalize(:nfkc).gsub(DOTS, '.').downcase.delete_suffix('.')
+      end
+
+      # The domain half of an address, normalised. Splits on the LAST "@", so a
+      # quoted local part containing one (`"a@b"@example.com`) still yields the
+      # right domain. "" when there is no "@" or nothing after it. An empty
+      # local part (`@tempmail.com`) still has a domain worth judging, so it
+      # is not this method's place to refuse it.
+      #
+      # `unicode: true` returns the #unicode form instead of the punycode one:
+      # what DisposableDomain needs to also match rows 0.4.0 stored unencoded.
+      def from_email(email, unicode: false)
+        email = email.to_s
+        email = email.b unless email.valid_encoding? # unreadable bytes must not raise
+        _local, at, domain = email.rpartition('@')
+        return '' if at.empty?
+
+        unicode ? unicode(domain) : normalize(domain)
+      end
+
+      private
+
+      # A String in UTF-8 when its bytes are UTF-8, whatever encoding it was
+      # labelled with: a form param or a log line handed over as ASCII-8BIT is
+      # still `bücher.de`, and 0.4.0 accepted it without raising.
+      def utf8(value)
+        return value if value.encoding == Encoding::UTF_8 && value.valid_encoding?
+
+        copy = value.dup.force_encoding(Encoding::UTF_8)
+        copy.valid_encoding? ? copy : value.b # BINARY: "not UTF-8"
+      end
+
+      def oversized?(value)
+        value.length > MAX_LENGTH || value.split('.', -1).any? { |label| label.length > MAX_LABEL_LENGTH }
+      end
+
+
+      # RFC 3492 §6.3, the encoder only: we never need to decode.
+      def punycode(label)
+        input = label.codepoints
+        output = input.select { |c| c < 0x80 }.pack('U*')
+        basic = output.length
+        handled = basic
+        output += '-' if basic.positive?
+
+        n = INITIAL_N
+        delta = 0
+        bias = INITIAL_BIAS
+
+        while handled < input.length
+          m = input.select { |c| c >= n }.min
+          delta += (m - n) * (handled + 1)
+          n = m
+
+          input.each do |c|
+            delta += 1 if c < n
+            next unless c == n
+
+            q = delta
+            k = BASE
+            loop do
+              t = threshold(k, bias)
+              break if q < t
+
+              output << digit(t + ((q - t) % (BASE - t)))
+              q = (q - t) / (BASE - t)
+              k += BASE
+            end
+            output << digit(q)
+            bias = adapt(delta, handled + 1, handled == basic)
+            delta = 0
+            handled += 1
+          end
+
+          delta += 1
+          n += 1
+        end
+
+        output
+      end
+
+      def threshold(k, bias)
+        return TMIN if k <= bias
+        return TMAX if k >= bias + TMAX
+
+        k - bias
+      end
+
+      def digit(d)
+        (d < 26 ? d + 97 : d + 22).chr
+      end
+
+      def adapt(delta, points, first)
+        delta = first ? delta / DAMP : delta / 2
+        delta += delta / points
+        k = 0
+        while delta > ((BASE - TMIN) * TMAX) / 2
+          delta /= BASE - TMIN
+          k += BASE
+        end
+        k + (((BASE - TMIN + 1) * delta) / (delta + SKEW))
+      end
+    end
+  end
+end