gman 8.0.0 → 8.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05dbe9c50c24586c2c071eccee35d04c016224fc909aea2699e09701bf0ac56b
-  data.tar.gz: f6aab93cdcec8b9c6330dfaead40263272a2cc48a075ec6549d2378c6f13c12b
+  metadata.gz: 29b30e2a1e50e8e0f8f7d5ca50a6c2a8671b1e69a1d8213415bd6d6a70a35af9
+  data.tar.gz: 07b8f53008e8699d2f67ef162460540196cd90c3e89e1ed3a4054a185b0a325f
 SHA512:
-  metadata.gz: ce3099efeaef338677d10bcfa222c7be2d10405682c9bcd0dbc33d887e6e0050eecf31bbdac2a5b703431e1516534a18f2247cf921d42931e6c12ace08171f57
-  data.tar.gz: dfe56fb1e9a5c73710fddc489c6715cd8db0835b5725bc9814c497845d98cef52ed45b9af37464fd679fa7cc1493c7e057656e5b4b3ae8497d9734f439775277
+  metadata.gz: 25c6626204645cfbc03b7875ddb5a7eb51d18d56b5f70af1f993def0ece5841a36b27e151c2e517eee4060059627f648f1883c45216d50b68ee1f749ea4999de
+  data.tar.gz: 5ca3b6cba72c5bf9238abaa014c61f9caaeaa10bcc4a0f147b27f92fb5fa96a2fff835fbd2b479764429d6ce6e0e1bd15937b8f371c8483dbb63724ebbafb032

data/lib/gman/domain_list.rb

@@ -4,7 +4,7 @@ class Gman
   class DomainList
     COMMENT_REGEX = %r{//[/\s]*(.*)$}i
 
-    attr_writer :data, :path, :contents
+    attr_writer :path, :contents
 
     class << self
       # The current, government domain list
@@ -41,6 +41,12 @@ class Gman
                     end
     end
 
+    # Replaces the list data, a hash of group => domains
+    def data=(data)
+      @data = data
+      clear_cache
+    end
+
     # Returns the parsed contents of the domain list as a hash
     # in the form for group => domains
     def data
@@ -73,7 +79,7 @@ class Gman
 
     # Return an array of strings representing all domains on the list
     def domains
-      data.values.flatten.compact.sort.uniq
+      @domains ||= data.values.flatten.compact.sort.uniq.freeze
     end
 
     # Return the total number of domains in the list
@@ -84,11 +90,12 @@ class Gman
     # Alphabetize groups and domains within each group
     # We need to ensure exceptions appear after their coresponding rules
     def alphabetize
-      @data = data.sort_by { |k, _v| k.downcase }.to_h
-      @data.map do |_group, domains|
+      self.data = data.sort_by { |k, _v| k.downcase }.to_h
+      @data.each_value do |domains|
         domains.sort! { |a, b| sort_with_exceptions(a, b) }
         domains.uniq!
       end
+      clear_cache
     end
 
     # Write the domain list to disk
@@ -112,14 +119,60 @@ class Gman
     end
     alias to_public_suffix to_s
 
-    # Given a domain, find any domain on the list that includes that domain
-    # E.g., `fcc.gov` would be the parent of `data.fcc.gov`
+    # Returns the group a list entry belongs to, e.g., "US Federal" for "fcc.gov"
+    # The entry must match the list verbatim. If an entry appears in more than
+    # one group, the first group wins.
+    def group_for(entry)
+      group_index[entry]
+    end
+
+    # Given a domain, find the list entry that covers it, using public suffix
+    # rules. E.g., `fcc.gov` would be the parent of `data.fcc.gov`, and
+    # `*.foo.gov` the parent of `bar.foo.gov`.
+    #
+    # If more than one entry matches, the longest wins. A domain under an
+    # exception rule (e.g., `!mail.gov.ua`) has no parent. A domain that is
+    # itself on the list returns the entry that covers its parent, if any.
+    #
+    # Returns the list entry as written (e.g., `*.foo.gov`), or nil
     def parent_domain(domain)
-      domains.find { |c| domain =~ /\.#{Regexp.escape(c)}$/ }
+      rule = parent_list.find(domain, default: nil)
+      if rule && !rule.is_a?(PublicSuffix::Rule::Exception) && rule.value == domain
+        ancestor = domain.split('.', 2)[1]
+        rule = ancestor && parent_list.find(ancestor, default: nil)
+      end
+      return if rule.nil? || rule.is_a?(PublicSuffix::Rule::Exception)
+
+      rule.rule
+    end
+
+    # Returns the exception rule (e.g., `!mail.gov.ua`) that excludes the given
+    # domain from the list, or nil if none does
+    def exception_for(domain)
+      rule = parent_list.find(domain, default: nil)
+      rule.rule if rule.is_a?(PublicSuffix::Rule::Exception)
     end
 
     private
 
+    # Clear values derived from data. Call after mutating data in place.
+    def clear_cache
+      @domains = @parent_list = @group_index = nil
+    end
+
+    # PublicSuffix::List of the current data, for parent lookups. Unlike
+    # #public_suffix_list, it is rebuilt when the data changes.
+    def parent_list
+      @parent_list ||= PublicSuffix::List.parse(domains.join("\n"))
+    end
+
+    # Hash of list entry => group, for constant-time group lookups
+    def group_index
+      @group_index ||= data.each_with_object({}) do |(group, domains), index|
+        domains.each { |domain| index[domain] ||= group }
+      end
+    end
+
     # Parse a public-suffix formatted string into a hash of groups => [domains]
     def string_to_hash(string)
       return unless string

data/lib/gman/identifier.rb

@@ -110,12 +110,8 @@ class Gman
   def list_category
     return @list_category if defined?(@list_category)
 
-    match = Gman.list.public_suffix_list.find(domain.to_s)
-    return @list_category = nil unless match
-
-    regex = %r{// ([^\n]+)\n?[^/]*\n#{Regexp.escape(match.value)}\n}im
-    matches = Gman.list.contents.match(regex)
-    @list_category = matches ? matches[1] : nil
+    match = Gman.list.public_suffix_list.find(domain.to_s, default: nil)
+    @list_category = match && Gman.list.group_for(match.value)
   end
 
   def matches

data/lib/gman/importer.rb

@@ -6,6 +6,7 @@ require 'yaml'
 require 'open-uri'
 require 'resolv'
 require 'logger'
+require 'addressable/uri'
 require 'swot'
 require_relative '../gman'
 require_relative 'domain_list'
@@ -71,7 +72,7 @@ class Gman
 
     def normalize_domain(domain)
       domain = Gman.new(domain).to_s
-      domain.to_s.downcase.strip.gsub(/^www./, '').gsub(%r{/$}, '')
+      domain.to_s.downcase.strip.delete_prefix('www.').delete_suffix('/')
     end
 
     def valid_domain?(domain, options = {})
@@ -152,34 +153,33 @@ class Gman
 
       if current.domains.include?(domain)
         reject(domain, 'duplicate')
-      else
-        parent = current.parent_domain(domain)
+      elsif (parent = current.parent_domain(domain))
         reject(domain, "subdomain of #{parent}")
+      else
+        reject(domain, "excluded by #{current.exception_for(domain)}")
       end
     end
 
+    # A domain is a dupe if it's on the list, covered by a list entry, or
+    # excluded by an exception rule
     def dupe?(domain)
-      current.domains.include?(domain) || current.parent_domain(domain)
+      current.domains.include?(domain) || current.parent_domain(domain) || current.exception_for(domain)
     end
 
     def normalize_domains!
-      domain_list.to_h.each_value do |domains|
-        domains.map! { |domain| normalize_domain(domain) }
-        domains.uniq!
+      domain_list.data = domain_list.data.transform_values do |domains|
+        domains.map { |domain| normalize_domain(domain) }.uniq
       end
     end
 
     def ensure_validity!(options = {})
-      domain_list.data.each_value do |domains|
-        domains.select! { |domain| valid_domain?(domain, options) }
+      domain_list.data = domain_list.data.transform_values do |domains|
+        domains.select { |domain| valid_domain?(domain, options) }
       end
     end
 
     def add_to_current
-      domain_list.data.each do |group, domains|
-        current.data[group] ||= []
-        current.data[group].concat domains
-      end
+      current.data = current.data.merge(domain_list.data) { |_group, existing, added| existing + added }
       current.write
     end
 

data/lib/gman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Gman
-  VERSION = '8.0.0'
+  VERSION = '8.1.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gman
 version: !ruby/object:Gem::Version
-  version: 8.0.0
+  version: 8.1.0
 platform: ruby
 authors:
 - Ben Balter