dns-sniper 0.0.1.pre7 → 0.0.1.pre10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fdb713f65960730a1ac99906c65aeb79b30b69514eedab948ee20eb27bf30920
-  data.tar.gz: 28d04385a9759e9bada850e6c4dde26d7f3d983ddedd9d5782d31c6001cab19d
+  metadata.gz: 04db0c0c25e714a9540154cef9dcc6352101fc4111dec94804d298f70fa609c3
+  data.tar.gz: 8d55fd5b07b905aaf1ee93261b78e9d1c9429a7bf0baafc7144d04557ffbab5f
 SHA512:
-  metadata.gz: c02d1bb299780f04041d8899aeb72b7457ec8954184c95a0bae61731c1b36763f44055b41d4e8115c42bf0ccfdb68b407db3aef5ff71a3dc0e01d8dde0bcee99
-  data.tar.gz: fd3f08b34762d39e3610ad23810736a667e83f7b77459d8508e625f67007b0fa7adcd028091c65c4f05e28f39eacc6d805313e6733c9d8592f4657b5ade50e2b
+  metadata.gz: e7a5b16bc843c1e593e7cca234fd4dfff5c6347f0af2723a863d420bf045cd1a1e0f53e181581ce273738e1e7e24ac078d1f92b6671101d0980379ce9f13f64e
+  data.tar.gz: 5e73efea0322fc1662312619dca0cac2333ea8fd2b6842f85fbe5fecf2e71621433726fdecc07c8f01054eeed13cc150b84d616bc2969a1bc76848cc577e55cc

data/Gemfile.lock

@@ -1,19 +1,31 @@
 PATH
   remote: .
   specs:
-    dns-sniper (0.0.1.pre6)
+    dns-sniper (0.0.1.pre10)
+      activesupport (>= 4.2, < 7.0.3)
       down (~> 5.1)
       hosts_file (~> 1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.7.0)
+    activesupport (7.0.2.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
-    down (5.2.2)
-      addressable (~> 2.5)
+    concurrent-ruby (1.1.10)
+    down (5.3.1)
+      addressable (~> 2.8)
     hosts_file (1.0.3)
+    i18n (1.10.0)
+      concurrent-ruby (~> 1.0)
+    minitest (5.15.0)
     public_suffix (4.0.6)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
 
 PLATFORMS
   ruby
@@ -22,4 +34,4 @@ DEPENDENCIES
   dns-sniper!
 
 BUNDLED WITH
-   2.1.2
+   2.3.7

data/README.md

@@ -51,8 +51,11 @@ hostnames.blacklist += DNSSniper::HostsImporter.new('https://pgl.yoyo.org/as/ser
 # Blocklist is accessible as an Array
 hostnames.blocklist
 
-# Use an Exporter to convert to other formats
-UnboundExporter.new(hostnames.blocklist).data
+# List available formats
+DNSSniper::Exporters.all
+
+# Convert to desired format
+hostnames.to_unbound
 ```
 
 ### From CLI

data/dns-sniper.gemspec

@@ -3,8 +3,8 @@
 Gem::Specification.new do |spec|
   spec.name = 'dns-sniper'
   spec.license = 'MIT'
-  spec.version = '0.0.1.pre7'
-  spec.date = '2021-07-26'
+  spec.version = '0.0.1.pre10'
+  spec.date = '2022-03-07'
 
   spec.authors = ['Brody Hoskins']
   spec.email = ['brody@brody.digital']
@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
   spec.bindir = 'bin'
   spec.executables = 'dns-sniper'
 
+  spec.add_dependency 'activesupport', '>= 4.2', '< 7.0.3'
   spec.add_dependency 'down', '~> 5.1'
   spec.add_dependency 'hosts_file', '~> 1.0'
 end

data/lib/dns-sniper/conditional_fetch.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class ConditionalFetch
+    attr_reader :data, :headers, :response
+
+    def initialize(uri, options = {})
+      @options = options
+      @uri = uri
+      @uri = CGI.escape_html(@uri).to_s
+      @uri = URI.parse(@uri)
+
+      @data = nil
+      @headers = nil
+      @response = nil
+
+      fetch
+    end
+
+    # Headers
+
+    def etag
+      @etag ||= @headers&.dig('etag')
+    end
+
+    def expires_at
+      @expires_at ||= !date || !max_age ? nil : (date + max_age)
+    end
+
+    private
+
+    def fetch
+      last_etag = @options[:last_etag]
+      last_updated_at = @options[:last_updated_at]
+
+      http = Net::HTTP.new(@uri.host, @uri.port)
+      http.use_ssl = true if @uri.instance_of?(URI::HTTPS)
+
+      if @options[:last_updated_at] || @options[:last_etag]
+        response = http.head(@uri.request_uri)
+        if response.is_a?(Net::HTTPSuccess)
+          @headers = response&.each_header.to_h
+          @response = response
+          @data = response&.body
+
+          return self unless update?
+        end
+      end
+
+      request = Net::HTTP::Get.new(@uri.request_uri)
+      response = http.request(request)
+
+      @headers = response.each_header&.to_h
+      @response = response
+      @data = response&.body
+
+      self
+    end
+
+    def update?
+      last_etag = @options[:last_etag]
+      last_updated_at = @options[:last_updated_at]
+
+      return last_etag != etag if last_etag && etag && etag.strip != ''
+
+      if last_updated_at && created_at
+        if max_age
+          return (last_updated_at + max_age) < created_at
+        elsif age
+          return (last_updated_at + age) < created_at
+        else
+          return last_updated_at < created_at
+        end
+      end
+
+      true
+    end
+
+    # Headers
+
+    def age
+      @age ||= @headers&.dig('age')&.to_i
+    end
+
+    def cache_control
+      return @cache_control if @cache_control
+
+      header = @headers&.dig('cache-control')
+      @cache_control = if !header || !header.include?('max-age=')
+                         nil
+                       else
+                         header
+                       end
+    end
+
+    def created_at
+      !date || !age ? nil : (date - age)
+    end
+
+    def date
+      return @date if @date
+
+      date = @headers&.dig('date')
+      @date = date ? Time.rfc2822(date) : nil
+    end
+
+    def max_age
+      @max_age ||= cache_control ? cache_control.split('=')[1]&.to_i : nil
+    end
+  end
+end

data/lib/dns-sniper/hostnames.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
-require 'down'
-require 'hosts_file'
-require 'open-uri'
-require 'resolv'
-
 module DNSSniper
   class Hostnames
-    attr_accessor :blacklist
-    attr_accessor :whitelist
+    attr_accessor :blacklist, :whitelist
+
+    Exporters.all.each do |exporter|
+      format = exporter.to_s.split('::').last[..-9].tableize.singularize
+
+      define_method(:"to_#{format}") { exporter.new(blocklist).data }
+    end
 
     def initialize
       @blacklist = []
@@ -24,6 +24,7 @@ module DNSSniper
         domain_parts = domain.split('.')
         domain_parts.count.times do |count|
           next if count == 1
+
           whitelist += [domain_parts[count - 1, domain_parts.length].join('.')]
         end
       end

data/lib/dns-sniper/importer.rb

@@ -4,16 +4,16 @@ module DNSSniper
   class Importer
     attr_accessor :uri, :hostnames
 
-    def initialize(uri, *)
+    def initialize(uri, options = {})
       @uri = uri
-      @hostnames = File.exist?(uri) ? import_file(uri) : import_uri(uri)
+      @hostnames = File.exist?(uri) ? import_file(uri, options) : import_uri(uri, options)
     end
 
     def import_file(*)
       raise NotImplementedError, "#{self.class.name}: #import_file not supported"
     end
 
-    def import_uri(*)
+    def import_uri(_uri, _options = {})
       raise NotImplementedError, "#{self.class.name}: #import_uri not supported"
     end
 
@@ -39,7 +39,7 @@ module DNSSniper
       return false if domain.include?('?')
       return false if ip_addr?(domain)
       return false unless /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/.match?(domain)
-      return false unless /^[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}$/.match?(domain)
+      return false unless /^[a-z0-9]+([\-.]{1}[a-z0-9]+)*\.[a-z]{2,5}$/.match?(domain)
 
       begin
         return false if URI.parse(domain).is_a?(URI::HTTP)

data/lib/dns-sniper/importers/configuration_importer.rb

@@ -9,15 +9,15 @@ module DNSSniper
 
     def import_file(path, list:)
       raise ArgumentError, "#{self.class.name}: #from_path requies list to be defined" unless list
-      return [].to_set unless File.exist?(path)
+      return [] unless File.exist?(path)
 
       yaml = YAML.safe_load(File.read(path), permitted_classes: [Symbol])
-      return [].to_set unless yaml.dig(:sources)&.dig(list.to_sym)
+      return [] unless yaml.dig(:sources)&.dig(list.to_sym)
 
-      hostnames = [].to_set
+      hostnames = []
       yaml.dig(:sources).dig(list.to_sym).each do |source|
-        return [].to_set unless source.dig(:importer)
-        return [].to_set unless source.dig(:uri)
+        return [] unless source.dig(:importer)
+        return [] unless source.dig(:uri)
 
         importer = DNSSniper.const_get("#{source.dig(:importer).to_s.split('_').map(&:capitalize).join}Importer")
         if !importer

data/lib/dns-sniper/importers/domains_importer.rb

@@ -2,21 +2,17 @@
 
 module DNSSniper
   class DomainsImporter < Importer
-    def import_file(path)
-      return [].to_set unless File.exist?(path)
+    def import_file(path, *)
+      return [] unless File.exist?(path)
 
-      File.open(path).readlines(chomp: true).map { |hostname| clean(hostname) }.reject { |hostname| rejector(hostname) }.to_set
+      File.open(path).readlines(chomp: true).map { |hostname| clean(hostname) }.reject { |hostname| rejector(hostname) }
     end
 
-    def import_uri(uri)
-      begin
-        down = Down.download(uri)
-        return down.readlines(chomp: true).map { |hostname| clean(hostname) }.reject { |hostname| rejector(hostname) }.to_set
-      rescue Down::InvalidUrl => e
-        warn "#{self.class.name}: #{e}"
-      end
+    def import_uri(uri, options = {})
+      data = ConditionalFetch.new(uri, options).data
+      return [] unless data
 
-      [].to_set
+      data.split(/\n+|\r+/).reject(&:empty?)
     end
   end
 end

data/lib/dns-sniper/importers/hosts_importer.rb

@@ -3,24 +3,19 @@
 module DNSSniper
   class HostsImporter < Importer
     def import_file(path, *)
-      return [].to_set unless File.exist?(path)
+      return [] unless File.exist?(path)
 
-      HostsFile.load(path).map(&:name).map { |hostname| clean(hostname) }.reject { |hostname| rejector(hostname) }.to_set
+      HostsFile.load(path).map(&:name).map { |hostname| clean(hostname) }.reject { |hostname| rejector(hostname) }
     end
 
-    def import_uri(uri, *)
-      begin
-        down = Down.download(uri)
-        path = down.path
-      rescue Down::InvalidUrl => e
-        warn "#{self.class.name}: #{e}"
-      end
+    def import_uri(uri, options = {})
+      data = ConditionalFetch.new(uri, options).data || ''
 
-      if path
-        return HostsFile.load(path).map(&:name).map { |hostname| clean(hostname) }.reject { |hostname| rejector(hostname) }.to_set
-      end
+      arr = []
+      domains = HostsFile::Parser.new(data)
+      domains.each { |domain| arr << domain.name }
 
-      [].to_set
+      arr
     end
   end
 end

data/lib/dns-sniper.rb

@@ -1,10 +1,19 @@
 # frozen_string_literal: true
 
+require 'active_support/all'
+require 'cgi'
+require 'down'
+require 'hosts_file'
+require 'net/http'
+require 'net/https'
+require 'open-uri'
 require 'resolv'
+require 'time'
 require 'yaml'
 
+require 'dns-sniper/conditional_fetch'
 require 'dns-sniper/exporter'
 require 'dns-sniper/exporters'
 require 'dns-sniper/hostnames'
 require 'dns-sniper/importer'
-require 'dns-sniper/importers'
+require 'dns-sniper/importers'

metadata

@@ -1,15 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: dns-sniper
 version: !ruby/object:Gem::Version
-  version: 0.0.1.pre7
+  version: 0.0.1.pre10
 platform: ruby
 authors:
 - Brody Hoskins
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-26 00:00:00.000000000 Z
+date: 2022-03-07 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 7.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 7.0.3
 - !ruby/object:Gem::Dependency
   name: down
   requirement: !ruby/object:Gem::Requirement
@@ -57,6 +77,7 @@ files:
 - config.yml.example
 - dns-sniper.gemspec
 - lib/dns-sniper.rb
+- lib/dns-sniper/conditional_fetch.rb
 - lib/dns-sniper/exporter.rb
 - lib/dns-sniper/exporters.rb
 - lib/dns-sniper/exporters/bind8_exporter.rb
@@ -92,7 +113,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: Combine DNS blacklists into desired configuration format