dns-sniper 0.0.1pre → 0.0.1.pre6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c6bab67f9579e0762d197d29dd188ced90277fa05573bb84e472be2c42a5a3e
-  data.tar.gz: 4123ba5a1c02a7f5d4972055b4b1743aae7bd4f995cd1336626a3f8a68ca7e23
+  metadata.gz: f366147ef18ba1adc0449b75001bd217b18b27727fe81406980cb3dec815da82
+  data.tar.gz: b7ecdce1690d770cfc6cbe69c4745465e45d70d1704fe955cd17786ff5cc6b14
 SHA512:
-  metadata.gz: 479adbba0f5d557ef178acadc65461ee0be27684f4a4ab95bdd8f2b129a15b2583658fcea86193ee1c30519c5dba712b6c784619d79eaa345b7fcd993a1a19c1
-  data.tar.gz: b5ef2952418417f81d8c47b7cfe3052fe5e18242d5d87c3328f9713b5175bf94f48dc0ac3e3a9c8b4ddcc944637f2d8aea07fef00d15dc2b84c24e8f72fc3b8c
+  metadata.gz: b4ef85c2cbb26409ff3bd52097ff5c95c80ec1b8303c12ab0f871fe056a33fe7a5139272e0ea6e4fd9ea8cb2c459d528155250dee70b62174284a07baefba6b9
+  data.tar.gz: 9f2c3aa82e0f604314a9de321e5960bef302906d5dec42ab4a7801b14bbd932f11b8dd1a05fed346a75e3a895aee72ac7506085766b25e45fe228eca55ef6360

data/Gemfile

@@ -1,3 +1,5 @@
-source "https://rubygems.org"
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
 
 gemspec

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2020 Brody Hoskins
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,77 @@
+# dns-sniper
+
+dns-sniper is a command line utility that combines online DNS blacklists and combines them into the desired configuration format.
+
+## Supported Formats
+
+### Input formats
+
+* HOSTS
+* plaintext (hostnames as newline-seperated list)
+
+### Output formats
+
+* bind 8
+* dnsmasq
+* HOSTS
+* plaintext (hostnames as newline-seperated list)
+* unbound
+
+## Installation
+
+Using bundler, add to the `Gemfile`:
+
+```ruby
+gem 'dns-sniper'
+```
+
+Or standalone:
+
+```
+$ gem install dns-sniper
+```
+
+## Sample Usage
+
+### From within Ruby
+
+```ruby
+require 'dns-sniper'
+
+hostnames = DNSSniper::Hostnames.new
+
+# Block domain names
+hostnames.add_from('https://pgl.yoyo.org/as/serverlist.php?hostformat=hosts;showintro=0;mimetype=plaintext') # From the web
+hostnames.add_from('https://raw.githubusercontent.com/brodyhoskins/dns-blocklists/master/tracking.list')
+hostnames.add_from('~/.config/dns-sniper/blocklists.list') # From filesystem
+
+# Manually add domain name
+hostnames.add('ads.yahoo.com')
+hostnames.add(['ads.doubleclick.net', 'ads.msn.com'])
+
+# Remove whitelisted domain names
+hostnames.remove_from('~/.config/dns-sniper/whitelisted-hostnames.list')
+hostnames.remove_from('https://example.com/whitelisted.hosts')
+
+# Manually remove domain name
+hostnames.remove('favoritewebsite.com')
+hostnames.remove(['favoritewebsite.com', 'otherfavoritewebsite.com'])
+
+# Convert to configuration file
+hostnames.to_format('dnsmasq')
+hostnames.to_format('unbound')
+```
+
+### From CLI
+
+See `dns-sniper --help`
+
+Using the CLI version makes it easy to update configuration formats automatically. For example:
+
+```bash
+#!/usr/bin/env bash
+
+/path/to/dns-sniper -f ~/.config/dns-sniper/blacklist.list -w ~/.config/dns-sniper/whitelist.list -o unbound > /etc/unbound/unbound.conf.t/blocklist.conf
+
+service unbound reload
+```

data/Rakefile

@@ -1,6 +1,8 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/dns-sniper

@@ -1,52 +1,50 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-require "optparse"
-require "set"
-require "dns-sniper"
+require 'optparse'
+require 'set'
+require 'dns-sniper'
 
-VALID_FORMATS = ["text", "unbound"]
-Options = Struct.new(:blacklist_urls_file, :whitelisted_hosts_file, :format)
+VALID_FORMATS = DNSSniper::Exporters.all.inject([]) { |arr, f| arr << f.name.to_s.sub('DNSSniper::', '').sub('Exporter', '').downcase }
+Options = Struct.new(:conf_path, :format)
 
 class Parser
   def self.parse(options)
-    args = Options.new("world")
+    args = Options.new('world')
     opt_parser = OptionParser.new do |opts|
       opts.banner = "Usage: #{File.basename(__FILE__)} -f PATH -o FORMAT [options]\n#{File.basename(__FILE__)} combines online DNS blacklists and combines them into the desired configuration FORMAT.\n\n"
-      opts.on("-f", "--blacklist=PATH", "PATH to file with line-separated list of URL’s of blacklists") do |path|
-        args.blacklist_urls_file = path
+      opts.on('-c', '--conf=PATH', 'PATH to YAML configuration file') do |path|
+        args.conf_path = path
       end
-      opts.on("-w", "--whitelist=path", "Path to file with line-separated list of whitelisted hostnames") do |path|
-        args.whitelisted_hosts_file = path
-      end
-      opts.on("-o", "--output=FORMAT", "FORMAT to output — one of #{VALID_FORMATS.join(", ")}") do |format|
+      opts.on('-o', '--output=FORMAT', "FORMAT to output — one of #{VALID_FORMATS.join(', ')}") do |format|
         args.format = format
       end
-      opts.on("-h", "--help", "Prints this help") do
+      opts.on('-h', '--help', 'Prints this help') do
         puts opts
         exit
       end
     end
     opt_parser.parse!(options)
-    return args
+    args
   end
 end
 
 options = Parser.parse ARGV
 
-unless VALID_FORMATS.include?(options[:format])
-  STDERR.puts "Error: Format ”#{options[:format]}” unrecognized"
+unless File.exist?(options[:conf_path])
+  warn "Error: Unable to access ”#{options[:conf_path]}”"
   exit(-1)
 end
 
-unless File.exist?(options[:blacklist_urls_file])
-  STDERR.puts "Error: Unable to access ”#{options[:blacklist_urls_file]}”"
-  exit(-1)
-end
-
-if options[:whitelisted_hosts_file] and not File.exist?(options[:whitelisted_hosts_file])
-  STDERR.puts "Error: Unable to access ”#{options[:whitelisted_hosts_file]}”"
+exporter = DNSSniper.const_get("#{options[:format].to_s.split('_').map(&:capitalize).join}Exporter")
+if !exporter
+  warn "Error: Invalid format #{options[:format]}"
   exit(-1)
 end
 
 hostnames = DNSSniper::Hostnames.new
-puts hostnames.add_from(File.open(options[:blacklist_urls_file]).readlines).remove_from(options[:whitelisted_hosts_file]).to_format(options[:format])
+hostnames.blacklist += DNSSniper::ConfigurationImporter.new(options[:conf_path], list: :reject).hostnames
+hostnames.whitelist += DNSSniper::ConfigurationImporter.new(options[:conf_path], list: :allow).hostnames
+
+exporter = exporter.new(hostnames.blocklist)
+puts exporter.data

data/dns-sniper.gemspec

@@ -1,23 +1,35 @@
+# frozen_string_literal: true
+
 Gem::Specification.new do |spec|
-  spec.name = "dns-sniper"
-  spec.license = "MIT"
-  spec.version = "0.0.1pre"
-  spec.date = "2020-08-13"
-  spec.authors = ["Brody Hoskins"]
-  spec.email = ["brody@brody.digital"]
+  spec.name = 'dns-sniper'
+  spec.license = 'MIT'
+  spec.version = '0.0.1.pre6'
+  spec.date = '2020-11-11'
+
+  spec.authors = ['Brody Hoskins']
+  spec.email = ['brody@brody.digital']
 
-  spec.summary = "Command line utility that combines online DNS blacklists and combines them into the desired configuration format"
-  spec.description = "Command line utility that combines online DNS blacklists and combines them into the desired configuration format"
-  spec.homepage = "https://github.com/brodyhoskins/dns-sniper"
+  spec.summary = 'Combine DNS blacklists into desired configuration format'
+  spec.description = <<~DESC.gsub(/\n/, ' ').strip
+    dns-sniper generates DNS configuration files based on various user-defined
+    blacklists online. Configuration files can be generated for use in Ruby
+    applications or from the command line.
+  DESC
+  spec.homepage = 'https://github.com/brodyhoskins/dns-sniper'
 
-  spec.files = `git ls-files`.split($/)
+  spec.metadata = {
+    'homepage_uri' => 'https://github.com/brodyhoskins/dns-sniper',
+    'source_code_uri' => 'https://github.com/brodyhoskins/dns-sniper'
+  }
 
-  spec.bindir = "bin"
-  spec.executables = "dns-sniper"
-  spec.require_paths = ["lib"]
+  spec.files = Dir['lib/**/*']
+  spec.files += Dir['[A-Z]*'] + Dir['test/**/*']
+  spec.files.reject! { |fn| fn.include? 'CVS' }
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 2.1.2"
+  spec.bindir = 'bin'
+  spec.executables = 'dns-sniper'
 
-  spec.add_development_dependency "down", "~> 5.1"
-  spec.add_development_dependency "hosts_file", "~> 1.0"
+  spec.add_dependency 'down', '~> 5.1'
+  spec.add_dependency 'hosts_file', '~> 1.0'
 end

data/lib/dns-sniper.rb

@@ -1,3 +1,10 @@
-require "dns-sniper/hostnames"
-require "dns-sniper/formatter"
-require "dns-sniper/formatters"
+# frozen_string_literal: true
+
+require 'resolv'
+require 'yaml'
+
+require 'dns-sniper/exporter'
+require 'dns-sniper/exporters'
+require 'dns-sniper/hostnames'
+require 'dns-sniper/importer'
+require 'dns-sniper/importers'

data/lib/dns-sniper/{formatter.rb → exporter.rb}

@@ -1,11 +1,15 @@
+# frozen_string_literal: true
+
 module DNSSniper
-  class Formatter
+  class Exporter
+    attr_accessor :data, :hostnames
+
     def initialize(hostnames, options = {})
       @hostnames = hostnames
-      @options = options
+      @data = output(options)
     end
 
-    def output
+    def output(*)
       raise NotImplementedError, "Error: #output isn’t supported by #{self.class.name}"
     end
   end

data/lib/dns-sniper/exporters.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  module Exporters
+    module_function
+
+    attr_reader :registered
+    @registered = []
+
+    def register(class_name, autoload_require)
+      DNSSniper.autoload(class_name, autoload_require)
+      @registered << class_name
+    end
+
+    def all
+      @registered.map { |name| DNSSniper.const_get(name) }
+    end
+
+    def find(name)
+      all.find { |c| c.name.downcase == name.to_s.downcase } or raise NameError, "Unknown exporter \"#{name}\""
+    end
+  end
+end
+
+DNSSniper::Exporters.register :Bind8Exporter, 'dns-sniper/exporters/bind8_exporter'
+DNSSniper::Exporters.register :DnsmasqExporter, 'dns-sniper/exporters/dnsmasq_exporter'
+DNSSniper::Exporters.register :HostsExporter, 'dns-sniper/exporters/hosts_exporter'
+DNSSniper::Exporters.register :NetgearExporter, 'dns-sniper/exporters/netgear_exporter'
+DNSSniper::Exporters.register :TextExporter, 'dns-sniper/exporters/text_exporter'
+DNSSniper::Exporters.register :UnboundExporter, 'dns-sniper/exporters/unbound_exporter'

data/lib/dns-sniper/exporters/bind8_exporter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class Bind8Exporter < Exporter
+    def output(options = {})
+      raise ArgumentError, 'zone_file is required' unless defined?(options[:zone_file])
+
+      str = ''.dup
+      @hostnames.each do |hostname|
+        str << "zone \"#{hostname}\" { type master; notify no; file \"#{options[:zone_file]}\"; };#{$/}"
+      end
+      str
+    end
+  end
+end

data/lib/dns-sniper/exporters/dnsmasq_exporter.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class DnsmasqExporter < Exporter
+    def output(*)
+      str = ''.dup
+      @hostnames.each do |hostname|
+        str << "server=/#{hostname}/#{$/}"
+      end
+      str
+    end
+  end
+end

data/lib/dns-sniper/exporters/hosts_exporter.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class HostsExporter < Exporter
+    def output(*)
+      str = ''.dup
+      @hostnames.each do |hostname|
+        str << "127.0.0.1\t#{hostname}#{$/}"
+      end
+      str
+    end
+  end
+end

data/lib/dns-sniper/exporters/netgear_exporter.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class NetgearExporter < Exporter
+    def output(*)
+      str = ''.dup
+      @hostnames.each_with_index do |hostname, i|
+        str << "[517003_e]: #{i + 1}) #{hostname}\n"
+      end
+      str
+    end
+  end
+end

data/lib/dns-sniper/exporters/text_exporter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class TextExporter < Exporter
+    def output(_options = {})
+      @hostnames.to_a.join($/)
+    end
+  end
+end

data/lib/dns-sniper/exporters/unbound_exporter.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class UnboundExporter < Exporter
+    def output(*)
+      str = ''.dup
+      str << "server:#{$/}"
+      @hostnames.each do |hostname|
+        str << "  local-zone: \"#{hostname}\" static#{$/}"
+      end
+      str
+    end
+  end
+end

data/lib/dns-sniper/hostnames.rb

@@ -1,176 +1,34 @@
-require "down"
-require "hosts_file"
-require "open-uri"
-require "resolv"
+# frozen_string_literal: true
+
+require 'down'
+require 'hosts_file'
+require 'open-uri'
+require 'resolv'
 
 module DNSSniper
   class Hostnames
-    def initialize(options = {})
-      @hostnames = [].to_set
-      self
-    end
-
-    def add(hostname)
-      hostname = clean(hostname)
-      @hostnames << hostname if hostname
-      self
-    end
-
-    def add_many(hostnames)
-      hostnames.each { |hostname| add(hostname) }
-      self
-    end
-
-    def add_from(paths_or_urls)
-      return self unless paths_or_urls
-      if (paths_or_urls.class == String)
-        paths_or_urls = [paths_or_urls]
-      end
-
-      paths_or_urls.each do |path_or_url|
-        path_or_url = path_or_url.strip
-
-        if File.exist?(path_or_url)
-          contents = File.open(path_or_url).readlines
-        else
-          begin
-            down = Down.download(path_or_url)
-            path_or_url = down.path
-            contents = down.readlines
-          rescue Down::NotFound
-            STDERR.puts "\"#{path_or_url}\" does not exist"
-            return self
-          rescue Down::ResponseError
-            STDERR.puts "\"#{path_or_url}\": No data from server"
-            return self
-          end
-        end
-
-        case syntax(path_or_url, contents)
-        when nil
-          STDERR.puts "Error: Syntax: Syntax of \"#{path_or_url}\" not recognized, ignored"
-          return self
-        when "hosts"
-          add_many from_hosts_file(path_or_url)
-        when "hostnames"
-          add_many from_hostnames_file(contents)
-        end
-      end
-
-      self
-    end
-
-    def remove(hostname)
-      hostname = clean(hostname)
-      @hostnames = @hostnames - [hostname] if hostname
-      self
-    end
-
-    def remove_many(hostnames)
-      hostnames.each { |hostname| remove(hostname) }
-      self
-    end
-
-    def remove_from(paths_or_urls)
-      return self unless paths_or_urls
-      if (paths_or_urls.class == String)
-        paths_or_urls = [paths_or_urls]
-      end
-
-      paths_or_urls.each do |path_or_url|
-        path_or_url = path_or_url.strip
-
-        if File.exist?(path_or_url)
-          contents = File.open(path_or_url).readlines
-        else
-          begin
-            down = Down.download(path_or_url)
-            path_or_url = down.path
-            contents = down.readlines
-          rescue Down::NotFound
-            STDERR.puts "\"#{path_or_url}\" does not exist"
-            return self
-          rescue Down::ResponseError
-            STDERR.puts "\"#{path_or_url}\": No data from server"
-            return self
-          end
-        end
-
-        case syntax(path_or_url, contents)
-        when nil
-          STDERR.puts "Error: Syntax: Syntax of \"#{path_or_url}\" not recognized, ignored"
-          return self
-        when "hosts"
-          remove_many from_hosts_file(path_or_url)
-        when "hostnames"
-          remove_many from_hostnames_file(contents)
-        end
-      end
+    attr_accessor :blacklist
+    attr_accessor :whitelist
 
+    def initialize
+      @blacklist = [].to_set
+      @whitelist = [].to_set
       self
     end
 
-    def to_format(format)
-      format = format.capitalize
-      begin
-        klass = Sniper.const_get(format)
-        klass.new(@hostnames.to_a).output
-      rescue NameError
-        return false
-      end
-    end
-
-    def to_a
-      @hostnames.to_a
-    end
-
-    def to_text
-      @hostnames.to_a.join("\n")
-    end
-
-    def to_unbound
-      str = "server:\n"
-      @hostnames.each do |hostname|
-        str << "  local-zone: \"#{hostname}\" static\n"
-      end
-      str
-    end
+    def blocklist
+      blacklist = @blacklist
+      whitelist = @whitelist
 
-    private
-
-    def clean(hostname)
-      hostname = hostname.downcase.strip
-      hostname = hostname.sub("www.", "")
-      hostname_top_domain = "#{hostname.split(".")[-2]}.#{hostname.split(".")[-1]}"
-
-      if not hostname.include?("#") and not ["broadcasthost", "localhost", ""].include?(hostname) and not @hostnames.include?(hostname_top_domain)
-        hostname
-      else
-        nil
-      end
-    end
-
-    def syntax(path_or_url, contents)
-      contents.each do |line|
-        next if line.include?("#")
-        line = line.downcase
-
-        if line.strip.split(/\s/).first =~ Regexp.union([Resolv::IPv4::Regex, Resolv::IPv6::Regex])
-          return "hosts"
-        elsif line.include? "." and not line.include? "http" and path_or_url.end_with?(".list")
-          return "hostnames"
+      whitelist.each do |domain|
+        domain_parts = domain.split('.')
+        domain_parts.count.times do |count|
+          next if count == 1
+          whitelist += [domain_parts[count - 1, domain_parts.length].join('.')]
         end
       end
-      nil
-    end
-
-    def from_hosts_file(path_or_url)
-      # TODO: Remove downloading file twice
-      HostsFile.load(path_or_url).map(&:name)
-    end
 
-    def from_hostnames_file(contents)
-      contents.each { |line| add(line) }
+      blacklist - whitelist
     end
   end
 end

data/lib/dns-sniper/importer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class Importer
+    attr_accessor :uri, :hostnames
+
+    def initialize(uri, *)
+      @uri = uri
+      @hostnames = File.exist?(uri) ? import_file(uri) : import_uri(uri)
+    end
+
+    def import_file(*)
+      raise NotImplementedError, "#{self.class.name}: #import_file not supported"
+    end
+
+    def import_uri(*)
+      raise NotImplementedError, "#{self.class.name}: #import_uri not supported"
+    end
+
+    # Helper methods
+
+    def clean(domain)
+      domain = domain.split('#')[0] if domain[0] != '#' && domain.include?('#')
+      domain = domain.split(':')[0] if domain.include?(':') && !ip_addr?(domain)
+      domain = domain.sub('www.', '') if domain.start_with?('www.') && domain.scan('www.').count == 1
+
+      domain.chomp.gsub(/\s+/, '').downcase
+    end
+
+    def rejector(domain)
+      !domain?(domain)
+    end
+
+    def domain?(domain)
+      return false if domain == ''
+      return false if domain.gsub('#', '').gsub(/\s+/, '').empty?
+      return false if domain[0] == '#'
+      return false if domain.include?(':')
+      return false if domain.include?('?')
+      return false if ip_addr?(domain)
+      return false unless /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/.match?(domain)
+      return false unless /^[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}$/.match?(domain)
+
+      begin
+        return false if URI.parse(domain).is_a?(URI::HTTP)
+      rescue URI::InvalidURIError; end
+      true
+    end
+
+    def ip_addr?(domain)
+      domain =~ Regexp.union([Resolv::IPv4::Regex, Resolv::IPv6::Regex])
+    end
+  end
+end

data/lib/dns-sniper/importers.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  module Importers
+    module_function
+
+    attr_reader :registered
+    @registered = []
+
+    def register(class_name, autoload_require)
+      DNSSniper.autoload(class_name, autoload_require)
+      @registered << class_name
+    end
+
+    def all
+      @registered.map { |name| DNSSniper.const_get(name) }
+    end
+
+    def find(name)
+      all.find { |c| c.name.downcase == name.to_s.downcase } or raise NameError, "Unknown Importer \"#{name}\""
+    end
+  end
+end
+
+DNSSniper::Importers.register :ConfigurationImporter, 'dns-sniper/importers/configuration_importer'
+DNSSniper::Importers.register :DomainsImporter, 'dns-sniper/importers/domains_importer'
+DNSSniper::Importers.register :HostsImporter, 'dns-sniper/importers/hosts_importer'

data/lib/dns-sniper/importers/configuration_importer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class ConfigurationImporter < Importer
+    def initialize(uri, list:)
+      @uri = uri
+      @hostnames = File.exist?(uri) ? import_file(uri, list: list) : import_uri(uri, list: list)
+    end
+
+    def import_file(path, list:)
+      raise ArgumentError, "#{self.class.name}: #from_path requies list to be defined" unless list
+      return [].to_set unless File.exist?(path)
+
+      yaml = YAML.safe_load(File.read(path), permitted_classes: [Symbol])
+      return [].to_set unless yaml.dig(:sources)&.dig(list.to_sym)
+
+      hostnames = [].to_set
+      yaml.dig(:sources).dig(list.to_sym).each do |source|
+        return [].to_set unless source.dig(:importer)
+        return [].to_set unless source.dig(:uri)
+
+        importer = DNSSniper.const_get("#{source.dig(:importer).to_s.split('_').map(&:capitalize).join}Importer")
+        if !importer
+          next
+        else
+          importer = importer.new(source.dig(:uri))
+          hostnames += importer.hostnames
+        end
+      end
+
+      hostnames
+    end
+
+    def import_uri(_uri, list:)
+      raise NotImplementedError, "#{self.class.name}: #from_uri not supported"
+    end
+  end
+end

data/lib/dns-sniper/importers/domains_importer.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class DomainsImporter < Importer
+    def import_file(path)
+      return [].to_set unless File.exist?(path)
+
+      File.open(path).readlines(chomp: true).map { |hostname| clean(hostname) }.reject { |hostname| rejector(hostname) }.to_set
+    end
+
+    def import_uri(uri)
+      begin
+        down = Down.download(uri)
+        return down.readlines(chomp: true).map { |hostname| clean(hostname) }.reject { |hostname| rejector(hostname) }.to_set
+      rescue Down::InvalidUrl => e
+        warn "#{self.class.name}: #{e}"
+      end
+
+      [].to_set
+    end
+  end
+end

data/lib/dns-sniper/importers/hosts_importer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module DNSSniper
+  class HostsImporter < Importer
+    def import_file(path, *)
+      return [].to_set unless File.exist?(path)
+
+      HostsFile.load(path).map(&:name).map { |hostname| clean(hostname) }.reject { |hostname| rejector(hostname) }.to_set
+    end
+
+    def import_uri(uri, *)
+      begin
+        down = Down.download(uri)
+        path = down.path
+      rescue Down::InvalidUrl => e
+        warn "#{self.class.name}: #{e}"
+      end
+
+      if path
+        return HostsFile.load(path).map(&:name).map { |hostname| clean(hostname) }.reject { |hostname| rejector(hostname) }.to_set
+      end
+
+      [].to_set
+    end
+  end
+end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: dns-sniper
 version: !ruby/object:Gem::Version
-  version: 0.0.1pre
+  version: 0.0.1.pre6
 platform: ruby
 authors:
 - Brody Hoskins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-13 00:00:00.000000000 Z
+date: 2020-11-11 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.1.2
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.1.2
 - !ruby/object:Gem::Dependency
   name: down
   requirement: !ruby/object:Gem::Requirement
@@ -31,7 +17,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.1'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -45,15 +31,16 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-description: Command line utility that combines online DNS blacklists and combines
-  them into the desired configuration format
+description: dns-sniper generates DNS configuration files based on various user-defined
+  blacklists online. Configuration files can be generated for use in Ruby applications
+  or from the command line.
 email:
 - brody@brody.digital
 executables:
@@ -61,22 +48,34 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
 - Gemfile
 - Gemfile.lock
+- MIT-LICENSE
+- README.md
 - Rakefile
 - bin/dns-sniper
 - dns-sniper.gemspec
 - lib/dns-sniper.rb
-- lib/dns-sniper/formatter.rb
-- lib/dns-sniper/formatters.rb
-- lib/dns-sniper/formatters/text.rb
-- lib/dns-sniper/formatters/unbound.rb
+- lib/dns-sniper/exporter.rb
+- lib/dns-sniper/exporters.rb
+- lib/dns-sniper/exporters/bind8_exporter.rb
+- lib/dns-sniper/exporters/dnsmasq_exporter.rb
+- lib/dns-sniper/exporters/hosts_exporter.rb
+- lib/dns-sniper/exporters/netgear_exporter.rb
+- lib/dns-sniper/exporters/text_exporter.rb
+- lib/dns-sniper/exporters/unbound_exporter.rb
 - lib/dns-sniper/hostnames.rb
+- lib/dns-sniper/importer.rb
+- lib/dns-sniper/importers.rb
+- lib/dns-sniper/importers/configuration_importer.rb
+- lib/dns-sniper/importers/domains_importer.rb
+- lib/dns-sniper/importers/hosts_importer.rb
 homepage: https://github.com/brodyhoskins/dns-sniper
 licenses:
 - MIT
-metadata: {}
+metadata:
+  homepage_uri: https://github.com/brodyhoskins/dns-sniper
+  source_code_uri: https://github.com/brodyhoskins/dns-sniper
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -95,6 +94,5 @@ requirements: []
 rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
-summary: Command line utility that combines online DNS blacklists and combines them
-  into the desired configuration format
+summary: Combine DNS blacklists into desired configuration format
 test_files: []

data/.gitignore

@@ -1,9 +0,0 @@
-.DS_Store
-.DS_Store?
-._*
-.Spotlight-V100
-.Trashes
-ehthumbs.db
-Thumbs.db
-
-.ruby-version

data/lib/dns-sniper/formatters.rb

@@ -1,24 +0,0 @@
-module DNSSniper
-  module Formatters
-    extend self
-
-    attr_reader :registered
-    @registered = []
-
-    def register(class_name, autoload_require)
-      DNSSniper.autoload(class_name, autoload_require)
-      self.registered << class_name
-    end
-
-    def all
-      Sniper::Formatters.registered.map { |name| Sniper.const_get(name) }
-    end
-
-    def find(name)
-      all.find { |c| c.name.downcase == name.to_s.downcase } or raise NameError, "unknown carrier #{name}"
-    end
-  end
-end
-
-DNSSniper::Formatters.register :Text, "dns-sniper/formatters/text"
-DNSSniper::Formatters.register :Unbound, "dns-sniper/formatters/unbound"

data/lib/dns-sniper/formatters/text.rb

@@ -1,12 +0,0 @@
-module DNSSniper
-  class Text < Formatter
-    def initialize(hostnames, options = {})
-      @hostnames = hostnames
-      @options = options
-    end
-
-    def output
-      @hostnames.to_a.join($/)
-    end
-  end
-end

data/lib/dns-sniper/formatters/unbound.rb

@@ -1,16 +0,0 @@
-module DNSSniper
-  class Unbound < Formatter
-    def initialize(hostnames, options = {})
-      @hostnames = hostnames
-      @options = options
-    end
-
-    def output
-      str = "server:#{$/}"
-      @hostnames.each do |hostname|
-        str << "  local-zone: \"#{hostname}\" static#{$/}"
-      end
-      str
-    end
-  end
-end