hosts_updater 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f035702771324651cd8c2fa2f1e7248237e96b8b
+  data.tar.gz: afb2b6556b7133fc4261a5cd02006f33e42fda02
+SHA512:
+  metadata.gz: aa6f8c165bdfbec09b539ec9ed4602307fe09e9f3a7df91a65bc16eed63ec5a7a5178fb941b191f232640d41936a7a64fdff64a3f316fbb4a5151cada952f52b
+  data.tar.gz: cd03ec664d6908db1a0bc0c706441a43093ff999625ca8a745e28df3b669a0fa0db47ef94c12f57860d3dd604a565413ad10f21d4458da41dac3c29d0edbf08c

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+pkg/*.gem

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2014 Bernard Potocki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# Hosts Updater
+
+This script indends to help manage large collection of blacklisted domains. It's best to describe it as replacement for adblock, but instead of slowing down your browser it blocks domains of ad and scam websites at DNS level.
+
+## Quick start guide
+
+```sh
+$ gem install hosts_updater
+$ sudo hosts-updater --help
+```
+
+It required sudo in order to write to `/etc/hosts` file.
+
+## What it does?
+
+This script download lists of malicious domains from following websites:
+
+- http://www.malwaredomainlist.com/hostslist/hosts.txt
+- http://winhelp2002.mvps.org/hosts.txt
+- http://someonewhocares.org/hosts/hosts
+- http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext
+
+All of those lists are combined, duplicates removed and combined with your current `hosts` file (no data will should lost as original `hosts` file is copied and reused in future).
+
+## How is it better than AdBlock/Ghostery etc?
+
+Each browser plugin that blocks ads is using [large amount of resources, hogs browser, and often causes strange errors](http://www.reddit.com/r/programming/comments/25j41u/adblock_pluss_effect_on_firefoxs_memory_usage/chhpomw). In constrast to this blacklisting domains in `/etc/hosts` takes couple seconds after file change, but after that has nearly zero impact on performance. Additionally AdBlock is removing parts of websited very aggresively, often leaving indended layout broken - it should be less common problem with domain blacklisting.
+
+On the other hand there are small amount of ads that are unblockable by domain blacklisting - that are visible from time to time, but probably not often enough to complain ;)
+
+## How to configure what domains I want whitelisted/blacklisted?
+
+During first run `hosts-updater` will create folder in `/etc/hosts.d`, in which it will store all configuration files:
+
+- `hosts.custom` - this will be copy of old `/etc/hosts` and it will be added to `/etc/hosts` at top upon each use of `hosts-updater`. You should store you developer domains and all similar stuff inside of this file, as it will never be modified by this script. If you need to blacklist additional domains you should put them here too.
+- `hosts.auto` - this will be regenerated upon calling `hosts-updater` with `--update` flag. It stores downloaded blacklists and is used to regenerate `/etc/hosts`. It's good idea to refresh it from time to time.
+- `hosts.whitelist` - in this file you can place domain that you want to have access to despite figuring in one of downloaded lists. In order to do so simply paste full domain name inside there (one domain per line) and it will be picked up during next `hosts-updater` run.
+
+## Any other configuration options?
+
+See `hosts-updater --help` for more configuration options.
+
+## Are you owner of those blacklists?
+
+No - I'm just using them with proper attribution. All kudos and suggestions should be sent to maintainers of appropriate lists.
+
+## License
+
+MIT License
+
+Copyright © 2014 Bernard Potocki

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/bin/hosts-updater

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'hosts_updater'
+
+options = { :sources => {} }
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: hosts-updater [options]"
+
+  opts.on("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+
+  opts.on("--hosts-file", "Path to hosts file to write (default: /etc/hosts)") do |file|
+    opts[:hosts_file] = file
+  end
+
+  opts.on("--hosts-directory", "Path to hosts.d config directory (default: /etc/hosts.d/)") do |file|
+    opts[:hosts_dir] = file
+  end
+
+  opts.on("--hosts-auto-name", "Name for auto-generated file in hosts.d dir (default: hosts.auto)") do |file|
+    opts[:hosts_auto_name] = file
+  end
+
+  opts.on("--hosts-custom-name", "Name for custom hosts file in hosts.d dir (default: hosts.custom)") do |file|
+    opts[:hosts_custom_name] = file
+  end
+
+  opts.on("--hosts-whitelist-name", "Name for file with whitelisted domains in hosts.d dir (defaule: hosts.whitelist)") do |file|
+    opts[:hosts_whitelist_name] = file
+  end
+
+
+  opts.on("-i", "--ip", "Redirect all blacklisted domains to following IP (default: 0.0.0.0)") do |ip|
+    options[:id] = ip
+  end
+
+  opts.on("-u", "--[no-]update", "Update sources") do |u|
+    options[:update] = u
+  end
+
+  HostsUpdater::SOURCES.each do |source_key, source_data|
+    opts.on("--skip-#{source_key}", "Skip downloading #{source_data[1]} (#{source_data[0]})") do |s|
+      options[:sources][source_key] = false
+    end
+  end
+
+  opts.on("-q", "--quiet", "Run completely silently") do |q|
+    options[:quiet] = q
+  end
+
+  opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+    options[:verbose] = v
+  end
+end.parse!
+
+HostsUpdater.new(options).run

data/hosts_updater.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "hosts_updater/version"
+
+Gem::Specification.new do |s|
+  s.name        = "hosts_updater"
+  s.version     = HostsUpdater::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Bernard Potocki"]
+  s.email       = ["bernard.potocki@imanel.org"]
+  s.homepage    = "http://github.com/imanel/hosts_updater"
+  s.license     = 'MIT'
+  s.summary     = %q{Update your /etc/hosts with list of unwanted domains}
+  s.description = %q{Update your /etc/hosts with list of unwanted domains}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.lines.map { |f| File.basename(f.chomp) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency('hosts', '~> 0.1.1')
+end

data/lib/hosts_updater.rb

@@ -0,0 +1,146 @@
+require 'hosts'
+require 'logger'
+require 'open-uri'
+
+class HostsUpdater
+
+  SOURCES = {
+    :mdl => ['http://www.malwaredomainlist.com/hostslist/hosts.txt', 'Malware Domain List'],
+    :mvps => ['http://winhelp2002.mvps.org/hosts.txt', 'MVPS Hosts'],
+    :swc => ['http://someonewhocares.org/hosts/hosts', "Dan Pollock's List"],
+    :yoyo => ['http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext', "Peter Lowe's Ad Server List"]
+  }
+
+  DEFAULT_OPTIONS = {
+    :hosts_file => '/etc/hosts',
+    :hosts_directory => '/etc/hosts.d/',
+    :hosts_auto_name => 'hosts.auto',
+    :hosts_custom_name => 'hosts.custom',
+    :hosts_whitelist_name => 'hosts.whitelist',
+    :ip => '0.0.0.0',
+    :sources => SOURCES,
+    :update => false,
+    :quiet => false,
+    :verbose => false
+  }
+
+  def initialize(options = {})
+    sources = options.delete(:sources) || {}
+    @options = DEFAULT_OPTIONS.merge(options)
+    @options[:sources].merge!(sources)
+  end
+
+  # Create required files, update auto file if necessary and generate hosts file.
+  # Need to be ran as root in order to access /etc/hosts
+  def run
+    unless ENV['USER'] == 'root'
+      logger.error 'Error: Must run as root'
+      exit
+    end
+
+    bootstrap
+    update_auto_file if @options[:update]
+    update_hosts_file
+
+    logger.info 'Done.'
+  end
+
+  private
+
+  def logger
+    return @logger if defined?(@logger)
+    @logger = Logger.new(STDOUT)
+    @logger.level = if @options[:quiet]
+                      Logger::ERROR
+                    elsif @options[:verbose]
+                      Logger::DEBUG
+                    else
+                      Logger::INFO
+                    end
+    @logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
+    @logger
+  end
+
+  # Create /etc/hosts.d and files inside.
+  # If /etc/hosts.d/hosts.custom does not exists then it will copy content of /etc/hosts there.
+  def bootstrap
+    unless Dir.exists? @options[:hosts_directory]
+      logger.debug "Creating configuration directory at #{@options[:hosts_directory]}"
+      FileUtils.mkdir_p @options[:hosts_directory]
+    end
+
+    unless File.exists? hosts_custom_location
+      logger.debug "Copying #{@options[:hosts_file]} to #{hosts_custom_location}"
+      File.write(hosts_custom_location, File.read(@options[:hosts_file]))
+    end
+
+    unless File.exists? hosts_auto_location
+      logger.debug "Writing default #{hosts_auto_location}"
+      File.touch(hosts_auto_location)
+    end
+
+    unless File.exists? hosts_whitelist_location
+      logger.debug "Writing default #{hosts_whitelist_location}"
+      File.write(hosts_whitelist_location, "# List domains that should be ignored here.\n# One domain per line, lines starting with # will be ignored.\n")
+    end
+  end
+
+  # Download files mentioned in SOURCES and combine them into one saved as hosts_auto_location.
+  # Any comments and duplicates are ommited from original files.
+  def update_auto_file
+    elements = []
+    @options[:sources].each do |source_key, source_data|
+      next unless source_data
+      elements += download_source(source_data)
+    end
+    hosts = Hosts::File.new
+    hosts.elements = elements.uniq(&:name).sort { |e1,e2| e1.name <=> e2.name }
+    hosts.elements.insert 0, Hosts::Comment.new(' This file was auto-generated by hosts-updater.')
+    hosts.elements.insert 1, Hosts::Comment.new(' Changes to this file may cause incorrect behavior and will be lost if the code is regenerated.')
+    hosts.elements.insert 2, Hosts::EmptyElement.new
+
+    File.write(hosts_auto_location, hosts.to_s(:force_generation => true))
+  end
+
+  # Combine hosts.custom and hosts.auto files into one and write to hosts_location.
+  # Domains listed in hosts.whitelist are ommited from hosts.auto list, but not from hosts.custom
+  def update_hosts_file
+    auto = Hosts::File.read(hosts_auto_location).elements
+    auto.reject! { |el| ! el.is_a? Aef::Hosts::Entry }
+    auto.reject! { |el| whitelist.include? el.name }
+    auto.each do |el|
+      el.address = @options[:ip]
+    end
+
+    hosts = Hosts::File.new(@options[:hosts_file])
+    hosts.elements = Hosts::File.read(hosts_custom_location).elements
+    hosts.elements << Hosts::EmptyElement.new
+    hosts.elements << Hosts::Section.new('HOSTS-UPDATER', :elements => auto)
+    logger.debug "Writing to #{@options[:hosts_file]}"
+    hosts.write
+  end
+
+  def download_source(source)
+    logger.debug "Downloading source #{source[0]}" + (source[1] ? " (#{source[1]})" : "")
+    data = open(source[0])
+    Hosts::File.parse(data.read).elements.select { |el| el.is_a? Aef::Hosts::Entry }
+  end
+
+  # Read hosts.whitelist file and ignore all lines starting with #
+  def whitelist
+    @whitelist ||= File.read(hosts_whitelist_location).lines.collect(&:strip).reject { |el| /^#/ =~ el }
+  end
+
+  def hosts_auto_location
+    @options[:hosts_directory] + @options[:hosts_auto_name]
+  end
+
+  def hosts_custom_location
+    @options[:hosts_directory] + @options[:hosts_custom_name]
+  end
+
+  def hosts_whitelist_location
+    @options[:hosts_directory] + @options[:hosts_whitelist_name]
+  end
+
+end

data/lib/hosts_updater/version.rb

@@ -0,0 +1,3 @@
+class HostsUpdater
+  VERSION = "1.0.0"
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: hosts_updater
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Bernard Potocki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hosts
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+description: Update your /etc/hosts with list of unwanted domains
+email:
+- bernard.potocki@imanel.org
+executables:
+- hosts-updater
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/hosts-updater
+- hosts_updater.gemspec
+- lib/hosts_updater.rb
+- lib/hosts_updater/version.rb
+homepage: http://github.com/imanel/hosts_updater
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Update your /etc/hosts with list of unwanted domains
+test_files: []