spanx 0.1.1 → 0.3.0

data/lib/spanx/cli/report.rb

@@ -0,0 +1,77 @@
+require 'mixlib/cli'
+require 'spanx/logger'
+
+class Spanx::CLI::Report < Spanx::CLI
+
+  banner 'Usage: spanx report [ -b | -t ] [options]'
+  description 'Report on tracked and/or blocked IPs'
+
+  option :blocked,
+         :short => '-b',
+         :long => '--blocked',
+         :description => 'Show all currently blocked IPs',
+         :required => false
+
+  option :tracked,
+         :short => '-t',
+         :long => '--tracked',
+         :description => 'Show all IPs seen within the tracked period',
+         :required => false
+
+  option :summary,
+         :short => '-s',
+         :long => '--summary',
+         :description => 'Only show summary',
+         :required => false
+
+  option :config_file,
+         :short => '-c CONFIG',
+         :long => '--config CONFIG',
+         :description => 'Path to config file (YML)',
+         :required => true
+
+  option :debug,
+         :short => '-g',
+         :long => '--debug',
+         :description => 'Log to STDOUT status of execution and some time metrics',
+         :boolean => true,
+         :required => false,
+         :default => false
+
+  option :help,
+         :short => '-h',
+         :long => '--help',
+         :description => 'Show this message',
+         :on => :tail,
+         :boolean => true,
+         :show_options => true,
+         :exit => 0
+
+
+  def run(argv = ARGV)
+    generate_config(argv)
+    out = ''
+    if config[:blocked]
+      ips = Spanx::IPChecker.rate_limited_identifiers
+      out << report_ips('Blocked', ips)
+    end
+    if config[:tracked]
+      ips = Spanx::IPChecker.tracked_identifiers
+      out << report_ips('Tracked', ips)
+    end
+    if config[:summary] or (config[:blocked].nil? && config[:tracked].nil?)
+      out << "  Total tracked IPS: #{Spanx::IPChecker.tracked_identifiers.size}\n"
+      out << "  Total blocked IPS: #{Spanx::IPChecker.rate_limited_identifiers.size}\n"
+      out << "___________________\n\n"
+      out << "Keeping history for: #{config[:collector][:history] / 3600 }hrs\n"
+      out << "    Time resolution: #{config[:collector][:resolution] / 60 }min\n"
+    end
+    puts out
+    out
+  end
+
+  private
+  def report_ips name, ips = []
+    ips.empty? ? "No #{name.downcase} IPs were found.\n" : "#{name} IPs:\n" + ips.join("\n") + "\n"
+  end
+end

data/lib/spanx/cli/watch.rb

@@ -10,10 +10,17 @@ class Spanx::CLI::Watch < Spanx::CLI
   Usage: spanx watch [options]
   EOF
 
+  description 'Watch a server log file and write out a block list file'
+
   option :access_log,
-         :short => "-f ACCESS_LOG",
-         :long => "--file ACCESS_LOG",
-         :description => "Apache/nginx access log file to scan continuously",
+         :short => '-f ACCESS_LOG',
+         :long => '--file ACCESS_LOG',
+         :description => 'Apache/nginx access log file to scan continuously. Can be set multiple times.',
+         :proc => ->(f) {
+           @watched_log_files ||= []
+           @watched_log_files << f
+           @watched_log_files.uniq!
+         },
          :required => false
 
   option :config_file,
@@ -42,9 +49,9 @@ class Spanx::CLI::Watch < Spanx::CLI
          :required => false
 
   option :daemonize,
-         :short => "-d",
-         :long => "--daemonize",
-         :description => "Detach from TTY and run as a daemon",
+         :short => '-d',
+         :long => '--daemonize',
+         :description => 'Detach from TTY and run as a daemon',
          :boolean => true,
          :default => false
 
@@ -64,9 +71,9 @@ class Spanx::CLI::Watch < Spanx::CLI
          :default => false
 
   option :help,
-         :short => "-h",
-         :long => "--help",
-         :description => "Show this message",
+         :short => '-h',
+         :long => '--help',
+         :description => 'Show this message',
          :on => :tail,
          :boolean => true,
          :show_options => true,
@@ -84,8 +91,8 @@ class Spanx::CLI::Watch < Spanx::CLI
   private
 
   def validate!
-    error_exit_with_msg("Could not find file. Use -f or set :file in config_file") unless config[:access_log] && File.exists?(config[:access_log])
-    error_exit_with_msg("-b block_file is required") unless config[:block_file]
+    error_exit_with_msg('Could not find file. Use -f or set :file in config_file') unless config[:access_log] && File.exist?(config[:access_log].first)
+    error_exit_with_msg('-b block_file is required') unless config[:block_file]
   end
 
 end

data/lib/spanx/helper/exit.rb

@@ -2,10 +2,14 @@ module Spanx
   module Helper
     module Exit
       def error_exit_with_msg(msg)
-        $stderr.puts "Error: #{msg}"
-        $stderr.puts Spanx::USAGE
+        $stderr.puts "Error: #{msg}\n"
+        $stderr.puts Spanx::Usage.usage
         exit 1
       end
+      def help_exit
+        $stdout.puts Spanx::Usage.usage
+        exit 0
+      end
     end
   end
 end

data/lib/spanx/helper/subclassing.rb

@@ -24,6 +24,15 @@ module Spanx
 
         def inherited(subclass)
           subclasses[subclass.subclass_name] = subclass
+          subclass.instance_eval do
+            @description = nil
+            class << self
+              def description value = nil
+                @description ||= value
+                @description
+              end
+            end
+          end
         end
       end
     end

data/lib/spanx/logger.rb

@@ -38,7 +38,7 @@ module Spanx
         end
         log "(#{"%9.2f" % (1000 * elapsed_time)}ms) #{message}"
         returned_from_block
-      rescue Exception => e
+      rescue StandardError => e
         elapsed_time = Time.now - start
         log "(#{"%9.2f" % (1000 * elapsed_time)}ms) error: #{e.message} for #{message} "
       end

data/lib/spanx/notifier/slack.rb

@@ -0,0 +1,42 @@
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Spanx
+  module Notifier
+    class Slack < Base
+      attr_reader :config
+
+      def initialize(config)
+        @config = config[:slack]
+      end
+
+      def publish blocked_ip
+        return nil unless enabled?
+        message = generate_block_ip_message(blocked_ip)
+
+        uri = endpoint
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+        request = Net::HTTP::Post.new(uri.request_uri, {'Content-Type' => 'text/json'})
+        request.body = { text: message }.to_json
+
+        http.request(request)
+      end
+
+      def endpoint
+        return nil unless enabled?
+        token = config[:token]
+        base_url = config[:base_url]
+        URI.parse("#{base_url}/services/hooks/incoming-webhook?token=#{token}")
+      end
+
+      def enabled?
+        config && config[:enabled]
+      end
+    end
+  end
+end

data/lib/spanx/runner.rb

@@ -68,7 +68,7 @@ module Spanx
     private
 
     def validate_args!(args)
-      raise("Invalid actor") unless (args - %w[collector log_reader writer analyzer]).empty?
+      raise('Invalid actor') unless (args - %w[collector log_reader writer analyzer]).empty?
     end
   end
 end

data/lib/spanx/usage.rb

@@ -1,9 +1,13 @@
 module Spanx
-  USAGE = %q{Usage: spanx command [options]
-  watch   -- Watch a server log file and write out a block list file
-  analyze -- Analyze IP traffic and save blocked IPs into Redis
-  flush   -- Remove all IP blocks and delete previous tracking of that IP
-  disable -- Disable IP blocking
-  enable  -- Enable IP blocking if disabled
-}
+  class Usage
+    HEADER = %q{Usage: spanx [ --help | <command> ]  [options]}
+
+    def self.usage
+      out = ''
+      out << HEADER + "\n\n"
+      Spanx::CLI.subclasses.each_pair{|command, clazz| out << "#{sprintf "%10s", command}\t#{clazz.description}\n"}
+      out << "\nRun 'spanx <command> --help' to see command-specific options.\n"
+      out
+    end
+  end
 end

data/lib/spanx/version.rb

@@ -1,3 +1,3 @@
 module Spanx
-  VERSION = "0.1.1"
+  VERSION = "0.3.0"
 end

data/spanx.gemspec

@@ -15,18 +15,12 @@ Gem::Specification.new do |gem|
   gem.require_paths = %w(lib)
   gem.version       = Spanx::VERSION
 
-  gem.add_dependency 'pause', '~> 0.0.4'
+  gem.add_dependency 'pause'
   gem.add_dependency 'file-tail'
   gem.add_dependency 'mixlib-cli'
   gem.add_dependency 'daemons'
   gem.add_dependency 'tinder'
-  gem.add_dependency 'mail', '~> 2.4.4'
-
-  gem.add_development_dependency 'rspec'
-  gem.add_development_dependency 'fakeredis'
-  gem.add_development_dependency 'timecop'
-
-  gem.add_development_dependency 'guard-rspec'
-  gem.add_development_dependency 'rb-fsevent'
+  gem.add_dependency 'mail'
 
+  gem.add_dependency 'webmachine'
 end

data/spec/spanx/actor/analyzer_spec.rb

@@ -6,7 +6,7 @@ describe Spanx::Actor::Analyzer do
   include Spanx::Helper::Timing
 
   before do
-    pause_config = mock(resolution: 10, history: 100, redis_host: "1.2.3.4", redis_port: 1, redis_db: 1)
+    pause_config = double(resolution: 10, history: 100, redis_host: "1.2.3.4", redis_port: 1, redis_db: 1)
     Pause.stub(:config).and_return(pause_config)
     pause_analyzer = Pause::Analyzer.new
     Pause.stub(:analyzer).and_return(pause_analyzer)
@@ -76,8 +76,8 @@ describe Spanx::Actor::Analyzer do
 
       before do
         Spanx::IPChecker.should_receive(:tracked_identifiers).and_return([ip1, ip2])
-        Spanx::IPChecker.should_receive(:new).with(ip1).and_return(mock(analyze: nil))
-        Spanx::IPChecker.should_receive(:new).with(ip2).and_return(mock(analyze: nil))
+        Spanx::IPChecker.should_receive(:new).with(ip1).and_return(double(analyze: nil))
+        Spanx::IPChecker.should_receive(:new).with(ip2).and_return(double(analyze: nil))
       end
 
       it "analyzes each IP found" do
@@ -88,8 +88,8 @@ describe Spanx::Actor::Analyzer do
 
   context "notifiers" do
     let(:notifiers) { ["FakeNotifier"] }
-    let(:fake_notifier) { mock() }
-    let(:blocked_ip) { mock() }
+    let(:fake_notifier) { double() }
+    let(:blocked_ip) { double() }
 
     class FakeNotifier
     end

data/spec/spanx/actor/log_reader_spec.rb

@@ -3,66 +3,103 @@ require 'file/tail'
 require 'timeout'
 require 'thread'
 require 'tempfile'
+require 'timecop'
 
 describe Spanx::Actor::LogReader do
 
-  def test_log_file(file, expected_ip_count, expected_line_count, whitelist = nil)
-    counter = 0
-    ip_hash = {}
-    reader = Spanx::Actor::LogReader.new(file, Queue.new, 1, whitelist)
-    reader.file.backward(1000)
-
-    t_reader = Thread.new do
-      begin
-        timeout(read_timeout) do
-          reader.read do |ip|
-            counter += 1
-            ip_hash[ip] ||= 0
-            ip_hash[ip] += 1
+  subject(:reader) { Spanx::Actor::LogReader.new(files, queue, 1, whitelist) }
+  let(:files) { [] }
+  let(:queue) { [] }
+  let(:whitelist) { nil }
+  let(:read_timeout) { 0.1 }
+
+  describe '#read' do
+    let(:tempfile) { Tempfile.new('access.log') }
+    let!(:file) { Spanx::Actor::File.new(tempfile.path) }
+
+    it 'yields each ip in a file as it is written' do
+      expect { |b|
+        reader_thread = Thread.new do
+          begin
+            timeout(read_timeout) do
+              reader.read(file, &b)
+            end
+          rescue TimeoutError
           end
         end
-      rescue TimeoutError
-      end
+
+        ::File.open(tempfile.path, 'a') do |t|
+          t.puts '9.9.9.9 - some stuff'
+          t.puts '9.9.9.10 - some other stuff'
+          t.puts '9.9.9.9 - whoa moar stuff'
+        end
+
+        reader_thread.join
+      }.to yield_successive_args('9.9.9.9', '9.9.9.10', '9.9.9.9')
     end
 
-    yield if block_given?
+    context 'if a line matches a whitelist' do
+      let(:whitelist) { double }
 
-    t_reader.join
+      before do
+        allow(whitelist).to receive(:match?).with("9.9.9.9 - some stuff\n").and_return(true)
+        allow(whitelist).to receive(:match?).with("9.9.9.10 - some other stuff\n").and_return(false)
+      end
 
-    counter.should eql(expected_line_count)
-    ip_hash.keys.size.should eql(expected_ip_count)
-  end
+      it 'skips that line' do
+        expect { |b|
+          reader_thread = Thread.new do
+            begin
+              timeout(read_timeout) do
+                reader.read(file, &b)
+              end
+            rescue TimeoutError
+            end
+          end
 
-  let(:file_name) { "spec/fixtures/access.log.1" }
-  let(:read_timeout) { 0.05 }
+          ::File.open(tempfile.path, 'a') do |t|
+            t.puts '9.9.9.9 - some stuff'
+            t.puts '9.9.9.10 - some other stuff'
+          end
 
-  context "#read" do
-    it "should be able to read and parse IPs from a static file" do
-      test_log_file(file_name, 82, 104)
+          reader_thread.join
+        }.to yield_successive_args('9.9.9.10')
+      end
     end
+  end
 
-    it "should be able to read and parse IPs from a file being appended to" do
-      tempfile = Tempfile.new("access.log")
+  describe '#run' do
+    let!(:file1) { Tempfile.new('log.log') }
+    let!(:file2) { Tempfile.new('loge.log') }
+    let!(:files) { [file1.path, file2.path] }
+    let(:read_timeout) { 0.1 }
 
-      contents = ::File.read(file_name)
-      tempfile.write(contents)
-      tempfile.close
+    after do
+      file1.close
+      file2.close
+    end
 
-      test_log_file(tempfile.path, 83, 105) do
-        t_log_appender = Thread.new do
-          ::File.open(tempfile.path, "a") do |t|
-            t.write("9.9.9.9 - content")
+    xit 'pushes ips from each watched file onto queue' do
+      Timecop.freeze(Time.at(1409270561))
+
+      runner = Thread.new do
+        begin
+          timeout(read_timeout) do
+            reader.run
+            reader.threads.last.join
           end
+        rescue TimeoutError
         end
-        t_log_appender.join
       end
-    end
-  end
 
-  context "#whitelist" do
-    let(:whitelist_file) { "spec/fixtures/whitelist.txt" }
-    it "should exclude googlebot log lines" do
-      test_log_file("spec/fixtures/access.log.bots", 1, 1, Spanx::Whitelist.new(whitelist_file))
+      sleep 1
+
+      file1.puts '1.1.1.1 - everyone loves a log!'
+      file2.puts '2.2.2.2 - only some people love a loge'
+
+      runner.join
+
+      expect(queue).to match_array([['1.1.1.1', 1409270561], ['2.2.2.2', 1409270561]])
     end
   end
 end