spanx 0.1.0

data/lib/spanx/actor/collector.rb

@@ -0,0 +1,64 @@
+require 'pause'
+require 'spanx/logger'
+require 'spanx/helper/timing'
+
+module Spanx
+  module Actor
+    class Collector
+      include Spanx::Helper::Timing
+
+      attr_accessor :queue, :config, :semaphore, :cache
+
+      def initialize(config, queue)
+        @queue = queue
+        @config = config
+        @semaphore = Mutex.new
+        @cache = Hash.new(0)
+      end
+
+      def run
+        Thread.new do
+          Thread.current[:name] = "collector:queue"
+          loop do
+            unless queue.empty?
+              Logger.logging "caching [#{queue.size}] keys locally" do
+                while !queue.empty?
+                  semaphore.synchronize {
+                    increment_ip *(queue.pop)
+                  }
+                end
+              end
+            end
+            sleep 1
+          end
+        end
+
+        Thread.new do
+          Thread.current[:name] = "collector:flush"
+          loop do
+            semaphore.synchronize {
+              Logger.logging "flushing cache with [#{cache.keys.size}] keys" do
+                cache.each_pair do |key, count|
+                  Spanx::IPChecker.new(key[0]).increment!(key[1], count)
+                end
+                reset_cache
+              end
+            }
+            sleep config[:collector][:flush_interval]
+          end
+        end
+      end
+
+      def increment_ip(ip, timestamp)
+        cache[[ip, period_marker(config[:collector][:resolution], timestamp)]] += 1
+      end
+
+      private
+
+      def reset_cache
+        @cache.clear
+        GC.start
+      end
+    end
+  end
+end

data/lib/spanx/actor/log_reader.rb

@@ -0,0 +1,46 @@
+require 'file-tail'
+
+module Spanx
+  module Actor
+    class LogReader
+      attr_accessor :file, :queue, :whitelist
+
+      def initialize file, queue, interval = 1, whitelist = nil
+        @file = Spanx::Actor::File.new(file)
+        @file.interval = interval
+        @file.backward(0)
+        @whitelist = whitelist
+        @queue = queue
+      end
+
+      def run
+        Thread.new do
+          Thread.current[:name] = "log_reader"
+          Logger.log "tailing the log file #{file.path}...."
+          self.read do |line|
+            queue << [line, Time.now.to_i ] if line
+          end
+        end
+      end
+
+      def read &block
+        @file.tail do |line|
+          block.call(extract_ip(line)) unless whitelist && whitelist.match?(line)
+        end
+      end
+
+      def close
+        (@file.close if @file) rescue nil
+      end
+
+      def extract_ip line
+        matchers = line.match(/^((\d{1,3}\.?){4})/)
+        matchers[1] unless matchers.nil?
+      end
+    end
+
+    class File < ::File
+      include ::File::Tail
+    end
+  end
+end

data/lib/spanx/actor/writer.rb

@@ -0,0 +1,68 @@
+require 'spanx/logger'
+require 'spanx/helper/exit'
+
+module Spanx
+  module Actor
+    class Writer
+      include Spanx::Helper::Exit
+
+      attr_accessor :config
+
+      def initialize config
+        @config = config
+        @block_file = config[:block_file]
+        @run_command = config[:run_command]
+      end
+
+      def run
+        Thread.new do
+          Thread.current[:name] = "writer"
+          loop do
+            self.write
+            sleep config[:writer][:write_interval]
+          end
+        end
+      end
+
+      def write
+        if Spanx::IPChecker.enabled?
+          ips = Spanx::IPChecker.blocked_identifiers
+        else
+          Logger.log "writing empty block file due to disabled state"
+          ips = []
+        end
+
+        begin
+          contents_previous = File.read(@block_file) rescue nil
+          Logger.logging "writing out [#{ips.size}] IP block rules to [#{@block_file}]" do
+            File.open(@block_file, "w") do |file|
+              ips.sort.each do |ip|
+                # TODO: make this a customizable ERB template
+                file.puts("deny #{ip};")
+              end
+            end
+          end
+          contents_now = File.read(@block_file)
+          if contents_now != contents_previous && @run_command
+            Logger.logging "running command [#{@run_command}]" do
+              run_command
+            end
+          end
+        rescue Exception => e
+          error_exit_with_msg "ERROR writing to block file #{@block_file} or running command: #{e.inspect}"
+        end
+      end
+
+      def run_command
+        result = system(@run_command)
+        if result
+          "executed successfully"
+        elsif result == false
+          "returned non-zero exit status"
+        elsif result.nil?
+          "failed -- #{$?}"
+        end
+      end
+    end
+  end
+end

data/lib/spanx/cli.rb

@@ -0,0 +1,47 @@
+require 'mixlib/cli'
+require 'spanx/helper/exit'
+
+module Spanx
+  class CLI
+    include Mixlib::CLI
+    include Spanx::Helper::Exit
+    include Spanx::Helper::Subclassing
+
+    attr_reader :args
+
+    # the first element of ARGV should be a subcommand, which maps to
+    # a class in spanx/cli/
+    def run(args = ARGV)
+      @args = args
+      validate!
+      Spanx::CLI.subclass_class(args.shift).new.run(args)
+    end
+
+    private
+
+    def validate!
+      error_exit_with_msg("No command given") if args.empty?
+      @command = args.first
+      error_exit_with_msg("No command found matching #{@command}") unless Spanx::CLI.subclasses.include?(@command)
+    end
+
+    def generate_config(argv)
+      parse_options argv
+      config.merge! Spanx::Config.new(config[:config_file])
+      parse_options argv
+
+      if config[:debug]
+        STDOUT.sync = true
+      end
+
+      Spanx::Logger.enable if config[:debug]
+    end
+
+  end
+end
+
+require 'spanx/cli/watch'
+require 'spanx/cli/analyze'
+require 'spanx/cli/disable'
+require 'spanx/cli/enable'
+require 'spanx/cli/flush'

data/lib/spanx/cli/analyze.rb

@@ -0,0 +1,50 @@
+require 'thread'
+require 'mixlib/cli'
+require 'daemons/daemonize'
+require 'spanx/logger'
+require 'spanx/runner'
+
+class Spanx::CLI::Analyze < Spanx::CLI
+
+  banner "Usage: spanx analyze [options]"
+
+  option :daemonize,
+         :short => "-d",
+         :long => "--daemonize",
+         :boolean => true,
+         :default => false
+
+  option :config_file,
+         :short => '-c CONFIG',
+         :long => '--config CONFIG',
+         :description => 'Path to config file (YML)',
+         :required => true
+
+  option :debug,
+         :short => '-g',
+         :long => '--debug',
+         :description => 'Log status to STDOUT',
+         :boolean => true,
+         :required => false,
+         :default => false
+
+  option :audit_file,
+         :short => '-a AUDIT',
+         :long  => '--audit AUDIT_FILE',
+         :description => 'Historical record of IP blocking decisions',
+         :required => false
+
+  option :help,
+         :short => "-h",
+         :long => "--help",
+         :description => "Show this message",
+         :on => :tail,
+         :boolean => true,
+         :show_options => true,
+         :exit => 0
+
+  def run(argv = ARGV)
+    generate_config(argv)
+    Spanx::Runner.new("analyzer", config).run
+  end
+end

data/lib/spanx/cli/disable.rb

@@ -0,0 +1,36 @@
+require 'mixlib/cli'
+require 'spanx/logger'
+
+class Spanx::CLI::Disable < Spanx::CLI
+
+  banner "Usage: spanx disable [options]"
+
+  option :config_file,
+         :short => '-c CONFIG',
+         :long => '--config CONFIG',
+         :description => 'Path to config file (YML)',
+         :required => true
+
+  option :debug,
+         :short => '-g',
+         :long => '--debug',
+         :description => 'Log to STDOUT status of execution and some time metrics',
+         :boolean => true,
+         :required => false,
+         :default => false
+
+  option :help,
+         :short => "-h",
+         :long => "--help",
+         :description => "Show this message",
+         :on => :tail,
+         :boolean => true,
+         :show_options => true,
+         :exit => 0
+
+
+  def run(argv = ARGV)
+    generate_config(argv)
+    Spanx::IPChecker.disable
+  end
+end

data/lib/spanx/cli/enable.rb

@@ -0,0 +1,36 @@
+require 'mixlib/cli'
+require 'spanx/logger'
+
+class Spanx::CLI::Enable < Spanx::CLI
+
+  banner "Usage: spanx enable [options]"
+
+  option :config_file,
+         :short => '-c CONFIG',
+         :long => '--config CONFIG',
+         :description => 'Path to config file (YML)',
+         :required => true
+
+  option :debug,
+         :short => '-g',
+         :long => '--debug',
+         :description => 'Log to STDOUT status of execution and some time metrics',
+         :boolean => true,
+         :required => false,
+         :default => false
+
+  option :help,
+         :short => "-h",
+         :long => "--help",
+         :description => "Show this message",
+         :on => :tail,
+         :boolean => true,
+         :show_options => true,
+         :exit => 0
+
+
+  def run(argv = ARGV)
+    generate_config(argv)
+    Spanx::IPChecker.enable
+  end
+end

data/lib/spanx/cli/flush.rb

@@ -0,0 +1,36 @@
+require 'mixlib/cli'
+require 'spanx/logger'
+
+class Spanx::CLI::Flush < Spanx::CLI
+
+  banner "Usage: spanx flush [options]"
+
+  option :config_file,
+         :short => '-c CONFIG',
+         :long => '--config CONFIG',
+         :description => 'Path to config file (YML)',
+         :required => true
+
+  option :debug,
+         :short => '-g',
+         :long => '--debug',
+         :description => 'Log to STDOUT status of execution and some time metrics',
+         :boolean => true,
+         :required => false,
+         :default => false
+
+  option :help,
+         :short => "-h",
+         :long => "--help",
+         :description => "Show this message",
+         :on => :tail,
+         :boolean => true,
+         :show_options => true,
+         :exit => 0
+
+
+  def run(argv = ARGV)
+    generate_config(argv)
+    Spanx::IPChecker.unblock_all
+  end
+end

data/lib/spanx/cli/watch.rb

@@ -0,0 +1,91 @@
+require 'mixlib/cli'
+require 'thread'
+require 'daemons/daemonize'
+require 'spanx/runner'
+
+class Spanx::CLI::Watch < Spanx::CLI
+  include Spanx::Helper::Exit
+
+  banner <<-EOF
+  Usage: spanx watch [options]
+  EOF
+
+  option :access_log,
+         :short => "-f ACCESS_LOG",
+         :long => "--file ACCESS_LOG",
+         :description => "Apache/nginx access log file to scan continuously",
+         :required => false
+
+  option :config_file,
+         :short => '-c CONFIG',
+         :long => '--config CONFIG',
+         :description => 'Path to config file (YML)',
+         :required => true
+
+  option :block_file,
+         :short => '-b BLOCK_FILE',
+         :long => '--block_file BLOCK_FILE',
+         :description => 'Output file to store NGINX block list',
+         :required => false
+
+  option :whitelist_file,
+         :short => '-w WHITELIST',
+         :long => '--whitelist WHITELIST',
+         :description => 'File with newline separated reg exps, to exclude lines from access log',
+         :required => false,
+         :default => nil
+
+  option :run_command,
+         :short => '-r <shell command>',
+         :long => '--run <shell command>',
+         :description => 'Shell command to run anytime blocked ip file changes, for example "sudo pkill -HUP nginx"',
+         :required => false
+
+  option :daemonize,
+         :short => "-d",
+         :long => "--daemonize",
+         :description => "Detach from TTY and run as a daemon",
+         :boolean => true,
+         :default => false
+
+  option :analyze,
+         :short => '-z',
+         :long => '--analyze',
+         :description => 'Analyze IPs also (as opposed to running `spanx analyze` in another process)',
+         :boolean => true,
+         :default => false
+
+  option :debug,
+         :short => '-g',
+         :long => '--debug',
+         :description => 'Log to STDOUT status of execution and some time metrics',
+         :boolean => true,
+         :required => false,
+         :default => false
+
+  option :help,
+         :short => "-h",
+         :long => "--help",
+         :description => "Show this message",
+         :on => :tail,
+         :boolean => true,
+         :show_options => true,
+         :exit => 0
+
+  def run(argv = ARGV)
+    generate_config(argv)
+    validate!
+    runners = %w(log_reader collector writer)
+    runners << "analyzer" if config[:analyze]
+
+    Spanx::Runner.new(*runners, config).run
+  end
+
+  private
+
+  def validate!
+    error_exit_with_msg("Could not find file. Use -f or set :file in config_file") unless config[:access_log] && File.exists?(config[:access_log])
+    error_exit_with_msg("-b block_file is required") unless config[:block_file]
+  end
+
+end