spanx 0.1.0

data/lib/spanx/config.rb

@@ -0,0 +1,45 @@
+require 'yaml'
+require 'pause'
+require 'spanx/helper/exit'
+require 'spanx/ip_checker'
+
+module Spanx
+  class Config < Hash
+    include Spanx::Helper::Exit
+
+    attr_accessor :filename
+
+    def initialize(filename)
+      super
+      @filename = filename
+      load_file
+
+      Pause.configure do |pause|
+        pause.redis_host = self[:redis][:host]
+        pause.redis_port = self[:redis][:port]
+        pause.redis_db = self[:redis][:database]
+
+        pause.resolution = self[:collector][:resolution]
+        pause.history = self[:collector][:history]
+      end
+
+      if self.has_key?(:analyzer) && self[:analyzer].has_key?(:period_checks)
+        self[:analyzer][:period_checks].each do |check|
+          Spanx::IPChecker.check check[:period_seconds].to_i, check[:max_allowed].to_i, check[:block_ttl].to_i
+        end
+      end
+
+      self
+    end
+
+    private
+
+    def load_file
+      begin
+        self.merge! ::YAML.load_file(filename)
+      rescue Errno::ENOENT
+        error_exit_with_msg "Unable to find config_file at #{filename}"
+      end
+    end
+  end
+end

data/lib/spanx/helper.rb

@@ -0,0 +1,8 @@
+require 'spanx/helper/timing'
+require 'spanx/helper/exit'
+require 'spanx/helper/subclassing'
+
+module Spanx
+  module Helper
+  end
+end

data/lib/spanx/helper/exit.rb

@@ -0,0 +1,11 @@
+module Spanx
+  module Helper
+    module Exit
+      def error_exit_with_msg(msg)
+        $stderr.puts "Error: #{msg}"
+        $stderr.puts Spanx::USAGE
+        exit 1
+      end
+    end
+  end
+end

data/lib/spanx/helper/subclassing.rb

@@ -0,0 +1,31 @@
+module Spanx
+  module Helper
+    module Subclassing
+
+      def self.included(base)
+        base.extend ClassMethods
+      end
+
+      module ClassMethods
+
+        def subclasses
+          @@subclasses ||= {}
+        end
+
+        def subclass_name
+          name.split("::").last.downcase
+        end
+
+        def subclass_class(subclass)
+          subclasses[subclass]
+        end
+
+        private
+
+        def inherited(subclass)
+          subclasses[subclass.subclass_name] = subclass
+        end
+      end
+    end
+  end
+end

data/lib/spanx/helper/timing.rb

@@ -0,0 +1,9 @@
+module Spanx
+  module Helper
+    module Timing
+      def period_marker(resolution, timestamp = Time.now)
+        timestamp.to_i / resolution * resolution
+      end
+    end
+  end
+end

data/lib/spanx/ip_checker.rb

@@ -0,0 +1,5 @@
+require 'pause'
+
+class Spanx::IPChecker < Pause::Action
+  scope "spanx:ip"
+end

data/lib/spanx/logger.rb

@@ -0,0 +1,47 @@
+module Spanx
+  module Logger
+
+    class << self
+      def enable
+        class << self
+          self.send(:define_method, :log, proc { |msg| _log(msg) })
+          self.send(:define_method, :logging, proc { |msg, &block| _logging(msg, &block) })
+        end
+      end
+
+      def disable
+        class << self
+          self.send(:define_method, :log, proc { |msg|})
+          self.send(:define_method, :logging, proc { |msg, &block| block.call })
+        end
+      end
+
+      def log(msg)
+      end
+
+      def logging(msg, &block)
+        block.call
+      end
+
+      private
+
+      def _log(msg)
+        puts "#{Time.now}: #{sprintf("%-20s", Thread.current[:name])} - #{msg}"
+      end
+
+      def _logging(message, &block)
+        start = Time.now
+        returned_from_block = yield
+        elapsed_time = Time.now - start
+        if returned_from_block.is_a?(String) && returned_from_block != ""
+          message += " - #{returned_from_block}"
+        end
+        log "(#{"%9.2f" % (1000 * elapsed_time)}ms) #{message}"
+        returned_from_block
+      rescue Exception => e
+        elapsed_time = Time.now - start
+        log "(#{"%9.2f" % (1000 * elapsed_time)}ms) error: #{e.message} for #{message} "
+      end
+    end
+  end
+end

data/lib/spanx/notifier/audit_log.rb

@@ -0,0 +1,18 @@
+module Spanx
+  module Notifier
+    class AuditLog < Base
+      attr_accessor :audit_file
+
+      def initialize(config)
+        @audit_file = config[:audit_file]
+        raise ArgumentError.new("config[:audit_file] is required for this notifier to work") unless @audit_file
+      end
+
+      def publish(b)
+        File.open(@audit_file, "a") do |file|
+          file.puts "#{Time.now} -- #{sprintf("%-16s", b.identifier)} period=#{b.period_check.period_seconds}s max=#{b.period_check.max_allowed} count=#{b.sum} ttl=#{b.period_check.block_ttl}s"
+        end
+      end
+    end
+  end
+end

data/lib/spanx/notifier/base.rb

@@ -0,0 +1,22 @@
+module Spanx
+  module Notifier
+    class Base
+
+      # Takes an instance of the Spanx::BlockedIp struct.
+      # Overwrite this a subclass to define real behavior
+      def publish(blocked_ip)
+        raise 'Abstract Method Not Implemented'
+      end
+
+      protected
+
+        def generate_block_ip_message(blocked_ip)
+          violated_period = blocked_ip.period_check
+          "#{blocked_ip.identifier} blocked @ #{Time.at(blocked_ip.timestamp)} " \
+            "for #{violated_period.block_ttl/60}mins, for #{blocked_ip.sum} requests over " \
+            "#{violated_period.period_seconds/60}mins, with #{violated_period.max_allowed} allowed."
+        end
+
+    end
+  end
+end

data/lib/spanx/notifier/campfire.rb

@@ -0,0 +1,47 @@
+require 'tinder'
+module Spanx
+  module Notifier
+
+    # Notify Campfire room of a new IP blocked
+    class Campfire < Base
+
+      attr_accessor :account, :room_id, :token
+
+      def initialize(config)
+        @enabled = config[:campfire][:enabled]
+        if self.enabled?
+          _init(config[:campfire][:account], config[:campfire][:room_id], config[:campfire][:token])
+        end
+      end
+
+      def publish(blocked_ip)
+        speak generate_block_ip_message(blocked_ip) if enabled?
+      end
+
+      def _init(account, room_id, token)
+        @account = account
+        @room_id = room_id
+        @token = token
+      end
+
+      def enabled?
+        @enabled
+      end
+
+      private
+
+      def campfire
+        @campfire ||= Tinder::Campfire.new(account, :token => token)
+      end
+
+      def room
+        campfire.find_room_by_id(room_id)
+      end
+
+      def speak message
+        r = room
+        r.speak message if r
+      end
+    end
+  end
+end

data/lib/spanx/notifier/email.rb

@@ -0,0 +1,61 @@
+require 'mail'
+
+module Spanx
+  module Notifier
+    class Email < Base
+
+      attr_reader :config, :thread
+
+      def initialize(config)
+        @config = config[:email]
+
+        configure_email_gateway
+      end
+
+      def publish(blocked_ip)
+        return unless enabled?
+
+        @thread = Thread.new do
+          Thread.current[:name] = "email notifier"
+          Logger.log "sending notification email for #{blocked_ip.identifier}"
+
+          mail = Mail.new
+          mail.to = config[:to]
+          mail.from = config[:from]
+          mail.subject = subject(blocked_ip)
+          mail.body = generate_block_ip_message(blocked_ip)
+
+          mail.deliver
+        end
+      end
+
+      def enabled?
+        config && config[:enabled]
+      end
+
+      private
+
+      def subject(blocked_ip)
+        "#{config[:subject] || "IP Blocked:"} #{blocked_ip.identifier}"
+      end
+
+      def configure_email_gateway
+        return unless enabled?
+
+        Mail.defaults do
+          delivery_method :smtp, {}
+        end
+
+        settings = Mail::Configuration.instance.delivery_method.settings
+        settings[:address] = config[:gateway]
+        settings[:port] = '587'
+        settings[:domain] = config[:domain]
+        settings[:user_name] = config[:from]
+        settings[:password] = config[:password]
+        settings[:authentication] = :plain
+        settings[:enable_starttls_auto] = true
+        settings[:openssl_verify_mode] = 'none'
+      end
+    end
+  end
+end

data/lib/spanx/runner.rb

@@ -0,0 +1,74 @@
+require 'mixlib/cli'
+require 'spanx/logger'
+require 'spanx/actor/collector'
+require 'spanx/actor/analyzer'
+require 'spanx/actor/log_reader'
+require 'spanx/actor/writer'
+require 'spanx/whitelist'
+require 'thread'
+
+# Spanx::Runner is initialized with a list of actors to run
+# and a config hash. It is then run to activate each actor
+# and join one of the running threads.
+#
+# Example:
+#     Spanx::Runner.new("analyzer", {}).run
+#     Spanx::Runner.new("analyzer", "writer", {}).run
+#
+# Valid actors are:
+#    collector
+#    analyzer
+#    writer
+#    log_reader
+#
+module Spanx
+  class Runner
+    attr_accessor :config, :queue, :actors
+
+    def initialize(*args)
+      @config = args.last.is_a?(Hash) ? args.pop : {}
+      @queue = Queue.new
+      validate_args!(args)
+      @actors = args.map { |actor| self.send(actor.to_sym) }
+
+      Daemonize.daemonize if config[:daemonize]
+
+      STDOUT.sync = true if config[:debug]
+    end
+
+    def run
+      threads = actors.map(&:run)
+      threads.last.join
+    end
+
+    # actors
+
+    def collector
+      @collector ||= Spanx::Actor::Collector.new(config, queue)
+    end
+
+    def log_reader
+      @log_reader ||= Spanx::Actor::LogReader.new(config[:access_log], queue, config[:log_reader][:tail_interval], whitelist)
+    end
+
+    def writer
+      @writer ||= Spanx::Actor::Writer.new(config)
+    end
+
+    def analyzer
+      @analyzer ||= Spanx::Actor::Analyzer.new(config)
+    end
+
+    # helpers
+
+    def whitelist
+      @whitelist ||= Spanx::Whitelist.new(config[:whitelist_file])
+    end
+
+    private
+
+    def validate_args!(args)
+      raise("Invalid actor") unless (args - %w[collector log_reader writer analyzer]).empty?
+    end
+  end
+end

data/lib/spanx/usage.rb

@@ -0,0 +1,9 @@
+module Spanx
+  USAGE = %q{Usage: spanx command [options]
+  watch   -- Watch a server log file and write out a block list file
+  analyze -- Analyze IP traffic and save blocked IPs into Redis
+  flush   -- Remove all IP blocks and delete previous tracking of that IP
+  disable -- Disable IP blocking
+  enable  -- Enable IP blocking if disabled
+}
+end

data/lib/spanx/version.rb

@@ -0,0 +1,3 @@
+module Spanx
+  VERSION = "0.1.0"
+end

data/lib/spanx/whitelist.rb

@@ -0,0 +1,31 @@
+require 'spanx/helper/exit'
+
+module Spanx
+  class Whitelist
+    include Spanx::Helper::Exit
+    attr_accessor :patterns, :filename
+
+    def initialize(filename)
+      @patterns = []
+      @filename = filename
+
+      load_file
+    end
+
+    def match?(line)
+      @patterns.any? do |p|
+        p.match(line)
+      end
+    end
+
+    def load_file
+      if filename
+        begin
+          @patterns = ::File.readlines(filename).reject{|line| line =~ /^#/}.map{|p| %r{#{p.chomp()}} }
+        rescue Errno::ENOENT
+          error_exit_with_msg("Unable to find whitelist file at #{filename}")
+        end
+      end
+    end
+  end
+end

data/spanx.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/spanx/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Konstantin Gredeskoul","Eric Saxby"]
+  gem.email         = %w(kigster@gmail.com sax@livinginthepast.org)
+  gem.description   = %q{Real time IP parsing and rate detection gem for access_log files}
+  gem.summary       = %q{Real time IP parsing and rate detection gem for access_log files}
+  gem.homepage      = "https://github.com/wanelo/spanx"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "spanx"
+  gem.require_paths = %w(lib)
+  gem.version       = Spanx::VERSION
+
+  gem.add_dependency 'pause', '~> 0.0.3'
+  gem.add_dependency 'file-tail'
+  gem.add_dependency 'mixlib-cli'
+  gem.add_dependency 'daemons'
+  gem.add_dependency 'tinder'
+  gem.add_dependency 'mail', '~> 2.4.4'
+
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'fakeredis'
+  gem.add_development_dependency 'timecop'
+
+  gem.add_development_dependency 'guard-rspec'
+  gem.add_development_dependency 'rb-fsevent'
+
+end