spanx 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. data/.gitignore +20 -0
  2. data/.pairs +13 -0
  3. data/.rspec +2 -0
  4. data/.rvmrc +1 -0
  5. data/Gemfile +4 -0
  6. data/Guardfile +13 -0
  7. data/LICENSE +22 -0
  8. data/README.md +175 -0
  9. data/Rakefile +2 -0
  10. data/bin/spanx +7 -0
  11. data/conf/spanx-config.yml.example +44 -0
  12. data/conf/spanx-whitelist.txt.example +2 -0
  13. data/lib/spanx.rb +38 -0
  14. data/lib/spanx/actor/analyzer.rb +94 -0
  15. data/lib/spanx/actor/collector.rb +64 -0
  16. data/lib/spanx/actor/log_reader.rb +46 -0
  17. data/lib/spanx/actor/writer.rb +68 -0
  18. data/lib/spanx/cli.rb +47 -0
  19. data/lib/spanx/cli/analyze.rb +50 -0
  20. data/lib/spanx/cli/disable.rb +36 -0
  21. data/lib/spanx/cli/enable.rb +36 -0
  22. data/lib/spanx/cli/flush.rb +36 -0
  23. data/lib/spanx/cli/watch.rb +91 -0
  24. data/lib/spanx/config.rb +45 -0
  25. data/lib/spanx/helper.rb +8 -0
  26. data/lib/spanx/helper/exit.rb +11 -0
  27. data/lib/spanx/helper/subclassing.rb +31 -0
  28. data/lib/spanx/helper/timing.rb +9 -0
  29. data/lib/spanx/ip_checker.rb +5 -0
  30. data/lib/spanx/logger.rb +47 -0
  31. data/lib/spanx/notifier/audit_log.rb +18 -0
  32. data/lib/spanx/notifier/base.rb +22 -0
  33. data/lib/spanx/notifier/campfire.rb +47 -0
  34. data/lib/spanx/notifier/email.rb +61 -0
  35. data/lib/spanx/runner.rb +74 -0
  36. data/lib/spanx/usage.rb +9 -0
  37. data/lib/spanx/version.rb +3 -0
  38. data/lib/spanx/whitelist.rb +31 -0
  39. data/spanx.gemspec +32 -0
  40. data/spec/fixtures/access.log.1 +104 -0
  41. data/spec/fixtures/access.log.bots +7 -0
  42. data/spec/fixtures/config.yml +10 -0
  43. data/spec/fixtures/config_with_checks.yml +18 -0
  44. data/spec/fixtures/whitelist.txt +4 -0
  45. data/spec/spanx/actor/analyzer_spec.rb +114 -0
  46. data/spec/spanx/actor/collector_spec.rb +4 -0
  47. data/spec/spanx/actor/log_reader_spec.rb +68 -0
  48. data/spec/spanx/actor/writer_spec.rb +63 -0
  49. data/spec/spanx/config_spec.rb +62 -0
  50. data/spec/spanx/helper/timing_spec.rb +22 -0
  51. data/spec/spanx/notifier/base_spec.rb +16 -0
  52. data/spec/spanx/notifier/campfire_spec.rb +5 -0
  53. data/spec/spanx/notifier/email_spec.rb +121 -0
  54. data/spec/spanx/runner_spec.rb +102 -0
  55. data/spec/spanx/whitelist_spec.rb +66 -0
  56. data/spec/spec_helper.rb +25 -0
  57. data/spec/support/fakeredis.rb +1 -0
  58. data/spec/support/mail.rb +10 -0
  59. metadata +302 -0
@@ -0,0 +1,45 @@
1
+ require 'yaml'
2
+ require 'pause'
3
+ require 'spanx/helper/exit'
4
+ require 'spanx/ip_checker'
5
+
6
+ module Spanx
7
+ class Config < Hash
8
+ include Spanx::Helper::Exit
9
+
10
+ attr_accessor :filename
11
+
12
+ def initialize(filename)
13
+ super
14
+ @filename = filename
15
+ load_file
16
+
17
+ Pause.configure do |pause|
18
+ pause.redis_host = self[:redis][:host]
19
+ pause.redis_port = self[:redis][:port]
20
+ pause.redis_db = self[:redis][:database]
21
+
22
+ pause.resolution = self[:collector][:resolution]
23
+ pause.history = self[:collector][:history]
24
+ end
25
+
26
+ if self.has_key?(:analyzer) && self[:analyzer].has_key?(:period_checks)
27
+ self[:analyzer][:period_checks].each do |check|
28
+ Spanx::IPChecker.check check[:period_seconds].to_i, check[:max_allowed].to_i, check[:block_ttl].to_i
29
+ end
30
+ end
31
+
32
+ self
33
+ end
34
+
35
+ private
36
+
37
+ def load_file
38
+ begin
39
+ self.merge! ::YAML.load_file(filename)
40
+ rescue Errno::ENOENT
41
+ error_exit_with_msg "Unable to find config_file at #{filename}"
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,8 @@
1
+ require 'spanx/helper/timing'
2
+ require 'spanx/helper/exit'
3
+ require 'spanx/helper/subclassing'
4
+
5
+ module Spanx
6
+ module Helper
7
+ end
8
+ end
@@ -0,0 +1,11 @@
1
+ module Spanx
2
+ module Helper
3
+ module Exit
4
+ def error_exit_with_msg(msg)
5
+ $stderr.puts "Error: #{msg}"
6
+ $stderr.puts Spanx::USAGE
7
+ exit 1
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,31 @@
1
+ module Spanx
2
+ module Helper
3
+ module Subclassing
4
+
5
+ def self.included(base)
6
+ base.extend ClassMethods
7
+ end
8
+
9
+ module ClassMethods
10
+
11
+ def subclasses
12
+ @@subclasses ||= {}
13
+ end
14
+
15
+ def subclass_name
16
+ name.split("::").last.downcase
17
+ end
18
+
19
+ def subclass_class(subclass)
20
+ subclasses[subclass]
21
+ end
22
+
23
+ private
24
+
25
+ def inherited(subclass)
26
+ subclasses[subclass.subclass_name] = subclass
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,9 @@
1
+ module Spanx
2
+ module Helper
3
+ module Timing
4
+ def period_marker(resolution, timestamp = Time.now)
5
+ timestamp.to_i / resolution * resolution
6
+ end
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,5 @@
1
+ require 'pause'
2
+
3
+ class Spanx::IPChecker < Pause::Action
4
+ scope "spanx:ip"
5
+ end
@@ -0,0 +1,47 @@
1
+ module Spanx
2
+ module Logger
3
+
4
+ class << self
5
+ def enable
6
+ class << self
7
+ self.send(:define_method, :log, proc { |msg| _log(msg) })
8
+ self.send(:define_method, :logging, proc { |msg, &block| _logging(msg, &block) })
9
+ end
10
+ end
11
+
12
+ def disable
13
+ class << self
14
+ self.send(:define_method, :log, proc { |msg|})
15
+ self.send(:define_method, :logging, proc { |msg, &block| block.call })
16
+ end
17
+ end
18
+
19
+ def log(msg)
20
+ end
21
+
22
+ def logging(msg, &block)
23
+ block.call
24
+ end
25
+
26
+ private
27
+
28
+ def _log(msg)
29
+ puts "#{Time.now}: #{sprintf("%-20s", Thread.current[:name])} - #{msg}"
30
+ end
31
+
32
+ def _logging(message, &block)
33
+ start = Time.now
34
+ returned_from_block = yield
35
+ elapsed_time = Time.now - start
36
+ if returned_from_block.is_a?(String) && returned_from_block != ""
37
+ message += " - #{returned_from_block}"
38
+ end
39
+ log "(#{"%9.2f" % (1000 * elapsed_time)}ms) #{message}"
40
+ returned_from_block
41
+ rescue Exception => e
42
+ elapsed_time = Time.now - start
43
+ log "(#{"%9.2f" % (1000 * elapsed_time)}ms) error: #{e.message} for #{message} "
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,18 @@
1
+ module Spanx
2
+ module Notifier
3
+ class AuditLog < Base
4
+ attr_accessor :audit_file
5
+
6
+ def initialize(config)
7
+ @audit_file = config[:audit_file]
8
+ raise ArgumentError.new("config[:audit_file] is required for this notifier to work") unless @audit_file
9
+ end
10
+
11
+ def publish(b)
12
+ File.open(@audit_file, "a") do |file|
13
+ file.puts "#{Time.now} -- #{sprintf("%-16s", b.identifier)} period=#{b.period_check.period_seconds}s max=#{b.period_check.max_allowed} count=#{b.sum} ttl=#{b.period_check.block_ttl}s"
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,22 @@
1
+ module Spanx
2
+ module Notifier
3
+ class Base
4
+
5
+ # Takes an instance of the Spanx::BlockedIp struct.
6
+ # Overwrite this a subclass to define real behavior
7
+ def publish(blocked_ip)
8
+ raise 'Abstract Method Not Implemented'
9
+ end
10
+
11
+ protected
12
+
13
+ def generate_block_ip_message(blocked_ip)
14
+ violated_period = blocked_ip.period_check
15
+ "#{blocked_ip.identifier} blocked @ #{Time.at(blocked_ip.timestamp)} " \
16
+ "for #{violated_period.block_ttl/60}mins, for #{blocked_ip.sum} requests over " \
17
+ "#{violated_period.period_seconds/60}mins, with #{violated_period.max_allowed} allowed."
18
+ end
19
+
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,47 @@
1
+ require 'tinder'
2
+ module Spanx
3
+ module Notifier
4
+
5
+ # Notify Campfire room of a new IP blocked
6
+ class Campfire < Base
7
+
8
+ attr_accessor :account, :room_id, :token
9
+
10
+ def initialize(config)
11
+ @enabled = config[:campfire][:enabled]
12
+ if self.enabled?
13
+ _init(config[:campfire][:account], config[:campfire][:room_id], config[:campfire][:token])
14
+ end
15
+ end
16
+
17
+ def publish(blocked_ip)
18
+ speak generate_block_ip_message(blocked_ip) if enabled?
19
+ end
20
+
21
+ def _init(account, room_id, token)
22
+ @account = account
23
+ @room_id = room_id
24
+ @token = token
25
+ end
26
+
27
+ def enabled?
28
+ @enabled
29
+ end
30
+
31
+ private
32
+
33
+ def campfire
34
+ @campfire ||= Tinder::Campfire.new(account, :token => token)
35
+ end
36
+
37
+ def room
38
+ campfire.find_room_by_id(room_id)
39
+ end
40
+
41
+ def speak message
42
+ r = room
43
+ r.speak message if r
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,61 @@
1
+ require 'mail'
2
+
3
+ module Spanx
4
+ module Notifier
5
+ class Email < Base
6
+
7
+ attr_reader :config, :thread
8
+
9
+ def initialize(config)
10
+ @config = config[:email]
11
+
12
+ configure_email_gateway
13
+ end
14
+
15
+ def publish(blocked_ip)
16
+ return unless enabled?
17
+
18
+ @thread = Thread.new do
19
+ Thread.current[:name] = "email notifier"
20
+ Logger.log "sending notification email for #{blocked_ip.identifier}"
21
+
22
+ mail = Mail.new
23
+ mail.to = config[:to]
24
+ mail.from = config[:from]
25
+ mail.subject = subject(blocked_ip)
26
+ mail.body = generate_block_ip_message(blocked_ip)
27
+
28
+ mail.deliver
29
+ end
30
+ end
31
+
32
+ def enabled?
33
+ config && config[:enabled]
34
+ end
35
+
36
+ private
37
+
38
+ def subject(blocked_ip)
39
+ "#{config[:subject] || "IP Blocked:"} #{blocked_ip.identifier}"
40
+ end
41
+
42
+ def configure_email_gateway
43
+ return unless enabled?
44
+
45
+ Mail.defaults do
46
+ delivery_method :smtp, {}
47
+ end
48
+
49
+ settings = Mail::Configuration.instance.delivery_method.settings
50
+ settings[:address] = config[:gateway]
51
+ settings[:port] = '587'
52
+ settings[:domain] = config[:domain]
53
+ settings[:user_name] = config[:from]
54
+ settings[:password] = config[:password]
55
+ settings[:authentication] = :plain
56
+ settings[:enable_starttls_auto] = true
57
+ settings[:openssl_verify_mode] = 'none'
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,74 @@
1
+ require 'mixlib/cli'
2
+ require 'spanx/logger'
3
+ require 'spanx/actor/collector'
4
+ require 'spanx/actor/analyzer'
5
+ require 'spanx/actor/log_reader'
6
+ require 'spanx/actor/writer'
7
+ require 'spanx/whitelist'
8
+ require 'thread'
9
+
10
+ # Spanx::Runner is initialized with a list of actors to run
11
+ # and a config hash. It is then run to activate each actor
12
+ # and join one of the running threads.
13
+ #
14
+ # Example:
15
+ # Spanx::Runner.new("analyzer", {}).run
16
+ # Spanx::Runner.new("analyzer", "writer", {}).run
17
+ #
18
+ # Valid actors are:
19
+ # collector
20
+ # analyzer
21
+ # writer
22
+ # log_reader
23
+ #
24
+ module Spanx
25
+ class Runner
26
+ attr_accessor :config, :queue, :actors
27
+
28
+ def initialize(*args)
29
+ @config = args.last.is_a?(Hash) ? args.pop : {}
30
+ @queue = Queue.new
31
+ validate_args!(args)
32
+ @actors = args.map { |actor| self.send(actor.to_sym) }
33
+
34
+ Daemonize.daemonize if config[:daemonize]
35
+
36
+ STDOUT.sync = true if config[:debug]
37
+ end
38
+
39
+ def run
40
+ threads = actors.map(&:run)
41
+ threads.last.join
42
+ end
43
+
44
+ # actors
45
+
46
+ def collector
47
+ @collector ||= Spanx::Actor::Collector.new(config, queue)
48
+ end
49
+
50
+ def log_reader
51
+ @log_reader ||= Spanx::Actor::LogReader.new(config[:access_log], queue, config[:log_reader][:tail_interval], whitelist)
52
+ end
53
+
54
+ def writer
55
+ @writer ||= Spanx::Actor::Writer.new(config)
56
+ end
57
+
58
+ def analyzer
59
+ @analyzer ||= Spanx::Actor::Analyzer.new(config)
60
+ end
61
+
62
+ # helpers
63
+
64
+ def whitelist
65
+ @whitelist ||= Spanx::Whitelist.new(config[:whitelist_file])
66
+ end
67
+
68
+ private
69
+
70
+ def validate_args!(args)
71
+ raise("Invalid actor") unless (args - %w[collector log_reader writer analyzer]).empty?
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,9 @@
1
+ module Spanx
2
+ USAGE = %q{Usage: spanx command [options]
3
+ watch -- Watch a server log file and write out a block list file
4
+ analyze -- Analyze IP traffic and save blocked IPs into Redis
5
+ flush -- Remove all IP blocks and delete previous tracking of that IP
6
+ disable -- Disable IP blocking
7
+ enable -- Enable IP blocking if disabled
8
+ }
9
+ end
@@ -0,0 +1,3 @@
1
+ module Spanx
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,31 @@
1
+ require 'spanx/helper/exit'
2
+
3
+ module Spanx
4
+ class Whitelist
5
+ include Spanx::Helper::Exit
6
+ attr_accessor :patterns, :filename
7
+
8
+ def initialize(filename)
9
+ @patterns = []
10
+ @filename = filename
11
+
12
+ load_file
13
+ end
14
+
15
+ def match?(line)
16
+ @patterns.any? do |p|
17
+ p.match(line)
18
+ end
19
+ end
20
+
21
+ def load_file
22
+ if filename
23
+ begin
24
+ @patterns = ::File.readlines(filename).reject{|line| line =~ /^#/}.map{|p| %r{#{p.chomp()}} }
25
+ rescue Errno::ENOENT
26
+ error_exit_with_msg("Unable to find whitelist file at #{filename}")
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,32 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require File.expand_path('../lib/spanx/version', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.authors = ["Konstantin Gredeskoul","Eric Saxby"]
6
+ gem.email = %w(kigster@gmail.com sax@livinginthepast.org)
7
+ gem.description = %q{Real time IP parsing and rate detection gem for access_log files}
8
+ gem.summary = %q{Real time IP parsing and rate detection gem for access_log files}
9
+ gem.homepage = "https://github.com/wanelo/spanx"
10
+
11
+ gem.files = `git ls-files`.split($\)
12
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
13
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
+ gem.name = "spanx"
15
+ gem.require_paths = %w(lib)
16
+ gem.version = Spanx::VERSION
17
+
18
+ gem.add_dependency 'pause', '~> 0.0.3'
19
+ gem.add_dependency 'file-tail'
20
+ gem.add_dependency 'mixlib-cli'
21
+ gem.add_dependency 'daemons'
22
+ gem.add_dependency 'tinder'
23
+ gem.add_dependency 'mail', '~> 2.4.4'
24
+
25
+ gem.add_development_dependency 'rspec'
26
+ gem.add_development_dependency 'fakeredis'
27
+ gem.add_development_dependency 'timecop'
28
+
29
+ gem.add_development_dependency 'guard-rspec'
30
+ gem.add_development_dependency 'rb-fsevent'
31
+
32
+ end