spanx 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +20 -0
- data/.pairs +13 -0
- data/.rspec +2 -0
- data/.rvmrc +1 -0
- data/Gemfile +4 -0
- data/Guardfile +13 -0
- data/LICENSE +22 -0
- data/README.md +175 -0
- data/Rakefile +2 -0
- data/bin/spanx +7 -0
- data/conf/spanx-config.yml.example +44 -0
- data/conf/spanx-whitelist.txt.example +2 -0
- data/lib/spanx.rb +38 -0
- data/lib/spanx/actor/analyzer.rb +94 -0
- data/lib/spanx/actor/collector.rb +64 -0
- data/lib/spanx/actor/log_reader.rb +46 -0
- data/lib/spanx/actor/writer.rb +68 -0
- data/lib/spanx/cli.rb +47 -0
- data/lib/spanx/cli/analyze.rb +50 -0
- data/lib/spanx/cli/disable.rb +36 -0
- data/lib/spanx/cli/enable.rb +36 -0
- data/lib/spanx/cli/flush.rb +36 -0
- data/lib/spanx/cli/watch.rb +91 -0
- data/lib/spanx/config.rb +45 -0
- data/lib/spanx/helper.rb +8 -0
- data/lib/spanx/helper/exit.rb +11 -0
- data/lib/spanx/helper/subclassing.rb +31 -0
- data/lib/spanx/helper/timing.rb +9 -0
- data/lib/spanx/ip_checker.rb +5 -0
- data/lib/spanx/logger.rb +47 -0
- data/lib/spanx/notifier/audit_log.rb +18 -0
- data/lib/spanx/notifier/base.rb +22 -0
- data/lib/spanx/notifier/campfire.rb +47 -0
- data/lib/spanx/notifier/email.rb +61 -0
- data/lib/spanx/runner.rb +74 -0
- data/lib/spanx/usage.rb +9 -0
- data/lib/spanx/version.rb +3 -0
- data/lib/spanx/whitelist.rb +31 -0
- data/spanx.gemspec +32 -0
- data/spec/fixtures/access.log.1 +104 -0
- data/spec/fixtures/access.log.bots +7 -0
- data/spec/fixtures/config.yml +10 -0
- data/spec/fixtures/config_with_checks.yml +18 -0
- data/spec/fixtures/whitelist.txt +4 -0
- data/spec/spanx/actor/analyzer_spec.rb +114 -0
- data/spec/spanx/actor/collector_spec.rb +4 -0
- data/spec/spanx/actor/log_reader_spec.rb +68 -0
- data/spec/spanx/actor/writer_spec.rb +63 -0
- data/spec/spanx/config_spec.rb +62 -0
- data/spec/spanx/helper/timing_spec.rb +22 -0
- data/spec/spanx/notifier/base_spec.rb +16 -0
- data/spec/spanx/notifier/campfire_spec.rb +5 -0
- data/spec/spanx/notifier/email_spec.rb +121 -0
- data/spec/spanx/runner_spec.rb +102 -0
- data/spec/spanx/whitelist_spec.rb +66 -0
- data/spec/spec_helper.rb +25 -0
- data/spec/support/fakeredis.rb +1 -0
- data/spec/support/mail.rb +10 -0
- metadata +302 -0
data/lib/spanx/config.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'pause'
|
3
|
+
require 'spanx/helper/exit'
|
4
|
+
require 'spanx/ip_checker'
|
5
|
+
|
6
|
+
module Spanx
|
7
|
+
class Config < Hash
|
8
|
+
include Spanx::Helper::Exit
|
9
|
+
|
10
|
+
attr_accessor :filename
|
11
|
+
|
12
|
+
def initialize(filename)
|
13
|
+
super
|
14
|
+
@filename = filename
|
15
|
+
load_file
|
16
|
+
|
17
|
+
Pause.configure do |pause|
|
18
|
+
pause.redis_host = self[:redis][:host]
|
19
|
+
pause.redis_port = self[:redis][:port]
|
20
|
+
pause.redis_db = self[:redis][:database]
|
21
|
+
|
22
|
+
pause.resolution = self[:collector][:resolution]
|
23
|
+
pause.history = self[:collector][:history]
|
24
|
+
end
|
25
|
+
|
26
|
+
if self.has_key?(:analyzer) && self[:analyzer].has_key?(:period_checks)
|
27
|
+
self[:analyzer][:period_checks].each do |check|
|
28
|
+
Spanx::IPChecker.check check[:period_seconds].to_i, check[:max_allowed].to_i, check[:block_ttl].to_i
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
self
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def load_file
|
38
|
+
begin
|
39
|
+
self.merge! ::YAML.load_file(filename)
|
40
|
+
rescue Errno::ENOENT
|
41
|
+
error_exit_with_msg "Unable to find config_file at #{filename}"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
data/lib/spanx/helper.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
module Spanx
|
2
|
+
module Helper
|
3
|
+
module Subclassing
|
4
|
+
|
5
|
+
def self.included(base)
|
6
|
+
base.extend ClassMethods
|
7
|
+
end
|
8
|
+
|
9
|
+
module ClassMethods
|
10
|
+
|
11
|
+
def subclasses
|
12
|
+
@@subclasses ||= {}
|
13
|
+
end
|
14
|
+
|
15
|
+
def subclass_name
|
16
|
+
name.split("::").last.downcase
|
17
|
+
end
|
18
|
+
|
19
|
+
def subclass_class(subclass)
|
20
|
+
subclasses[subclass]
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def inherited(subclass)
|
26
|
+
subclasses[subclass.subclass_name] = subclass
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/lib/spanx/logger.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
module Spanx
|
2
|
+
module Logger
|
3
|
+
|
4
|
+
class << self
|
5
|
+
def enable
|
6
|
+
class << self
|
7
|
+
self.send(:define_method, :log, proc { |msg| _log(msg) })
|
8
|
+
self.send(:define_method, :logging, proc { |msg, &block| _logging(msg, &block) })
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def disable
|
13
|
+
class << self
|
14
|
+
self.send(:define_method, :log, proc { |msg|})
|
15
|
+
self.send(:define_method, :logging, proc { |msg, &block| block.call })
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def log(msg)
|
20
|
+
end
|
21
|
+
|
22
|
+
def logging(msg, &block)
|
23
|
+
block.call
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def _log(msg)
|
29
|
+
puts "#{Time.now}: #{sprintf("%-20s", Thread.current[:name])} - #{msg}"
|
30
|
+
end
|
31
|
+
|
32
|
+
def _logging(message, &block)
|
33
|
+
start = Time.now
|
34
|
+
returned_from_block = yield
|
35
|
+
elapsed_time = Time.now - start
|
36
|
+
if returned_from_block.is_a?(String) && returned_from_block != ""
|
37
|
+
message += " - #{returned_from_block}"
|
38
|
+
end
|
39
|
+
log "(#{"%9.2f" % (1000 * elapsed_time)}ms) #{message}"
|
40
|
+
returned_from_block
|
41
|
+
rescue Exception => e
|
42
|
+
elapsed_time = Time.now - start
|
43
|
+
log "(#{"%9.2f" % (1000 * elapsed_time)}ms) error: #{e.message} for #{message} "
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Spanx
|
2
|
+
module Notifier
|
3
|
+
class AuditLog < Base
|
4
|
+
attr_accessor :audit_file
|
5
|
+
|
6
|
+
def initialize(config)
|
7
|
+
@audit_file = config[:audit_file]
|
8
|
+
raise ArgumentError.new("config[:audit_file] is required for this notifier to work") unless @audit_file
|
9
|
+
end
|
10
|
+
|
11
|
+
def publish(b)
|
12
|
+
File.open(@audit_file, "a") do |file|
|
13
|
+
file.puts "#{Time.now} -- #{sprintf("%-16s", b.identifier)} period=#{b.period_check.period_seconds}s max=#{b.period_check.max_allowed} count=#{b.sum} ttl=#{b.period_check.block_ttl}s"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Spanx
|
2
|
+
module Notifier
|
3
|
+
class Base
|
4
|
+
|
5
|
+
# Takes an instance of the Spanx::BlockedIp struct.
|
6
|
+
# Overwrite this a subclass to define real behavior
|
7
|
+
def publish(blocked_ip)
|
8
|
+
raise 'Abstract Method Not Implemented'
|
9
|
+
end
|
10
|
+
|
11
|
+
protected
|
12
|
+
|
13
|
+
def generate_block_ip_message(blocked_ip)
|
14
|
+
violated_period = blocked_ip.period_check
|
15
|
+
"#{blocked_ip.identifier} blocked @ #{Time.at(blocked_ip.timestamp)} " \
|
16
|
+
"for #{violated_period.block_ttl/60}mins, for #{blocked_ip.sum} requests over " \
|
17
|
+
"#{violated_period.period_seconds/60}mins, with #{violated_period.max_allowed} allowed."
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'tinder'
|
2
|
+
module Spanx
|
3
|
+
module Notifier
|
4
|
+
|
5
|
+
# Notify Campfire room of a new IP blocked
|
6
|
+
class Campfire < Base
|
7
|
+
|
8
|
+
attr_accessor :account, :room_id, :token
|
9
|
+
|
10
|
+
def initialize(config)
|
11
|
+
@enabled = config[:campfire][:enabled]
|
12
|
+
if self.enabled?
|
13
|
+
_init(config[:campfire][:account], config[:campfire][:room_id], config[:campfire][:token])
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def publish(blocked_ip)
|
18
|
+
speak generate_block_ip_message(blocked_ip) if enabled?
|
19
|
+
end
|
20
|
+
|
21
|
+
def _init(account, room_id, token)
|
22
|
+
@account = account
|
23
|
+
@room_id = room_id
|
24
|
+
@token = token
|
25
|
+
end
|
26
|
+
|
27
|
+
def enabled?
|
28
|
+
@enabled
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def campfire
|
34
|
+
@campfire ||= Tinder::Campfire.new(account, :token => token)
|
35
|
+
end
|
36
|
+
|
37
|
+
def room
|
38
|
+
campfire.find_room_by_id(room_id)
|
39
|
+
end
|
40
|
+
|
41
|
+
def speak message
|
42
|
+
r = room
|
43
|
+
r.speak message if r
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'mail'
|
2
|
+
|
3
|
+
module Spanx
|
4
|
+
module Notifier
|
5
|
+
class Email < Base
|
6
|
+
|
7
|
+
attr_reader :config, :thread
|
8
|
+
|
9
|
+
def initialize(config)
|
10
|
+
@config = config[:email]
|
11
|
+
|
12
|
+
configure_email_gateway
|
13
|
+
end
|
14
|
+
|
15
|
+
def publish(blocked_ip)
|
16
|
+
return unless enabled?
|
17
|
+
|
18
|
+
@thread = Thread.new do
|
19
|
+
Thread.current[:name] = "email notifier"
|
20
|
+
Logger.log "sending notification email for #{blocked_ip.identifier}"
|
21
|
+
|
22
|
+
mail = Mail.new
|
23
|
+
mail.to = config[:to]
|
24
|
+
mail.from = config[:from]
|
25
|
+
mail.subject = subject(blocked_ip)
|
26
|
+
mail.body = generate_block_ip_message(blocked_ip)
|
27
|
+
|
28
|
+
mail.deliver
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def enabled?
|
33
|
+
config && config[:enabled]
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def subject(blocked_ip)
|
39
|
+
"#{config[:subject] || "IP Blocked:"} #{blocked_ip.identifier}"
|
40
|
+
end
|
41
|
+
|
42
|
+
def configure_email_gateway
|
43
|
+
return unless enabled?
|
44
|
+
|
45
|
+
Mail.defaults do
|
46
|
+
delivery_method :smtp, {}
|
47
|
+
end
|
48
|
+
|
49
|
+
settings = Mail::Configuration.instance.delivery_method.settings
|
50
|
+
settings[:address] = config[:gateway]
|
51
|
+
settings[:port] = '587'
|
52
|
+
settings[:domain] = config[:domain]
|
53
|
+
settings[:user_name] = config[:from]
|
54
|
+
settings[:password] = config[:password]
|
55
|
+
settings[:authentication] = :plain
|
56
|
+
settings[:enable_starttls_auto] = true
|
57
|
+
settings[:openssl_verify_mode] = 'none'
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
data/lib/spanx/runner.rb
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'mixlib/cli'
|
2
|
+
require 'spanx/logger'
|
3
|
+
require 'spanx/actor/collector'
|
4
|
+
require 'spanx/actor/analyzer'
|
5
|
+
require 'spanx/actor/log_reader'
|
6
|
+
require 'spanx/actor/writer'
|
7
|
+
require 'spanx/whitelist'
|
8
|
+
require 'thread'
|
9
|
+
|
10
|
+
# Spanx::Runner is initialized with a list of actors to run
|
11
|
+
# and a config hash. It is then run to activate each actor
|
12
|
+
# and join one of the running threads.
|
13
|
+
#
|
14
|
+
# Example:
|
15
|
+
# Spanx::Runner.new("analyzer", {}).run
|
16
|
+
# Spanx::Runner.new("analyzer", "writer", {}).run
|
17
|
+
#
|
18
|
+
# Valid actors are:
|
19
|
+
# collector
|
20
|
+
# analyzer
|
21
|
+
# writer
|
22
|
+
# log_reader
|
23
|
+
#
|
24
|
+
module Spanx
|
25
|
+
class Runner
|
26
|
+
attr_accessor :config, :queue, :actors
|
27
|
+
|
28
|
+
def initialize(*args)
|
29
|
+
@config = args.last.is_a?(Hash) ? args.pop : {}
|
30
|
+
@queue = Queue.new
|
31
|
+
validate_args!(args)
|
32
|
+
@actors = args.map { |actor| self.send(actor.to_sym) }
|
33
|
+
|
34
|
+
Daemonize.daemonize if config[:daemonize]
|
35
|
+
|
36
|
+
STDOUT.sync = true if config[:debug]
|
37
|
+
end
|
38
|
+
|
39
|
+
def run
|
40
|
+
threads = actors.map(&:run)
|
41
|
+
threads.last.join
|
42
|
+
end
|
43
|
+
|
44
|
+
# actors
|
45
|
+
|
46
|
+
def collector
|
47
|
+
@collector ||= Spanx::Actor::Collector.new(config, queue)
|
48
|
+
end
|
49
|
+
|
50
|
+
def log_reader
|
51
|
+
@log_reader ||= Spanx::Actor::LogReader.new(config[:access_log], queue, config[:log_reader][:tail_interval], whitelist)
|
52
|
+
end
|
53
|
+
|
54
|
+
def writer
|
55
|
+
@writer ||= Spanx::Actor::Writer.new(config)
|
56
|
+
end
|
57
|
+
|
58
|
+
def analyzer
|
59
|
+
@analyzer ||= Spanx::Actor::Analyzer.new(config)
|
60
|
+
end
|
61
|
+
|
62
|
+
# helpers
|
63
|
+
|
64
|
+
def whitelist
|
65
|
+
@whitelist ||= Spanx::Whitelist.new(config[:whitelist_file])
|
66
|
+
end
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
def validate_args!(args)
|
71
|
+
raise("Invalid actor") unless (args - %w[collector log_reader writer analyzer]).empty?
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
data/lib/spanx/usage.rb
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
module Spanx
|
2
|
+
USAGE = %q{Usage: spanx command [options]
|
3
|
+
watch -- Watch a server log file and write out a block list file
|
4
|
+
analyze -- Analyze IP traffic and save blocked IPs into Redis
|
5
|
+
flush -- Remove all IP blocks and delete previous tracking of that IP
|
6
|
+
disable -- Disable IP blocking
|
7
|
+
enable -- Enable IP blocking if disabled
|
8
|
+
}
|
9
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'spanx/helper/exit'
|
2
|
+
|
3
|
+
module Spanx
|
4
|
+
class Whitelist
|
5
|
+
include Spanx::Helper::Exit
|
6
|
+
attr_accessor :patterns, :filename
|
7
|
+
|
8
|
+
def initialize(filename)
|
9
|
+
@patterns = []
|
10
|
+
@filename = filename
|
11
|
+
|
12
|
+
load_file
|
13
|
+
end
|
14
|
+
|
15
|
+
def match?(line)
|
16
|
+
@patterns.any? do |p|
|
17
|
+
p.match(line)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def load_file
|
22
|
+
if filename
|
23
|
+
begin
|
24
|
+
@patterns = ::File.readlines(filename).reject{|line| line =~ /^#/}.map{|p| %r{#{p.chomp()}} }
|
25
|
+
rescue Errno::ENOENT
|
26
|
+
error_exit_with_msg("Unable to find whitelist file at #{filename}")
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/spanx.gemspec
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/spanx/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.authors = ["Konstantin Gredeskoul","Eric Saxby"]
|
6
|
+
gem.email = %w(kigster@gmail.com sax@livinginthepast.org)
|
7
|
+
gem.description = %q{Real time IP parsing and rate detection gem for access_log files}
|
8
|
+
gem.summary = %q{Real time IP parsing and rate detection gem for access_log files}
|
9
|
+
gem.homepage = "https://github.com/wanelo/spanx"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.name = "spanx"
|
15
|
+
gem.require_paths = %w(lib)
|
16
|
+
gem.version = Spanx::VERSION
|
17
|
+
|
18
|
+
gem.add_dependency 'pause', '~> 0.0.3'
|
19
|
+
gem.add_dependency 'file-tail'
|
20
|
+
gem.add_dependency 'mixlib-cli'
|
21
|
+
gem.add_dependency 'daemons'
|
22
|
+
gem.add_dependency 'tinder'
|
23
|
+
gem.add_dependency 'mail', '~> 2.4.4'
|
24
|
+
|
25
|
+
gem.add_development_dependency 'rspec'
|
26
|
+
gem.add_development_dependency 'fakeredis'
|
27
|
+
gem.add_development_dependency 'timecop'
|
28
|
+
|
29
|
+
gem.add_development_dependency 'guard-rspec'
|
30
|
+
gem.add_development_dependency 'rb-fsevent'
|
31
|
+
|
32
|
+
end
|