spanx 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +20 -0
- data/.pairs +13 -0
- data/.rspec +2 -0
- data/.rvmrc +1 -0
- data/Gemfile +4 -0
- data/Guardfile +13 -0
- data/LICENSE +22 -0
- data/README.md +175 -0
- data/Rakefile +2 -0
- data/bin/spanx +7 -0
- data/conf/spanx-config.yml.example +44 -0
- data/conf/spanx-whitelist.txt.example +2 -0
- data/lib/spanx.rb +38 -0
- data/lib/spanx/actor/analyzer.rb +94 -0
- data/lib/spanx/actor/collector.rb +64 -0
- data/lib/spanx/actor/log_reader.rb +46 -0
- data/lib/spanx/actor/writer.rb +68 -0
- data/lib/spanx/cli.rb +47 -0
- data/lib/spanx/cli/analyze.rb +50 -0
- data/lib/spanx/cli/disable.rb +36 -0
- data/lib/spanx/cli/enable.rb +36 -0
- data/lib/spanx/cli/flush.rb +36 -0
- data/lib/spanx/cli/watch.rb +91 -0
- data/lib/spanx/config.rb +45 -0
- data/lib/spanx/helper.rb +8 -0
- data/lib/spanx/helper/exit.rb +11 -0
- data/lib/spanx/helper/subclassing.rb +31 -0
- data/lib/spanx/helper/timing.rb +9 -0
- data/lib/spanx/ip_checker.rb +5 -0
- data/lib/spanx/logger.rb +47 -0
- data/lib/spanx/notifier/audit_log.rb +18 -0
- data/lib/spanx/notifier/base.rb +22 -0
- data/lib/spanx/notifier/campfire.rb +47 -0
- data/lib/spanx/notifier/email.rb +61 -0
- data/lib/spanx/runner.rb +74 -0
- data/lib/spanx/usage.rb +9 -0
- data/lib/spanx/version.rb +3 -0
- data/lib/spanx/whitelist.rb +31 -0
- data/spanx.gemspec +32 -0
- data/spec/fixtures/access.log.1 +104 -0
- data/spec/fixtures/access.log.bots +7 -0
- data/spec/fixtures/config.yml +10 -0
- data/spec/fixtures/config_with_checks.yml +18 -0
- data/spec/fixtures/whitelist.txt +4 -0
- data/spec/spanx/actor/analyzer_spec.rb +114 -0
- data/spec/spanx/actor/collector_spec.rb +4 -0
- data/spec/spanx/actor/log_reader_spec.rb +68 -0
- data/spec/spanx/actor/writer_spec.rb +63 -0
- data/spec/spanx/config_spec.rb +62 -0
- data/spec/spanx/helper/timing_spec.rb +22 -0
- data/spec/spanx/notifier/base_spec.rb +16 -0
- data/spec/spanx/notifier/campfire_spec.rb +5 -0
- data/spec/spanx/notifier/email_spec.rb +121 -0
- data/spec/spanx/runner_spec.rb +102 -0
- data/spec/spanx/whitelist_spec.rb +66 -0
- data/spec/spec_helper.rb +25 -0
- data/spec/support/fakeredis.rb +1 -0
- data/spec/support/mail.rb +10 -0
- metadata +302 -0
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'pause'
|
2
|
+
require 'spanx/logger'
|
3
|
+
require 'spanx/helper/timing'
|
4
|
+
|
5
|
+
module Spanx
|
6
|
+
module Actor
|
7
|
+
class Collector
|
8
|
+
include Spanx::Helper::Timing
|
9
|
+
|
10
|
+
attr_accessor :queue, :config, :semaphore, :cache
|
11
|
+
|
12
|
+
def initialize(config, queue)
|
13
|
+
@queue = queue
|
14
|
+
@config = config
|
15
|
+
@semaphore = Mutex.new
|
16
|
+
@cache = Hash.new(0)
|
17
|
+
end
|
18
|
+
|
19
|
+
def run
|
20
|
+
Thread.new do
|
21
|
+
Thread.current[:name] = "collector:queue"
|
22
|
+
loop do
|
23
|
+
unless queue.empty?
|
24
|
+
Logger.logging "caching [#{queue.size}] keys locally" do
|
25
|
+
while !queue.empty?
|
26
|
+
semaphore.synchronize {
|
27
|
+
increment_ip *(queue.pop)
|
28
|
+
}
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
sleep 1
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
Thread.new do
|
37
|
+
Thread.current[:name] = "collector:flush"
|
38
|
+
loop do
|
39
|
+
semaphore.synchronize {
|
40
|
+
Logger.logging "flushing cache with [#{cache.keys.size}] keys" do
|
41
|
+
cache.each_pair do |key, count|
|
42
|
+
Spanx::IPChecker.new(key[0]).increment!(key[1], count)
|
43
|
+
end
|
44
|
+
reset_cache
|
45
|
+
end
|
46
|
+
}
|
47
|
+
sleep config[:collector][:flush_interval]
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def increment_ip(ip, timestamp)
|
53
|
+
cache[[ip, period_marker(config[:collector][:resolution], timestamp)]] += 1
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def reset_cache
|
59
|
+
@cache.clear
|
60
|
+
GC.start
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'file-tail'
|
2
|
+
|
3
|
+
module Spanx
|
4
|
+
module Actor
|
5
|
+
class LogReader
|
6
|
+
attr_accessor :file, :queue, :whitelist
|
7
|
+
|
8
|
+
def initialize file, queue, interval = 1, whitelist = nil
|
9
|
+
@file = Spanx::Actor::File.new(file)
|
10
|
+
@file.interval = interval
|
11
|
+
@file.backward(0)
|
12
|
+
@whitelist = whitelist
|
13
|
+
@queue = queue
|
14
|
+
end
|
15
|
+
|
16
|
+
def run
|
17
|
+
Thread.new do
|
18
|
+
Thread.current[:name] = "log_reader"
|
19
|
+
Logger.log "tailing the log file #{file.path}...."
|
20
|
+
self.read do |line|
|
21
|
+
queue << [line, Time.now.to_i ] if line
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def read &block
|
27
|
+
@file.tail do |line|
|
28
|
+
block.call(extract_ip(line)) unless whitelist && whitelist.match?(line)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def close
|
33
|
+
(@file.close if @file) rescue nil
|
34
|
+
end
|
35
|
+
|
36
|
+
def extract_ip line
|
37
|
+
matchers = line.match(/^((\d{1,3}\.?){4})/)
|
38
|
+
matchers[1] unless matchers.nil?
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class File < ::File
|
43
|
+
include ::File::Tail
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'spanx/logger'
|
2
|
+
require 'spanx/helper/exit'
|
3
|
+
|
4
|
+
module Spanx
|
5
|
+
module Actor
|
6
|
+
class Writer
|
7
|
+
include Spanx::Helper::Exit
|
8
|
+
|
9
|
+
attr_accessor :config
|
10
|
+
|
11
|
+
def initialize config
|
12
|
+
@config = config
|
13
|
+
@block_file = config[:block_file]
|
14
|
+
@run_command = config[:run_command]
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
Thread.new do
|
19
|
+
Thread.current[:name] = "writer"
|
20
|
+
loop do
|
21
|
+
self.write
|
22
|
+
sleep config[:writer][:write_interval]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def write
|
28
|
+
if Spanx::IPChecker.enabled?
|
29
|
+
ips = Spanx::IPChecker.blocked_identifiers
|
30
|
+
else
|
31
|
+
Logger.log "writing empty block file due to disabled state"
|
32
|
+
ips = []
|
33
|
+
end
|
34
|
+
|
35
|
+
begin
|
36
|
+
contents_previous = File.read(@block_file) rescue nil
|
37
|
+
Logger.logging "writing out [#{ips.size}] IP block rules to [#{@block_file}]" do
|
38
|
+
File.open(@block_file, "w") do |file|
|
39
|
+
ips.sort.each do |ip|
|
40
|
+
# TODO: make this a customizable ERB template
|
41
|
+
file.puts("deny #{ip};")
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
contents_now = File.read(@block_file)
|
46
|
+
if contents_now != contents_previous && @run_command
|
47
|
+
Logger.logging "running command [#{@run_command}]" do
|
48
|
+
run_command
|
49
|
+
end
|
50
|
+
end
|
51
|
+
rescue Exception => e
|
52
|
+
error_exit_with_msg "ERROR writing to block file #{@block_file} or running command: #{e.inspect}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def run_command
|
57
|
+
result = system(@run_command)
|
58
|
+
if result
|
59
|
+
"executed successfully"
|
60
|
+
elsif result == false
|
61
|
+
"returned non-zero exit status"
|
62
|
+
elsif result.nil?
|
63
|
+
"failed -- #{$?}"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
data/lib/spanx/cli.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'mixlib/cli'
|
2
|
+
require 'spanx/helper/exit'
|
3
|
+
|
4
|
+
module Spanx
|
5
|
+
class CLI
|
6
|
+
include Mixlib::CLI
|
7
|
+
include Spanx::Helper::Exit
|
8
|
+
include Spanx::Helper::Subclassing
|
9
|
+
|
10
|
+
attr_reader :args
|
11
|
+
|
12
|
+
# the first element of ARGV should be a subcommand, which maps to
|
13
|
+
# a class in spanx/cli/
|
14
|
+
def run(args = ARGV)
|
15
|
+
@args = args
|
16
|
+
validate!
|
17
|
+
Spanx::CLI.subclass_class(args.shift).new.run(args)
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def validate!
|
23
|
+
error_exit_with_msg("No command given") if args.empty?
|
24
|
+
@command = args.first
|
25
|
+
error_exit_with_msg("No command found matching #{@command}") unless Spanx::CLI.subclasses.include?(@command)
|
26
|
+
end
|
27
|
+
|
28
|
+
def generate_config(argv)
|
29
|
+
parse_options argv
|
30
|
+
config.merge! Spanx::Config.new(config[:config_file])
|
31
|
+
parse_options argv
|
32
|
+
|
33
|
+
if config[:debug]
|
34
|
+
STDOUT.sync = true
|
35
|
+
end
|
36
|
+
|
37
|
+
Spanx::Logger.enable if config[:debug]
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
require 'spanx/cli/watch'
|
44
|
+
require 'spanx/cli/analyze'
|
45
|
+
require 'spanx/cli/disable'
|
46
|
+
require 'spanx/cli/enable'
|
47
|
+
require 'spanx/cli/flush'
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require 'mixlib/cli'
|
3
|
+
require 'daemons/daemonize'
|
4
|
+
require 'spanx/logger'
|
5
|
+
require 'spanx/runner'
|
6
|
+
|
7
|
+
class Spanx::CLI::Analyze < Spanx::CLI
|
8
|
+
|
9
|
+
banner "Usage: spanx analyze [options]"
|
10
|
+
|
11
|
+
option :daemonize,
|
12
|
+
:short => "-d",
|
13
|
+
:long => "--daemonize",
|
14
|
+
:boolean => true,
|
15
|
+
:default => false
|
16
|
+
|
17
|
+
option :config_file,
|
18
|
+
:short => '-c CONFIG',
|
19
|
+
:long => '--config CONFIG',
|
20
|
+
:description => 'Path to config file (YML)',
|
21
|
+
:required => true
|
22
|
+
|
23
|
+
option :debug,
|
24
|
+
:short => '-g',
|
25
|
+
:long => '--debug',
|
26
|
+
:description => 'Log status to STDOUT',
|
27
|
+
:boolean => true,
|
28
|
+
:required => false,
|
29
|
+
:default => false
|
30
|
+
|
31
|
+
option :audit_file,
|
32
|
+
:short => '-a AUDIT',
|
33
|
+
:long => '--audit AUDIT_FILE',
|
34
|
+
:description => 'Historical record of IP blocking decisions',
|
35
|
+
:required => false
|
36
|
+
|
37
|
+
option :help,
|
38
|
+
:short => "-h",
|
39
|
+
:long => "--help",
|
40
|
+
:description => "Show this message",
|
41
|
+
:on => :tail,
|
42
|
+
:boolean => true,
|
43
|
+
:show_options => true,
|
44
|
+
:exit => 0
|
45
|
+
|
46
|
+
def run(argv = ARGV)
|
47
|
+
generate_config(argv)
|
48
|
+
Spanx::Runner.new("analyzer", config).run
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'mixlib/cli'
|
2
|
+
require 'spanx/logger'
|
3
|
+
|
4
|
+
class Spanx::CLI::Disable < Spanx::CLI
|
5
|
+
|
6
|
+
banner "Usage: spanx disable [options]"
|
7
|
+
|
8
|
+
option :config_file,
|
9
|
+
:short => '-c CONFIG',
|
10
|
+
:long => '--config CONFIG',
|
11
|
+
:description => 'Path to config file (YML)',
|
12
|
+
:required => true
|
13
|
+
|
14
|
+
option :debug,
|
15
|
+
:short => '-g',
|
16
|
+
:long => '--debug',
|
17
|
+
:description => 'Log to STDOUT status of execution and some time metrics',
|
18
|
+
:boolean => true,
|
19
|
+
:required => false,
|
20
|
+
:default => false
|
21
|
+
|
22
|
+
option :help,
|
23
|
+
:short => "-h",
|
24
|
+
:long => "--help",
|
25
|
+
:description => "Show this message",
|
26
|
+
:on => :tail,
|
27
|
+
:boolean => true,
|
28
|
+
:show_options => true,
|
29
|
+
:exit => 0
|
30
|
+
|
31
|
+
|
32
|
+
def run(argv = ARGV)
|
33
|
+
generate_config(argv)
|
34
|
+
Spanx::IPChecker.disable
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'mixlib/cli'
|
2
|
+
require 'spanx/logger'
|
3
|
+
|
4
|
+
class Spanx::CLI::Enable < Spanx::CLI
|
5
|
+
|
6
|
+
banner "Usage: spanx enable [options]"
|
7
|
+
|
8
|
+
option :config_file,
|
9
|
+
:short => '-c CONFIG',
|
10
|
+
:long => '--config CONFIG',
|
11
|
+
:description => 'Path to config file (YML)',
|
12
|
+
:required => true
|
13
|
+
|
14
|
+
option :debug,
|
15
|
+
:short => '-g',
|
16
|
+
:long => '--debug',
|
17
|
+
:description => 'Log to STDOUT status of execution and some time metrics',
|
18
|
+
:boolean => true,
|
19
|
+
:required => false,
|
20
|
+
:default => false
|
21
|
+
|
22
|
+
option :help,
|
23
|
+
:short => "-h",
|
24
|
+
:long => "--help",
|
25
|
+
:description => "Show this message",
|
26
|
+
:on => :tail,
|
27
|
+
:boolean => true,
|
28
|
+
:show_options => true,
|
29
|
+
:exit => 0
|
30
|
+
|
31
|
+
|
32
|
+
def run(argv = ARGV)
|
33
|
+
generate_config(argv)
|
34
|
+
Spanx::IPChecker.enable
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'mixlib/cli'
|
2
|
+
require 'spanx/logger'
|
3
|
+
|
4
|
+
class Spanx::CLI::Flush < Spanx::CLI
|
5
|
+
|
6
|
+
banner "Usage: spanx flush [options]"
|
7
|
+
|
8
|
+
option :config_file,
|
9
|
+
:short => '-c CONFIG',
|
10
|
+
:long => '--config CONFIG',
|
11
|
+
:description => 'Path to config file (YML)',
|
12
|
+
:required => true
|
13
|
+
|
14
|
+
option :debug,
|
15
|
+
:short => '-g',
|
16
|
+
:long => '--debug',
|
17
|
+
:description => 'Log to STDOUT status of execution and some time metrics',
|
18
|
+
:boolean => true,
|
19
|
+
:required => false,
|
20
|
+
:default => false
|
21
|
+
|
22
|
+
option :help,
|
23
|
+
:short => "-h",
|
24
|
+
:long => "--help",
|
25
|
+
:description => "Show this message",
|
26
|
+
:on => :tail,
|
27
|
+
:boolean => true,
|
28
|
+
:show_options => true,
|
29
|
+
:exit => 0
|
30
|
+
|
31
|
+
|
32
|
+
def run(argv = ARGV)
|
33
|
+
generate_config(argv)
|
34
|
+
Spanx::IPChecker.unblock_all
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'mixlib/cli'
|
2
|
+
require 'thread'
|
3
|
+
require 'daemons/daemonize'
|
4
|
+
require 'spanx/runner'
|
5
|
+
|
6
|
+
class Spanx::CLI::Watch < Spanx::CLI
|
7
|
+
include Spanx::Helper::Exit
|
8
|
+
|
9
|
+
banner <<-EOF
|
10
|
+
Usage: spanx watch [options]
|
11
|
+
EOF
|
12
|
+
|
13
|
+
option :access_log,
|
14
|
+
:short => "-f ACCESS_LOG",
|
15
|
+
:long => "--file ACCESS_LOG",
|
16
|
+
:description => "Apache/nginx access log file to scan continuously",
|
17
|
+
:required => false
|
18
|
+
|
19
|
+
option :config_file,
|
20
|
+
:short => '-c CONFIG',
|
21
|
+
:long => '--config CONFIG',
|
22
|
+
:description => 'Path to config file (YML)',
|
23
|
+
:required => true
|
24
|
+
|
25
|
+
option :block_file,
|
26
|
+
:short => '-b BLOCK_FILE',
|
27
|
+
:long => '--block_file BLOCK_FILE',
|
28
|
+
:description => 'Output file to store NGINX block list',
|
29
|
+
:required => false
|
30
|
+
|
31
|
+
option :whitelist_file,
|
32
|
+
:short => '-w WHITELIST',
|
33
|
+
:long => '--whitelist WHITELIST',
|
34
|
+
:description => 'File with newline separated reg exps, to exclude lines from access log',
|
35
|
+
:required => false,
|
36
|
+
:default => nil
|
37
|
+
|
38
|
+
option :run_command,
|
39
|
+
:short => '-r <shell command>',
|
40
|
+
:long => '--run <shell command>',
|
41
|
+
:description => 'Shell command to run anytime blocked ip file changes, for example "sudo pkill -HUP nginx"',
|
42
|
+
:required => false
|
43
|
+
|
44
|
+
option :daemonize,
|
45
|
+
:short => "-d",
|
46
|
+
:long => "--daemonize",
|
47
|
+
:description => "Detach from TTY and run as a daemon",
|
48
|
+
:boolean => true,
|
49
|
+
:default => false
|
50
|
+
|
51
|
+
option :analyze,
|
52
|
+
:short => '-z',
|
53
|
+
:long => '--analyze',
|
54
|
+
:description => 'Analyze IPs also (as opposed to running `spanx analyze` in another process)',
|
55
|
+
:boolean => true,
|
56
|
+
:default => false
|
57
|
+
|
58
|
+
option :debug,
|
59
|
+
:short => '-g',
|
60
|
+
:long => '--debug',
|
61
|
+
:description => 'Log to STDOUT status of execution and some time metrics',
|
62
|
+
:boolean => true,
|
63
|
+
:required => false,
|
64
|
+
:default => false
|
65
|
+
|
66
|
+
option :help,
|
67
|
+
:short => "-h",
|
68
|
+
:long => "--help",
|
69
|
+
:description => "Show this message",
|
70
|
+
:on => :tail,
|
71
|
+
:boolean => true,
|
72
|
+
:show_options => true,
|
73
|
+
:exit => 0
|
74
|
+
|
75
|
+
def run(argv = ARGV)
|
76
|
+
generate_config(argv)
|
77
|
+
validate!
|
78
|
+
runners = %w(log_reader collector writer)
|
79
|
+
runners << "analyzer" if config[:analyze]
|
80
|
+
|
81
|
+
Spanx::Runner.new(*runners, config).run
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
def validate!
|
87
|
+
error_exit_with_msg("Could not find file. Use -f or set :file in config_file") unless config[:access_log] && File.exists?(config[:access_log])
|
88
|
+
error_exit_with_msg("-b block_file is required") unless config[:block_file]
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|