ip-world-map 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +45 -0
- data/bin/ip-world-map +183 -0
- data/lib/ip-world-map.rb +14 -0
- data/lib/ip-world-map/apache_log_analyzer.rb +25 -0
- data/lib/ip-world-map/apache_log_visualizer.rb +75 -0
- data/lib/ip-world-map/decay.rb +12 -0
- data/lib/ip-world-map/file_utils.rb +16 -0
- data/lib/ip-world-map/information_drawer.rb +34 -0
- data/lib/ip-world-map/ip_lookup_service.rb +69 -0
- data/lib/ip-world-map/log_analyzer.rb +60 -0
- data/lib/ip-world-map/logfile_mock.rb +18 -0
- data/lib/ip-world-map/point_in_time.rb +23 -0
- data/lib/ip-world-map/visualization.rb +80 -0
- data/resources/maps/earthmap-1920x960.tif +0 -0
- data/spec/ip-world-map/decay.rb +18 -0
- data/spec/ip-world-map/ip_lookup_service.rb +37 -0
- data/spec/ip-world-map/time_format.rb +27 -0
- data/spec/spec_helper.rb +5 -0
- metadata +118 -0
data/Rakefile
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'rubygems'
|
3
|
+
require 'rubygems/package_task'
|
4
|
+
require 'rspec/core/rake_task'
|
5
|
+
|
6
|
+
RSpec::Core::RakeTask.new do |t|
|
7
|
+
t.pattern = 'spec/**/*.rb'
|
8
|
+
t.rcov = false
|
9
|
+
t.rcov_opts = %q[--exclude "spec"]
|
10
|
+
t.verbose = true
|
11
|
+
end
|
12
|
+
|
13
|
+
spec = Gem::Specification.new do |s|
|
14
|
+
s.platform = Gem::Platform::RUBY
|
15
|
+
s.required_ruby_version = '>= 1.8.7'
|
16
|
+
s.summary = 'A tool to generate images/videos of user locations based on Apache log files.'
|
17
|
+
s.name = 'ip-world-map'
|
18
|
+
s.version = '1.0.0'
|
19
|
+
s.license = 'GPL-2'
|
20
|
+
s.executables = ['ip-world-map']
|
21
|
+
s.files = FileList['{lib,spec}/**/*'].to_a +
|
22
|
+
['Rakefile'] +
|
23
|
+
['resources/maps/earthmap-1920x960.tif']
|
24
|
+
s.has_rdoc = false
|
25
|
+
s.description = s.summary
|
26
|
+
s.homepage = 'http://github.com/darxriggs/ip-world-map'
|
27
|
+
s.author = 'René Scheibe'
|
28
|
+
s.email = 'rene.scheibe@gmail.com'
|
29
|
+
|
30
|
+
s.requirements = ['ImageMagick (used by rmagick)', 'ffmpeg (only for animations)']
|
31
|
+
s.add_runtime_dependency('rmagick', '~> 2.13.1')
|
32
|
+
s.add_development_dependency('rspec', '~> 2.6.0')
|
33
|
+
end
|
34
|
+
|
35
|
+
Gem::PackageTask.new(spec) do |pkg|
|
36
|
+
pkg.need_tar = false
|
37
|
+
end
|
38
|
+
|
39
|
+
desc 'Install gem locally'
|
40
|
+
task :install_gem => :package do
|
41
|
+
`sudo gem install pkg/*.gem --no-ri --no-rdoc`
|
42
|
+
end
|
43
|
+
|
44
|
+
task :default => :install_gem
|
45
|
+
|
data/bin/ip-world-map
ADDED
@@ -0,0 +1,183 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'optparse'
|
5
|
+
require 'ostruct'
|
6
|
+
require 'date'
|
7
|
+
require 'ip-world-map'
|
8
|
+
require 'RMagick'
|
9
|
+
|
10
|
+
class App
|
11
|
+
VERSION = '0.0.1'
|
12
|
+
|
13
|
+
attr_reader :options
|
14
|
+
|
15
|
+
def initialize arguments, stdin
|
16
|
+
@arguments = arguments
|
17
|
+
@stdin = stdin
|
18
|
+
|
19
|
+
set_defaults
|
20
|
+
end
|
21
|
+
|
22
|
+
def run
|
23
|
+
if parsed_arguments? && arguments_valid?
|
24
|
+
output_options if @options.verbose
|
25
|
+
process_command
|
26
|
+
else
|
27
|
+
output_help
|
28
|
+
exit 1
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
protected
|
33
|
+
|
34
|
+
def set_defaults
|
35
|
+
@options = OpenStruct.new({
|
36
|
+
:verbose => false,
|
37
|
+
:quiet => false,
|
38
|
+
:map_filename => File.join(File.dirname(__FILE__), '..' , 'resources', 'maps', 'earthmap-1920x960.tif'),
|
39
|
+
:map_width => 800,
|
40
|
+
:map_height => 400,
|
41
|
+
:frames_per_second => 25,
|
42
|
+
:fill_dot_color => 'red',
|
43
|
+
:fill_dot_scale => 10,
|
44
|
+
:fill_dot_opacity => 1.0,
|
45
|
+
:fill_dot_lifetime => 15,
|
46
|
+
:time_format => nil,
|
47
|
+
:group_seconds => 24 * 1 * 60 * 60,
|
48
|
+
:output_format => 'png',
|
49
|
+
:animate => false
|
50
|
+
})
|
51
|
+
end
|
52
|
+
|
53
|
+
def parsed_arguments?
|
54
|
+
@opts = opts = OptionParser.new
|
55
|
+
|
56
|
+
opts.banner = 'Usage: ip-world-map [options] log_file1 [logfile2] ...'
|
57
|
+
|
58
|
+
opts.on('--version', 'Display the version') do
|
59
|
+
output_version
|
60
|
+
exit 0
|
61
|
+
end
|
62
|
+
|
63
|
+
opts.on('-h', '--help', 'Display this help message') do
|
64
|
+
output_help
|
65
|
+
exit 0
|
66
|
+
end
|
67
|
+
|
68
|
+
opts.on('-v', '--verbose', 'Verbose output') do
|
69
|
+
@options.verbose = true
|
70
|
+
end
|
71
|
+
|
72
|
+
opts.on('--map-filename VALUE', 'The image to use as background') do |filename|
|
73
|
+
raise 'invalid map file' unless File.readable? filename
|
74
|
+
@options.map_filename = filename
|
75
|
+
end
|
76
|
+
|
77
|
+
opts.on('--resolution VALUE', '(eg.: 640x480)') do |resolution|
|
78
|
+
raise 'invalid resolution format' unless resolution =~ /^[0-9]+x[0-9]+$/
|
79
|
+
width, height = resolution.split('x').collect{|v| v.to_i}
|
80
|
+
@options.map_width = width
|
81
|
+
@options.map_height = height
|
82
|
+
end
|
83
|
+
|
84
|
+
opts.on('--fps VALUE', 'Animation frames per second (eg.: 25)') do |fps|
|
85
|
+
raise 'invalid fps format' unless fps =~ /^[1-9][0-9]*$/
|
86
|
+
@options.frames_per_second = fps.to_i
|
87
|
+
end
|
88
|
+
|
89
|
+
opts.on('--fill-dot-color VALUE', "(eg.: red, 'rgb(255,0,0)', '#FF0000')") do |color|
|
90
|
+
Magick::Pixel.from_color color rescue raise 'invalid color (see help for examples)'
|
91
|
+
@options.fill_dot_color = color
|
92
|
+
end
|
93
|
+
|
94
|
+
opts.on('--fill-dot-scale VALUE', '(eg.: 10.0)') do |scale|
|
95
|
+
raise 'invalid dot scale' unless scale =~ /^[1-9][0-9]*([.][0-9]*)?$/
|
96
|
+
@options.fill_dot_scale = scale.to_f
|
97
|
+
end
|
98
|
+
|
99
|
+
opts.on('--fill-dot-opacity VALUE', 'range 0.0-1.0 (eg.: 0.0, 0.5, 1.0)') do |opacity|
|
100
|
+
raise 'invalid dot opacity' unless opacity =~ /^(0([.][0-9]*)?|1([.]0*)?)$/
|
101
|
+
@options.fill_dot_opacity = opacity.to_f
|
102
|
+
end
|
103
|
+
|
104
|
+
opts.on('--fill-dot-lifetime VALUE', '(eg.: 15)') do |lifetime|
|
105
|
+
raise 'invalid dot lifetime' unless lifetime =~ /^[1-9][0-9]*$/
|
106
|
+
@options.fill_dot_lifetime = lifetime.to_i
|
107
|
+
end
|
108
|
+
|
109
|
+
opts.on('--time-slot VALUE', 'real life time visualized per video frame (eg.: 10secs, 1min, 99hours, 1day)') do |slot|
|
110
|
+
raise 'invalid time slot' unless slot =~ /^[1-9][0-9]*(sec|min|hour|day)s?$/
|
111
|
+
value = slot.scan(/[0-9]+/)[0].to_i
|
112
|
+
unit = slot.scan(/[a-z]+[^s]/)[0]
|
113
|
+
unit2secs = {'sec' => 1, 'min' => 60, 'hour' => 60*60, 'day' => 24*60*60}
|
114
|
+
@options.group_seconds = value * unit2secs[unit]
|
115
|
+
end
|
116
|
+
|
117
|
+
# TODO
|
118
|
+
# opts.on('--time-format', 'gets auto-detected if not specified') { |v| @options.time_format = v }
|
119
|
+
|
120
|
+
opts.on('--output-format VALUE', 'image format (e.g.: gif, jpg, png) or video format (avi, mpg, mp4)') do |format|
|
121
|
+
video_formats = %w[avi mpg mp4]
|
122
|
+
is_supported = Magick.formats.any?{ |supported_format, properties| supported_format.upcase == format.upcase && properties.include?('w') }
|
123
|
+
is_supported |= video_formats.any?{ |supported_format| supported_format.upcase == format.upcase }
|
124
|
+
raise 'invalid output format' unless is_supported
|
125
|
+
@options.output_format = format
|
126
|
+
end
|
127
|
+
|
128
|
+
opts.on('--[no-]animate', 'generate an image or a video') do |animate|
|
129
|
+
@options.animate = animate
|
130
|
+
end
|
131
|
+
|
132
|
+
begin
|
133
|
+
opts.parse!(@arguments)
|
134
|
+
@log_files = Dir.glob(@arguments)
|
135
|
+
raise 'no log files given' if @log_files.empty?
|
136
|
+
raise 'invalid log file given' unless @log_files.all?{|file| File.readable? file}
|
137
|
+
process_options
|
138
|
+
rescue
|
139
|
+
puts 'Error: ' + $!
|
140
|
+
return false
|
141
|
+
end
|
142
|
+
|
143
|
+
true
|
144
|
+
end
|
145
|
+
|
146
|
+
# Performs post-parse processing on options
|
147
|
+
def process_options
|
148
|
+
$visualization_config = @options
|
149
|
+
end
|
150
|
+
|
151
|
+
def output_options
|
152
|
+
puts 'Options:'
|
153
|
+
@options.marshal_dump.sort{|a,b| a[0].to_s <=> b[0].to_s}.each do |name, val|
|
154
|
+
puts " #{name} = #{val}"
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
# True if required arguments were provided
|
159
|
+
def arguments_valid?
|
160
|
+
video_formats = %w[avi mpg mp4]
|
161
|
+
is_video_format = video_formats.any?{ |video_format| video_format == @options.output_format.downcase }
|
162
|
+
return false if !@options.animate && is_video_format
|
163
|
+
return false if @options.animate && !is_video_format
|
164
|
+
true
|
165
|
+
end
|
166
|
+
|
167
|
+
def output_help
|
168
|
+
puts @opts.help
|
169
|
+
end
|
170
|
+
|
171
|
+
def output_version
|
172
|
+
puts "#{File.basename(__FILE__)} version #{VERSION}"
|
173
|
+
end
|
174
|
+
|
175
|
+
def process_command
|
176
|
+
ApacheLogVisualizer.new(@log_files).visualize
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
# Create and run the application
|
181
|
+
app = App.new(ARGV, STDIN)
|
182
|
+
app.run
|
183
|
+
|
data/lib/ip-world-map.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
|
3
|
+
require 'ip-world-map/ip_lookup_service'
|
4
|
+
require 'ip-world-map/log_analyzer'
|
5
|
+
require 'ip-world-map/information_drawer'
|
6
|
+
require 'ip-world-map/apache_log_analyzer'
|
7
|
+
require 'ip-world-map/apache_log_visualizer'
|
8
|
+
require 'ip-world-map/decay'
|
9
|
+
require 'ip-world-map/file_utils'
|
10
|
+
require 'ip-world-map/point_in_time'
|
11
|
+
require 'ip-world-map/visualization'
|
12
|
+
|
13
|
+
require 'ip-world-map/logfile_mock'
|
14
|
+
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'time'
|
2
|
+
|
3
|
+
class ApacheLogAnalyzer < LogAnalyzer
|
4
|
+
def initialize *filenames
|
5
|
+
super
|
6
|
+
@@host_regex = /^([\w.-]+)/
|
7
|
+
@@time_regex = /\[(\d{2})\/([a-zA-Z]{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) [+-](\d{2})(\d{2})\]/
|
8
|
+
end
|
9
|
+
|
10
|
+
def extract_host_from_line line
|
11
|
+
# IP: "123.1.2.3" or HOSTNAME: "hostname.domain"
|
12
|
+
host = $1 if line =~ @@host_regex
|
13
|
+
end
|
14
|
+
|
15
|
+
def extract_time_from_line line
|
16
|
+
# CLF format: "[dd/MMM/yyyy:hh:mm:ss +-hhmm]"
|
17
|
+
|
18
|
+
# TODO: add timezone information
|
19
|
+
#dd, mmm, yyyy, hh, mm, ss, tz_hh, tz_mm = $1, $2, $3, $4, $5, $6, $7, $8 if line =~ @@time_regex
|
20
|
+
#Time.utc(yyyy, mmm, dd, hh.to_i - tz_hh.to_i, mm, ss)
|
21
|
+
dd, mmm, yyyy, hh, mm, ss = $1, $2, $3, $4, $5, $6 if line =~ @@time_regex
|
22
|
+
Time.utc(yyyy, mmm, dd, hh, mm, ss)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'RMagick'
|
2
|
+
|
3
|
+
class ApacheLogVisualizer
|
4
|
+
def initialize log_files
|
5
|
+
@log_files = log_files
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.detect_time_format times
|
9
|
+
some_samples = times.sort[0..99]
|
10
|
+
smallest_period = some_samples.each_cons(2).collect{ |time1, time2| (time1 - time2).abs }.min || 1
|
11
|
+
|
12
|
+
return '%b %d %Y %H:%M' if smallest_period < 3600 # scale: minutes
|
13
|
+
return '%b %d %Y %H:00' if smallest_period < 86400 # scale: hours
|
14
|
+
return '%b %d %Y' # scale: days
|
15
|
+
end
|
16
|
+
|
17
|
+
def generate_image
|
18
|
+
analyzer = ApacheLogAnalyzer.new(@log_files)
|
19
|
+
details = analyzer.analyze
|
20
|
+
positions = details.collect{ |data| data[:coordinates] }.select{ |coords| coords.any? }
|
21
|
+
|
22
|
+
visualization = Visualization.new
|
23
|
+
image = visualization.draw_positions(positions)
|
24
|
+
save_image image
|
25
|
+
end
|
26
|
+
|
27
|
+
def generate_animation
|
28
|
+
analyzer = ApacheLogAnalyzer.new(@log_files)
|
29
|
+
details = analyzer.analyze
|
30
|
+
grouped_details = analyzer.group_by_time(details, $visualization_config.group_seconds)
|
31
|
+
|
32
|
+
animation = Magick::ImageList.new
|
33
|
+
visualization = Visualization.new
|
34
|
+
time_format = $visualization_config.time_format || ApacheLogVisualizer.detect_time_format(grouped_details.keys)
|
35
|
+
frame_number = 0
|
36
|
+
|
37
|
+
puts "\nGenerating frames:" if $visualization_config.verbose
|
38
|
+
grouped_details.sort.each do |time, details|
|
39
|
+
frame_number += 1
|
40
|
+
visualization.new_frame
|
41
|
+
positions = details.collect{ |data| data[:coordinates] }.select{ |coords| coords.any? }
|
42
|
+
p [time, details.size, positions.size] if $visualization_config.verbose
|
43
|
+
image = visualization.draw_positions(positions)
|
44
|
+
|
45
|
+
InformationDrawer.new.draw_info(image, visualization, time.strftime(time_format))
|
46
|
+
save_image image, frame_number
|
47
|
+
end
|
48
|
+
|
49
|
+
render_frames_as_video
|
50
|
+
end
|
51
|
+
|
52
|
+
def save_image image, frame_number = 0
|
53
|
+
if $visualization_config.animate
|
54
|
+
image.write "animation.#{'%09d' % frame_number}.bmp"
|
55
|
+
else
|
56
|
+
image.write "snapshot.#{$visualization_config.output_format}"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def render_frames_as_video
|
61
|
+
puts "\nGenerating video:" if $visualization_config.verbose
|
62
|
+
output = `ffmpeg -r #{$visualization_config.frames_per_second} -qscale 1 -y -i animation.%09d.bmp animation.#{$visualization_config.output_format} 2>&1`
|
63
|
+
puts output if $visualization_config.verbose
|
64
|
+
raise 'could not create the animation' unless $?.exitstatus == 0
|
65
|
+
end
|
66
|
+
|
67
|
+
def visualize
|
68
|
+
if $visualization_config.animate
|
69
|
+
generate_animation
|
70
|
+
else
|
71
|
+
generate_image
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# see http://en.wikipedia.org/wiki/Exponential_decay
|
2
|
+
class Decay
|
3
|
+
def initialize initial_value, lifetime
|
4
|
+
@initial_value = initial_value.to_f
|
5
|
+
@lifetime = lifetime.to_f
|
6
|
+
end
|
7
|
+
|
8
|
+
def value time
|
9
|
+
@initial_value * Math::exp(-time / @lifetime)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
|
3
|
+
module FileUtils
|
4
|
+
def self.zipped? filename
|
5
|
+
%w[.gz .Z].include? File.extname(filename)
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.open filename, &block
|
9
|
+
if zipped? filename
|
10
|
+
Zlib::GzipReader.open(filename, &block)
|
11
|
+
else
|
12
|
+
File.open(filename, &block)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'RMagick'
|
2
|
+
|
3
|
+
# Draws the timestamp.
|
4
|
+
class InformationDrawer
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@draw = Magick::Draw.new
|
8
|
+
end
|
9
|
+
|
10
|
+
def draw_info image, visualization, info
|
11
|
+
size = { :width => visualization.map_size[:width], :height => visualization.map_size[:height] }
|
12
|
+
draw_background(size)
|
13
|
+
draw_message(size, info)
|
14
|
+
@draw.draw(image)
|
15
|
+
end
|
16
|
+
|
17
|
+
protected
|
18
|
+
|
19
|
+
def draw_background size
|
20
|
+
width, height = size[:width], size[:height]
|
21
|
+
@draw.fill('grey')
|
22
|
+
@draw.fill_opacity('50%')
|
23
|
+
@draw.rectangle(0.2 * width, 0.9 * height, 0.8 * width, 0.9 * height + 30)
|
24
|
+
end
|
25
|
+
|
26
|
+
def draw_message size, info
|
27
|
+
@draw.fill('black')
|
28
|
+
@draw.fill_opacity('100%')
|
29
|
+
@draw.text_align(Magick::CenterAlign)
|
30
|
+
@draw.pointsize(20)
|
31
|
+
@draw.text(0.5 * size[:width], 0.9 * size[:height] + 20, info)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'typhoeus'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
class IpLookupService
|
6
|
+
|
7
|
+
def initialize filename = nil
|
8
|
+
@filename = filename || File.join(File.dirname(__FILE__), '..', '..', 'resources', 'coordinates.yml')
|
9
|
+
reset
|
10
|
+
end
|
11
|
+
|
12
|
+
def reset
|
13
|
+
@host_coordinates = {}
|
14
|
+
@host_ips = {}
|
15
|
+
end
|
16
|
+
|
17
|
+
def coordinates_for_hosts hosts
|
18
|
+
uniq_hosts = hosts.uniq
|
19
|
+
|
20
|
+
uniq_hosts.each do |host|
|
21
|
+
@host_ips[host] ||= IPSocket.getaddress(host) rescue nil
|
22
|
+
end
|
23
|
+
|
24
|
+
hydra = Typhoeus::Hydra.new
|
25
|
+
uniq_hosts.each do |host|
|
26
|
+
unless @host_coordinates[host]
|
27
|
+
request = Typhoeus::Request.new("http://api.hostip.info/get_html.php?position=true&ip=#{@host_ips[host]}")
|
28
|
+
request.on_complete do |response|
|
29
|
+
@host_coordinates[host] = extract_longitude_and_latitude(response.body)
|
30
|
+
end
|
31
|
+
hydra.queue(request)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
hydra.run
|
35
|
+
|
36
|
+
@host_coordinates
|
37
|
+
end
|
38
|
+
|
39
|
+
def coordinates_for_host host
|
40
|
+
unless @host_coordinates[host]
|
41
|
+
@host_ips[host] ||= IPSocket.getaddress(host) rescue nil
|
42
|
+
response = Net::HTTP.get('api.hostip.info', "/get_html.php?position=true&ip=#{@host_ips[host]}")
|
43
|
+
@host_coordinates[host] = extract_longitude_and_latitude(response)
|
44
|
+
end
|
45
|
+
|
46
|
+
@host_coordinates[host]
|
47
|
+
end
|
48
|
+
|
49
|
+
def extract_longitude_and_latitude string
|
50
|
+
latitude = string.match(/Latitude: (-?[0-9.]+)/)[1].to_f rescue nil
|
51
|
+
longitude = string.match(/Longitude: (-?[0-9.]+)/)[1].to_f rescue nil
|
52
|
+
[longitude, latitude]
|
53
|
+
end
|
54
|
+
|
55
|
+
def save_coordinates
|
56
|
+
File.open(@filename, 'w'){ |f| f.write @host_coordinates.to_yaml }
|
57
|
+
end
|
58
|
+
|
59
|
+
def load_coordinates
|
60
|
+
@host_coordinates = YAML.load_file(@filename) if File.readable? @filename
|
61
|
+
end
|
62
|
+
|
63
|
+
def stats
|
64
|
+
unknown_coordinates, known_coordinates = @host_coordinates.partition{ |ip, coords| coords.include? nil }
|
65
|
+
|
66
|
+
{ :unknown_coordinates => unknown_coordinates.size, :known_coordinates => known_coordinates.size }
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
@@ -0,0 +1,60 @@
|
|
1
|
+
class LogAnalyzer
|
2
|
+
attr_accessor :host_coordinates
|
3
|
+
|
4
|
+
def initialize *filenames
|
5
|
+
@filenames = filenames.flatten.sort.uniq
|
6
|
+
@ip_lookup_service = IpLookupService.new
|
7
|
+
@ip_lookup_service.load_coordinates
|
8
|
+
end
|
9
|
+
|
10
|
+
def analyze
|
11
|
+
details = []
|
12
|
+
|
13
|
+
puts "\nReading files:" if $visualization_config.verbose
|
14
|
+
@filenames.each do |filename|
|
15
|
+
puts filename if $visualization_config.verbose
|
16
|
+
FileUtils.open(filename).each_line do |line|
|
17
|
+
details << details_from_line(line)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
hosts = details.collect{|detail| detail[:host]}
|
22
|
+
coordinates = @ip_lookup_service.coordinates_for_hosts(hosts)
|
23
|
+
|
24
|
+
details.collect! do |detail|
|
25
|
+
detail[:coordinates] = coordinates[detail[:host]]
|
26
|
+
detail
|
27
|
+
end
|
28
|
+
|
29
|
+
details
|
30
|
+
end
|
31
|
+
|
32
|
+
def details_from_line line
|
33
|
+
host = extract_host_from_line(line)
|
34
|
+
time = extract_time_from_line(line)
|
35
|
+
{ :time => time, :host => host }
|
36
|
+
end
|
37
|
+
|
38
|
+
def calculate_oldest_time details
|
39
|
+
details.min{ |a, b| a[:time] <=> b[:time] }[:time]
|
40
|
+
end
|
41
|
+
|
42
|
+
def group_by_time details, slot_in_seconds
|
43
|
+
return {} unless details && slot_in_seconds
|
44
|
+
details_per_slot = {}
|
45
|
+
|
46
|
+
# TODO: maybe assign empty arrays to missing slots where no traffic was detected
|
47
|
+
details.each do |detail|
|
48
|
+
slot_start_time = calculate_slot_start_time(detail[:time], slot_in_seconds)
|
49
|
+
details_per_slot[slot_start_time] ||= []
|
50
|
+
details_per_slot[slot_start_time] << detail
|
51
|
+
end
|
52
|
+
|
53
|
+
details_per_slot
|
54
|
+
end
|
55
|
+
|
56
|
+
def calculate_slot_start_time time, slot_in_seconds
|
57
|
+
Time.at(time.tv_sec - (time.tv_sec % slot_in_seconds))
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class LogfileMock
|
2
|
+
attr_accessor :min_random_positions, :max_random_positions
|
3
|
+
|
4
|
+
def initialize
|
5
|
+
@min_random_positions = 5
|
6
|
+
@max_random_positions = 50
|
7
|
+
end
|
8
|
+
|
9
|
+
def random_longitude_latitude
|
10
|
+
[ -10 + rand(25) + rand, 40 + rand(15) + rand ]
|
11
|
+
end
|
12
|
+
|
13
|
+
def positions
|
14
|
+
amount = @min_random_positions + rand(@max_random_positions - @min_random_positions + 1)
|
15
|
+
Array.new(amount){ random_longitude_latitude }
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
class PointInTime
|
2
|
+
attr_reader :x, :y
|
3
|
+
|
4
|
+
def initialize x, y, initial_opacity, lifetime
|
5
|
+
@x, @y = x, y
|
6
|
+
@time = 0
|
7
|
+
@decay = Decay.new(initial_opacity, lifetime)
|
8
|
+
end
|
9
|
+
|
10
|
+
def opacity_in_time time
|
11
|
+
@time = time if time
|
12
|
+
opacity
|
13
|
+
end
|
14
|
+
|
15
|
+
def opacity
|
16
|
+
@decay.value(@time)
|
17
|
+
end
|
18
|
+
|
19
|
+
def age
|
20
|
+
@time += 1
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'RMagick'
|
2
|
+
|
3
|
+
class Visualization
|
4
|
+
attr_accessor :position_quantization_in_degrees, :circle_radius
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@map_filename = $visualization_config.map_filename
|
8
|
+
@raw_image = Magick::ImageList.new(@map_filename).first
|
9
|
+
if $visualization_config.map_width || $visualization_config.map_height
|
10
|
+
width = $visualization_config.map_width || @raw_image.columns
|
11
|
+
height = $visualization_config.map_height || @raw_image.rows
|
12
|
+
@raw_image.resize! width, height
|
13
|
+
end
|
14
|
+
new_frame
|
15
|
+
@position_quantization_in_degrees = 0
|
16
|
+
@opacity_visibility_threshold = 0.1
|
17
|
+
@circle_radius = (map_size[:width] ** 1.25) / (map_size[:width] * $visualization_config.fill_dot_scale).to_f
|
18
|
+
@points = {}
|
19
|
+
end
|
20
|
+
|
21
|
+
def map_size
|
22
|
+
@map_size ||= { :width => @frame.columns, :height => @frame.rows }
|
23
|
+
end
|
24
|
+
|
25
|
+
def scale
|
26
|
+
@scale ||= { :x => 360.0 / map_size[:width], :y => 180.0 / map_size[:height] }
|
27
|
+
end
|
28
|
+
|
29
|
+
def x_y_from_longitude_latitude longitude, latitude
|
30
|
+
[ (180 + longitude) / scale[:x], (90 - latitude) / scale[:y] ]
|
31
|
+
end
|
32
|
+
|
33
|
+
def circle_parameters center_x, center_y
|
34
|
+
[ center_x, center_y, center_x + circle_radius, center_y ]
|
35
|
+
end
|
36
|
+
|
37
|
+
def quantize_position *position
|
38
|
+
return position if @position_quantization_in_degrees == 0
|
39
|
+
position.collect{ |element| element - element.remainder(@position_quantization_in_degrees) }
|
40
|
+
end
|
41
|
+
|
42
|
+
def select_visible_points points
|
43
|
+
#points.select{ |point| point.opacity >= @opacity_visibility_threshold }
|
44
|
+
selected_points = {}
|
45
|
+
points.each{ |key, point| selected_points[key] = point if point.opacity >= @opacity_visibility_threshold }
|
46
|
+
end
|
47
|
+
|
48
|
+
def draw_positions positions_lon_lat
|
49
|
+
@draw.fill($visualization_config.fill_dot_color)
|
50
|
+
|
51
|
+
new_points = {}
|
52
|
+
positions_lon_lat.each do |longitude, latitude|
|
53
|
+
x, y = x_y_from_longitude_latitude(longitude, latitude)
|
54
|
+
x, y = quantize_position(x, y)
|
55
|
+
new_points[[x,y]] = PointInTime.new(x, y, $visualization_config.fill_dot_opacity, $visualization_config.fill_dot_lifetime)
|
56
|
+
end
|
57
|
+
|
58
|
+
@points = select_visible_points(@points)
|
59
|
+
@points.merge!(new_points)
|
60
|
+
|
61
|
+
@points.each do |key, point|
|
62
|
+
@draw.fill_opacity(point.opacity)
|
63
|
+
@draw.circle(*circle_parameters(point.x, point.y))
|
64
|
+
point.age
|
65
|
+
end
|
66
|
+
|
67
|
+
@draw.draw(@frame)
|
68
|
+
@frame
|
69
|
+
end
|
70
|
+
|
71
|
+
def display
|
72
|
+
@frame.display
|
73
|
+
end
|
74
|
+
|
75
|
+
def new_frame
|
76
|
+
@draw = Magick::Draw.new
|
77
|
+
@frame = @raw_image.clone
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
Binary file
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Decay do
|
4
|
+
it 'should be initialized with either Integers and/or Floats' do
|
5
|
+
Decay.new(10, 5 )
|
6
|
+
Decay.new(10, 5.0)
|
7
|
+
Decay.new(10.0, 5 )
|
8
|
+
Decay.new(10.0, 5.0)
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should return a value depending on the time' do
|
12
|
+
decay = Decay.new(10, 5)
|
13
|
+
|
14
|
+
decay.value(0).should == 10.0
|
15
|
+
decay.value(5).should < 5.0
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe IpLookupService do
|
4
|
+
describe 'hostname -> coordinates resolval' do
|
5
|
+
it 'should return the coordinates for an IP' do
|
6
|
+
stubbed_result = "Country: GERMANY (DE)\nCity: Berlin\n\nLatitude: 52.5\nLongitude: 13.4167\nIP: 1.1.1.1\n"
|
7
|
+
Net::HTTP.stub!(:get).and_return(stubbed_result)
|
8
|
+
|
9
|
+
IpLookupService.new.coordinates_for_host('1.1.1.1').should == [13.4167, 52.5]
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'should return the coordinates for a hostname' do
|
13
|
+
stubbed_result = "Country: GERMANY (DE)\nCity: Berlin\n\nLatitude: 52.5\nLongitude: 13.4167\nIP: 1.1.1.1\n"
|
14
|
+
Net::HTTP.stub!(:get).and_return(stubbed_result)
|
15
|
+
IPSocket.stub!(:getaddr).and_return('1.1.1.1')
|
16
|
+
|
17
|
+
IpLookupService.new.coordinates_for_host('some hostname').should == [13.4167, 52.5]
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should handle the case that no IP can be resolved for a hostname' do
|
21
|
+
stubbed_result = "Country: GERMANY (DE)\nCity: Berlin\n\nLatitude: \nLongitude: \nIP: 1.1.1.1\n"
|
22
|
+
Net::HTTP.stub!(:get).and_return(stubbed_result)
|
23
|
+
IPSocket.stub!(:getaddrinfo).and_raise(SocketError)
|
24
|
+
|
25
|
+
IpLookupService.new.coordinates_for_host('some hostname').should == [nil, nil]
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should handle the case that no coordinates can be resolved for a hostname' do
|
29
|
+
stubbed_result = "Country: GERMANY (DE)\nCity: Berlin\n\nLatitude: \nLongitude: \nIP: 1.1.1.1\n"
|
30
|
+
Net::HTTP.stub!(:get).and_return(stubbed_result)
|
31
|
+
IPSocket.stub!(:getaddr)
|
32
|
+
|
33
|
+
IpLookupService.new.coordinates_for_host('some hostname').should == [nil, nil]
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe ApacheLogVisualizer do
|
4
|
+
describe 'time format' do
|
5
|
+
it 'should return a minute based time format for times within an hour' do
|
6
|
+
now = Time.now
|
7
|
+
times = [now, now + 3599]
|
8
|
+
|
9
|
+
ApacheLogVisualizer.detect_time_format(times).should == '%b %d %Y %H:%M'
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'should return a hour based time format for times within a day' do
|
13
|
+
now = Time.now
|
14
|
+
times = [now, now + 86399]
|
15
|
+
|
16
|
+
ApacheLogVisualizer.detect_time_format(times).should == '%b %d %Y %H:00'
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'should return a day based time format for times greater than a day' do
|
20
|
+
now = Time.now
|
21
|
+
times = [now, now + 86400]
|
22
|
+
|
23
|
+
ApacheLogVisualizer.detect_time_format(times).should == '%b %d %Y'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ip-world-map
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 23
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
- 0
|
10
|
+
version: 1.0.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- "Ren\xC3\xA9 Scheibe"
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-07-26 00:00:00 +02:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: rmagick
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 57
|
30
|
+
segments:
|
31
|
+
- 2
|
32
|
+
- 13
|
33
|
+
- 1
|
34
|
+
version: 2.13.1
|
35
|
+
type: :runtime
|
36
|
+
version_requirements: *id001
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: rspec
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
hash: 23
|
46
|
+
segments:
|
47
|
+
- 2
|
48
|
+
- 6
|
49
|
+
- 0
|
50
|
+
version: 2.6.0
|
51
|
+
type: :development
|
52
|
+
version_requirements: *id002
|
53
|
+
description: A tool to generate images/videos of user locations based on Apache log files.
|
54
|
+
email: rene.scheibe@gmail.com
|
55
|
+
executables:
|
56
|
+
- ip-world-map
|
57
|
+
extensions: []
|
58
|
+
|
59
|
+
extra_rdoc_files: []
|
60
|
+
|
61
|
+
files:
|
62
|
+
- lib/ip-world-map/information_drawer.rb
|
63
|
+
- lib/ip-world-map/logfile_mock.rb
|
64
|
+
- lib/ip-world-map/decay.rb
|
65
|
+
- lib/ip-world-map/log_analyzer.rb
|
66
|
+
- lib/ip-world-map/apache_log_analyzer.rb
|
67
|
+
- lib/ip-world-map/apache_log_visualizer.rb
|
68
|
+
- lib/ip-world-map/point_in_time.rb
|
69
|
+
- lib/ip-world-map/ip_lookup_service.rb
|
70
|
+
- lib/ip-world-map/file_utils.rb
|
71
|
+
- lib/ip-world-map/visualization.rb
|
72
|
+
- lib/ip-world-map.rb
|
73
|
+
- spec/ip-world-map/decay.rb
|
74
|
+
- spec/ip-world-map/time_format.rb
|
75
|
+
- spec/ip-world-map/ip_lookup_service.rb
|
76
|
+
- spec/spec_helper.rb
|
77
|
+
- Rakefile
|
78
|
+
- resources/maps/earthmap-1920x960.tif
|
79
|
+
- bin/ip-world-map
|
80
|
+
has_rdoc: true
|
81
|
+
homepage: http://github.com/darxriggs/ip-world-map
|
82
|
+
licenses:
|
83
|
+
- GPL-2
|
84
|
+
post_install_message:
|
85
|
+
rdoc_options: []
|
86
|
+
|
87
|
+
require_paths:
|
88
|
+
- lib
|
89
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
hash: 57
|
95
|
+
segments:
|
96
|
+
- 1
|
97
|
+
- 8
|
98
|
+
- 7
|
99
|
+
version: 1.8.7
|
100
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
101
|
+
none: false
|
102
|
+
requirements:
|
103
|
+
- - ">="
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
hash: 3
|
106
|
+
segments:
|
107
|
+
- 0
|
108
|
+
version: "0"
|
109
|
+
requirements:
|
110
|
+
- ImageMagick (used by rmagick)
|
111
|
+
- ffmpeg (only for animations)
|
112
|
+
rubyforge_project:
|
113
|
+
rubygems_version: 1.3.7
|
114
|
+
signing_key:
|
115
|
+
specification_version: 3
|
116
|
+
summary: A tool to generate images/videos of user locations based on Apache log files.
|
117
|
+
test_files: []
|
118
|
+
|