ip-world-map 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +45 -0
- data/bin/ip-world-map +183 -0
- data/lib/ip-world-map.rb +14 -0
- data/lib/ip-world-map/apache_log_analyzer.rb +25 -0
- data/lib/ip-world-map/apache_log_visualizer.rb +75 -0
- data/lib/ip-world-map/decay.rb +12 -0
- data/lib/ip-world-map/file_utils.rb +16 -0
- data/lib/ip-world-map/information_drawer.rb +34 -0
- data/lib/ip-world-map/ip_lookup_service.rb +69 -0
- data/lib/ip-world-map/log_analyzer.rb +60 -0
- data/lib/ip-world-map/logfile_mock.rb +18 -0
- data/lib/ip-world-map/point_in_time.rb +23 -0
- data/lib/ip-world-map/visualization.rb +80 -0
- data/resources/maps/earthmap-1920x960.tif +0 -0
- data/spec/ip-world-map/decay.rb +18 -0
- data/spec/ip-world-map/ip_lookup_service.rb +37 -0
- data/spec/ip-world-map/time_format.rb +27 -0
- data/spec/spec_helper.rb +5 -0
- metadata +118 -0
data/Rakefile
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'rubygems'
|
3
|
+
require 'rubygems/package_task'
|
4
|
+
require 'rspec/core/rake_task'
|
5
|
+
|
6
|
+
RSpec::Core::RakeTask.new do |t|
|
7
|
+
t.pattern = 'spec/**/*.rb'
|
8
|
+
t.rcov = false
|
9
|
+
t.rcov_opts = %q[--exclude "spec"]
|
10
|
+
t.verbose = true
|
11
|
+
end
|
12
|
+
|
13
|
+
spec = Gem::Specification.new do |s|
|
14
|
+
s.platform = Gem::Platform::RUBY
|
15
|
+
s.required_ruby_version = '>= 1.8.7'
|
16
|
+
s.summary = 'A tool to generate images/videos of user locations based on Apache log files.'
|
17
|
+
s.name = 'ip-world-map'
|
18
|
+
s.version = '1.0.0'
|
19
|
+
s.license = 'GPL-2'
|
20
|
+
s.executables = ['ip-world-map']
|
21
|
+
s.files = FileList['{lib,spec}/**/*'].to_a +
|
22
|
+
['Rakefile'] +
|
23
|
+
['resources/maps/earthmap-1920x960.tif']
|
24
|
+
s.has_rdoc = false
|
25
|
+
s.description = s.summary
|
26
|
+
s.homepage = 'http://github.com/darxriggs/ip-world-map'
|
27
|
+
s.author = 'René Scheibe'
|
28
|
+
s.email = 'rene.scheibe@gmail.com'
|
29
|
+
|
30
|
+
s.requirements = ['ImageMagick (used by rmagick)', 'ffmpeg (only for animations)']
|
31
|
+
s.add_runtime_dependency('rmagick', '~> 2.13.1')
|
32
|
+
s.add_development_dependency('rspec', '~> 2.6.0')
|
33
|
+
end
|
34
|
+
|
35
|
+
Gem::PackageTask.new(spec) do |pkg|
|
36
|
+
pkg.need_tar = false
|
37
|
+
end
|
38
|
+
|
39
|
+
desc 'Install gem locally'
|
40
|
+
task :install_gem => :package do
|
41
|
+
`sudo gem install pkg/*.gem --no-ri --no-rdoc`
|
42
|
+
end
|
43
|
+
|
44
|
+
task :default => :install_gem
|
45
|
+
|
data/bin/ip-world-map
ADDED
@@ -0,0 +1,183 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'optparse'
|
5
|
+
require 'ostruct'
|
6
|
+
require 'date'
|
7
|
+
require 'ip-world-map'
|
8
|
+
require 'RMagick'
|
9
|
+
|
10
|
+
class App
|
11
|
+
VERSION = '0.0.1'
|
12
|
+
|
13
|
+
attr_reader :options
|
14
|
+
|
15
|
+
def initialize arguments, stdin
|
16
|
+
@arguments = arguments
|
17
|
+
@stdin = stdin
|
18
|
+
|
19
|
+
set_defaults
|
20
|
+
end
|
21
|
+
|
22
|
+
def run
|
23
|
+
if parsed_arguments? && arguments_valid?
|
24
|
+
output_options if @options.verbose
|
25
|
+
process_command
|
26
|
+
else
|
27
|
+
output_help
|
28
|
+
exit 1
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
protected
|
33
|
+
|
34
|
+
def set_defaults
|
35
|
+
@options = OpenStruct.new({
|
36
|
+
:verbose => false,
|
37
|
+
:quiet => false,
|
38
|
+
:map_filename => File.join(File.dirname(__FILE__), '..' , 'resources', 'maps', 'earthmap-1920x960.tif'),
|
39
|
+
:map_width => 800,
|
40
|
+
:map_height => 400,
|
41
|
+
:frames_per_second => 25,
|
42
|
+
:fill_dot_color => 'red',
|
43
|
+
:fill_dot_scale => 10,
|
44
|
+
:fill_dot_opacity => 1.0,
|
45
|
+
:fill_dot_lifetime => 15,
|
46
|
+
:time_format => nil,
|
47
|
+
:group_seconds => 24 * 1 * 60 * 60,
|
48
|
+
:output_format => 'png',
|
49
|
+
:animate => false
|
50
|
+
})
|
51
|
+
end
|
52
|
+
|
53
|
+
def parsed_arguments?
|
54
|
+
@opts = opts = OptionParser.new
|
55
|
+
|
56
|
+
opts.banner = 'Usage: ip-world-map [options] log_file1 [logfile2] ...'
|
57
|
+
|
58
|
+
opts.on('--version', 'Display the version') do
|
59
|
+
output_version
|
60
|
+
exit 0
|
61
|
+
end
|
62
|
+
|
63
|
+
opts.on('-h', '--help', 'Display this help message') do
|
64
|
+
output_help
|
65
|
+
exit 0
|
66
|
+
end
|
67
|
+
|
68
|
+
opts.on('-v', '--verbose', 'Verbose output') do
|
69
|
+
@options.verbose = true
|
70
|
+
end
|
71
|
+
|
72
|
+
opts.on('--map-filename VALUE', 'The image to use as background') do |filename|
|
73
|
+
raise 'invalid map file' unless File.readable? filename
|
74
|
+
@options.map_filename = filename
|
75
|
+
end
|
76
|
+
|
77
|
+
opts.on('--resolution VALUE', '(eg.: 640x480)') do |resolution|
|
78
|
+
raise 'invalid resolution format' unless resolution =~ /^[0-9]+x[0-9]+$/
|
79
|
+
width, height = resolution.split('x').collect{|v| v.to_i}
|
80
|
+
@options.map_width = width
|
81
|
+
@options.map_height = height
|
82
|
+
end
|
83
|
+
|
84
|
+
opts.on('--fps VALUE', 'Animation frames per second (eg.: 25)') do |fps|
|
85
|
+
raise 'invalid fps format' unless fps =~ /^[1-9][0-9]*$/
|
86
|
+
@options.frames_per_second = fps.to_i
|
87
|
+
end
|
88
|
+
|
89
|
+
opts.on('--fill-dot-color VALUE', "(eg.: red, 'rgb(255,0,0)', '#FF0000')") do |color|
|
90
|
+
Magick::Pixel.from_color color rescue raise 'invalid color (see help for examples)'
|
91
|
+
@options.fill_dot_color = color
|
92
|
+
end
|
93
|
+
|
94
|
+
opts.on('--fill-dot-scale VALUE', '(eg.: 10.0)') do |scale|
|
95
|
+
raise 'invalid dot scale' unless scale =~ /^[1-9][0-9]*([.][0-9]*)?$/
|
96
|
+
@options.fill_dot_scale = scale.to_f
|
97
|
+
end
|
98
|
+
|
99
|
+
opts.on('--fill-dot-opacity VALUE', 'range 0.0-1.0 (eg.: 0.0, 0.5, 1.0)') do |opacity|
|
100
|
+
raise 'invalid dot opacity' unless opacity =~ /^(0([.][0-9]*)?|1([.]0*)?)$/
|
101
|
+
@options.fill_dot_opacity = opacity.to_f
|
102
|
+
end
|
103
|
+
|
104
|
+
opts.on('--fill-dot-lifetime VALUE', '(eg.: 15)') do |lifetime|
|
105
|
+
raise 'invalid dot lifetime' unless lifetime =~ /^[1-9][0-9]*$/
|
106
|
+
@options.fill_dot_lifetime = lifetime.to_i
|
107
|
+
end
|
108
|
+
|
109
|
+
opts.on('--time-slot VALUE', 'real life time visualized per video frame (eg.: 10secs, 1min, 99hours, 1day)') do |slot|
|
110
|
+
raise 'invalid time slot' unless slot =~ /^[1-9][0-9]*(sec|min|hour|day)s?$/
|
111
|
+
value = slot.scan(/[0-9]+/)[0].to_i
|
112
|
+
unit = slot.scan(/[a-z]+[^s]/)[0]
|
113
|
+
unit2secs = {'sec' => 1, 'min' => 60, 'hour' => 60*60, 'day' => 24*60*60}
|
114
|
+
@options.group_seconds = value * unit2secs[unit]
|
115
|
+
end
|
116
|
+
|
117
|
+
# TODO
|
118
|
+
# opts.on('--time-format', 'gets auto-detected if not specified') { |v| @options.time_format = v }
|
119
|
+
|
120
|
+
opts.on('--output-format VALUE', 'image format (e.g.: gif, jpg, png) or video format (avi, mpg, mp4)') do |format|
|
121
|
+
video_formats = %w[avi mpg mp4]
|
122
|
+
is_supported = Magick.formats.any?{ |supported_format, properties| supported_format.upcase == format.upcase && properties.include?('w') }
|
123
|
+
is_supported |= video_formats.any?{ |supported_format| supported_format.upcase == format.upcase }
|
124
|
+
raise 'invalid output format' unless is_supported
|
125
|
+
@options.output_format = format
|
126
|
+
end
|
127
|
+
|
128
|
+
opts.on('--[no-]animate', 'generate an image or a video') do |animate|
|
129
|
+
@options.animate = animate
|
130
|
+
end
|
131
|
+
|
132
|
+
begin
|
133
|
+
opts.parse!(@arguments)
|
134
|
+
@log_files = Dir.glob(@arguments)
|
135
|
+
raise 'no log files given' if @log_files.empty?
|
136
|
+
raise 'invalid log file given' unless @log_files.all?{|file| File.readable? file}
|
137
|
+
process_options
|
138
|
+
rescue
|
139
|
+
puts 'Error: ' + $!
|
140
|
+
return false
|
141
|
+
end
|
142
|
+
|
143
|
+
true
|
144
|
+
end
|
145
|
+
|
146
|
+
# Performs post-parse processing on options
|
147
|
+
def process_options
|
148
|
+
$visualization_config = @options
|
149
|
+
end
|
150
|
+
|
151
|
+
def output_options
|
152
|
+
puts 'Options:'
|
153
|
+
@options.marshal_dump.sort{|a,b| a[0].to_s <=> b[0].to_s}.each do |name, val|
|
154
|
+
puts " #{name} = #{val}"
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
# True if required arguments were provided
|
159
|
+
def arguments_valid?
|
160
|
+
video_formats = %w[avi mpg mp4]
|
161
|
+
is_video_format = video_formats.any?{ |video_format| video_format == @options.output_format.downcase }
|
162
|
+
return false if !@options.animate && is_video_format
|
163
|
+
return false if @options.animate && !is_video_format
|
164
|
+
true
|
165
|
+
end
|
166
|
+
|
167
|
+
def output_help
|
168
|
+
puts @opts.help
|
169
|
+
end
|
170
|
+
|
171
|
+
def output_version
|
172
|
+
puts "#{File.basename(__FILE__)} version #{VERSION}"
|
173
|
+
end
|
174
|
+
|
175
|
+
def process_command
|
176
|
+
ApacheLogVisualizer.new(@log_files).visualize
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
# Create and run the application
|
181
|
+
app = App.new(ARGV, STDIN)
|
182
|
+
app.run
|
183
|
+
|
data/lib/ip-world-map.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
|
3
|
+
require 'ip-world-map/ip_lookup_service'
|
4
|
+
require 'ip-world-map/log_analyzer'
|
5
|
+
require 'ip-world-map/information_drawer'
|
6
|
+
require 'ip-world-map/apache_log_analyzer'
|
7
|
+
require 'ip-world-map/apache_log_visualizer'
|
8
|
+
require 'ip-world-map/decay'
|
9
|
+
require 'ip-world-map/file_utils'
|
10
|
+
require 'ip-world-map/point_in_time'
|
11
|
+
require 'ip-world-map/visualization'
|
12
|
+
|
13
|
+
require 'ip-world-map/logfile_mock'
|
14
|
+
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'time'
|
2
|
+
|
3
|
+
class ApacheLogAnalyzer < LogAnalyzer
|
4
|
+
def initialize *filenames
|
5
|
+
super
|
6
|
+
@@host_regex = /^([\w.-]+)/
|
7
|
+
@@time_regex = /\[(\d{2})\/([a-zA-Z]{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) [+-](\d{2})(\d{2})\]/
|
8
|
+
end
|
9
|
+
|
10
|
+
def extract_host_from_line line
|
11
|
+
# IP: "123.1.2.3" or HOSTNAME: "hostname.domain"
|
12
|
+
host = $1 if line =~ @@host_regex
|
13
|
+
end
|
14
|
+
|
15
|
+
def extract_time_from_line line
|
16
|
+
# CLF format: "[dd/MMM/yyyy:hh:mm:ss +-hhmm]"
|
17
|
+
|
18
|
+
# TODO: add timezone information
|
19
|
+
#dd, mmm, yyyy, hh, mm, ss, tz_hh, tz_mm = $1, $2, $3, $4, $5, $6, $7, $8 if line =~ @@time_regex
|
20
|
+
#Time.utc(yyyy, mmm, dd, hh.to_i - tz_hh.to_i, mm, ss)
|
21
|
+
dd, mmm, yyyy, hh, mm, ss = $1, $2, $3, $4, $5, $6 if line =~ @@time_regex
|
22
|
+
Time.utc(yyyy, mmm, dd, hh, mm, ss)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'RMagick'
|
2
|
+
|
3
|
+
class ApacheLogVisualizer
|
4
|
+
def initialize log_files
|
5
|
+
@log_files = log_files
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.detect_time_format times
|
9
|
+
some_samples = times.sort[0..99]
|
10
|
+
smallest_period = some_samples.each_cons(2).collect{ |time1, time2| (time1 - time2).abs }.min || 1
|
11
|
+
|
12
|
+
return '%b %d %Y %H:%M' if smallest_period < 3600 # scale: minutes
|
13
|
+
return '%b %d %Y %H:00' if smallest_period < 86400 # scale: hours
|
14
|
+
return '%b %d %Y' # scale: days
|
15
|
+
end
|
16
|
+
|
17
|
+
def generate_image
|
18
|
+
analyzer = ApacheLogAnalyzer.new(@log_files)
|
19
|
+
details = analyzer.analyze
|
20
|
+
positions = details.collect{ |data| data[:coordinates] }.select{ |coords| coords.any? }
|
21
|
+
|
22
|
+
visualization = Visualization.new
|
23
|
+
image = visualization.draw_positions(positions)
|
24
|
+
save_image image
|
25
|
+
end
|
26
|
+
|
27
|
+
def generate_animation
|
28
|
+
analyzer = ApacheLogAnalyzer.new(@log_files)
|
29
|
+
details = analyzer.analyze
|
30
|
+
grouped_details = analyzer.group_by_time(details, $visualization_config.group_seconds)
|
31
|
+
|
32
|
+
animation = Magick::ImageList.new
|
33
|
+
visualization = Visualization.new
|
34
|
+
time_format = $visualization_config.time_format || ApacheLogVisualizer.detect_time_format(grouped_details.keys)
|
35
|
+
frame_number = 0
|
36
|
+
|
37
|
+
puts "\nGenerating frames:" if $visualization_config.verbose
|
38
|
+
grouped_details.sort.each do |time, details|
|
39
|
+
frame_number += 1
|
40
|
+
visualization.new_frame
|
41
|
+
positions = details.collect{ |data| data[:coordinates] }.select{ |coords| coords.any? }
|
42
|
+
p [time, details.size, positions.size] if $visualization_config.verbose
|
43
|
+
image = visualization.draw_positions(positions)
|
44
|
+
|
45
|
+
InformationDrawer.new.draw_info(image, visualization, time.strftime(time_format))
|
46
|
+
save_image image, frame_number
|
47
|
+
end
|
48
|
+
|
49
|
+
render_frames_as_video
|
50
|
+
end
|
51
|
+
|
52
|
+
def save_image image, frame_number = 0
|
53
|
+
if $visualization_config.animate
|
54
|
+
image.write "animation.#{'%09d' % frame_number}.bmp"
|
55
|
+
else
|
56
|
+
image.write "snapshot.#{$visualization_config.output_format}"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def render_frames_as_video
|
61
|
+
puts "\nGenerating video:" if $visualization_config.verbose
|
62
|
+
output = `ffmpeg -r #{$visualization_config.frames_per_second} -qscale 1 -y -i animation.%09d.bmp animation.#{$visualization_config.output_format} 2>&1`
|
63
|
+
puts output if $visualization_config.verbose
|
64
|
+
raise 'could not create the animation' unless $?.exitstatus == 0
|
65
|
+
end
|
66
|
+
|
67
|
+
def visualize
|
68
|
+
if $visualization_config.animate
|
69
|
+
generate_animation
|
70
|
+
else
|
71
|
+
generate_image
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# see http://en.wikipedia.org/wiki/Exponential_decay
|
2
|
+
class Decay
|
3
|
+
def initialize initial_value, lifetime
|
4
|
+
@initial_value = initial_value.to_f
|
5
|
+
@lifetime = lifetime.to_f
|
6
|
+
end
|
7
|
+
|
8
|
+
def value time
|
9
|
+
@initial_value * Math::exp(-time / @lifetime)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
|
3
|
+
module FileUtils
|
4
|
+
def self.zipped? filename
|
5
|
+
%w[.gz .Z].include? File.extname(filename)
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.open filename, &block
|
9
|
+
if zipped? filename
|
10
|
+
Zlib::GzipReader.open(filename, &block)
|
11
|
+
else
|
12
|
+
File.open(filename, &block)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'RMagick'
|
2
|
+
|
3
|
+
# Draws the timestamp.
|
4
|
+
class InformationDrawer
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@draw = Magick::Draw.new
|
8
|
+
end
|
9
|
+
|
10
|
+
def draw_info image, visualization, info
|
11
|
+
size = { :width => visualization.map_size[:width], :height => visualization.map_size[:height] }
|
12
|
+
draw_background(size)
|
13
|
+
draw_message(size, info)
|
14
|
+
@draw.draw(image)
|
15
|
+
end
|
16
|
+
|
17
|
+
protected
|
18
|
+
|
19
|
+
def draw_background size
|
20
|
+
width, height = size[:width], size[:height]
|
21
|
+
@draw.fill('grey')
|
22
|
+
@draw.fill_opacity('50%')
|
23
|
+
@draw.rectangle(0.2 * width, 0.9 * height, 0.8 * width, 0.9 * height + 30)
|
24
|
+
end
|
25
|
+
|
26
|
+
def draw_message size, info
|
27
|
+
@draw.fill('black')
|
28
|
+
@draw.fill_opacity('100%')
|
29
|
+
@draw.text_align(Magick::CenterAlign)
|
30
|
+
@draw.pointsize(20)
|
31
|
+
@draw.text(0.5 * size[:width], 0.9 * size[:height] + 20, info)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'typhoeus'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
class IpLookupService
|
6
|
+
|
7
|
+
def initialize filename = nil
|
8
|
+
@filename = filename || File.join(File.dirname(__FILE__), '..', '..', 'resources', 'coordinates.yml')
|
9
|
+
reset
|
10
|
+
end
|
11
|
+
|
12
|
+
def reset
|
13
|
+
@host_coordinates = {}
|
14
|
+
@host_ips = {}
|
15
|
+
end
|
16
|
+
|
17
|
+
def coordinates_for_hosts hosts
|
18
|
+
uniq_hosts = hosts.uniq
|
19
|
+
|
20
|
+
uniq_hosts.each do |host|
|
21
|
+
@host_ips[host] ||= IPSocket.getaddress(host) rescue nil
|
22
|
+
end
|
23
|
+
|
24
|
+
hydra = Typhoeus::Hydra.new
|
25
|
+
uniq_hosts.each do |host|
|
26
|
+
unless @host_coordinates[host]
|
27
|
+
request = Typhoeus::Request.new("http://api.hostip.info/get_html.php?position=true&ip=#{@host_ips[host]}")
|
28
|
+
request.on_complete do |response|
|
29
|
+
@host_coordinates[host] = extract_longitude_and_latitude(response.body)
|
30
|
+
end
|
31
|
+
hydra.queue(request)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
hydra.run
|
35
|
+
|
36
|
+
@host_coordinates
|
37
|
+
end
|
38
|
+
|
39
|
+
def coordinates_for_host host
|
40
|
+
unless @host_coordinates[host]
|
41
|
+
@host_ips[host] ||= IPSocket.getaddress(host) rescue nil
|
42
|
+
response = Net::HTTP.get('api.hostip.info', "/get_html.php?position=true&ip=#{@host_ips[host]}")
|
43
|
+
@host_coordinates[host] = extract_longitude_and_latitude(response)
|
44
|
+
end
|
45
|
+
|
46
|
+
@host_coordinates[host]
|
47
|
+
end
|
48
|
+
|
49
|
+
def extract_longitude_and_latitude string
|
50
|
+
latitude = string.match(/Latitude: (-?[0-9.]+)/)[1].to_f rescue nil
|
51
|
+
longitude = string.match(/Longitude: (-?[0-9.]+)/)[1].to_f rescue nil
|
52
|
+
[longitude, latitude]
|
53
|
+
end
|
54
|
+
|
55
|
+
def save_coordinates
|
56
|
+
File.open(@filename, 'w'){ |f| f.write @host_coordinates.to_yaml }
|
57
|
+
end
|
58
|
+
|
59
|
+
def load_coordinates
|
60
|
+
@host_coordinates = YAML.load_file(@filename) if File.readable? @filename
|
61
|
+
end
|
62
|
+
|
63
|
+
def stats
|
64
|
+
unknown_coordinates, known_coordinates = @host_coordinates.partition{ |ip, coords| coords.include? nil }
|
65
|
+
|
66
|
+
{ :unknown_coordinates => unknown_coordinates.size, :known_coordinates => known_coordinates.size }
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
@@ -0,0 +1,60 @@
|
|
1
|
+
class LogAnalyzer
|
2
|
+
attr_accessor :host_coordinates
|
3
|
+
|
4
|
+
def initialize *filenames
|
5
|
+
@filenames = filenames.flatten.sort.uniq
|
6
|
+
@ip_lookup_service = IpLookupService.new
|
7
|
+
@ip_lookup_service.load_coordinates
|
8
|
+
end
|
9
|
+
|
10
|
+
def analyze
|
11
|
+
details = []
|
12
|
+
|
13
|
+
puts "\nReading files:" if $visualization_config.verbose
|
14
|
+
@filenames.each do |filename|
|
15
|
+
puts filename if $visualization_config.verbose
|
16
|
+
FileUtils.open(filename).each_line do |line|
|
17
|
+
details << details_from_line(line)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
hosts = details.collect{|detail| detail[:host]}
|
22
|
+
coordinates = @ip_lookup_service.coordinates_for_hosts(hosts)
|
23
|
+
|
24
|
+
details.collect! do |detail|
|
25
|
+
detail[:coordinates] = coordinates[detail[:host]]
|
26
|
+
detail
|
27
|
+
end
|
28
|
+
|
29
|
+
details
|
30
|
+
end
|
31
|
+
|
32
|
+
def details_from_line line
|
33
|
+
host = extract_host_from_line(line)
|
34
|
+
time = extract_time_from_line(line)
|
35
|
+
{ :time => time, :host => host }
|
36
|
+
end
|
37
|
+
|
38
|
+
def calculate_oldest_time details
|
39
|
+
details.min{ |a, b| a[:time] <=> b[:time] }[:time]
|
40
|
+
end
|
41
|
+
|
42
|
+
def group_by_time details, slot_in_seconds
|
43
|
+
return {} unless details && slot_in_seconds
|
44
|
+
details_per_slot = {}
|
45
|
+
|
46
|
+
# TODO: maybe assign empty arrays to missing slots where no traffic was detected
|
47
|
+
details.each do |detail|
|
48
|
+
slot_start_time = calculate_slot_start_time(detail[:time], slot_in_seconds)
|
49
|
+
details_per_slot[slot_start_time] ||= []
|
50
|
+
details_per_slot[slot_start_time] << detail
|
51
|
+
end
|
52
|
+
|
53
|
+
details_per_slot
|
54
|
+
end
|
55
|
+
|
56
|
+
def calculate_slot_start_time time, slot_in_seconds
|
57
|
+
Time.at(time.tv_sec - (time.tv_sec % slot_in_seconds))
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class LogfileMock
|
2
|
+
attr_accessor :min_random_positions, :max_random_positions
|
3
|
+
|
4
|
+
def initialize
|
5
|
+
@min_random_positions = 5
|
6
|
+
@max_random_positions = 50
|
7
|
+
end
|
8
|
+
|
9
|
+
def random_longitude_latitude
|
10
|
+
[ -10 + rand(25) + rand, 40 + rand(15) + rand ]
|
11
|
+
end
|
12
|
+
|
13
|
+
def positions
|
14
|
+
amount = @min_random_positions + rand(@max_random_positions - @min_random_positions + 1)
|
15
|
+
Array.new(amount){ random_longitude_latitude }
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
class PointInTime
|
2
|
+
attr_reader :x, :y
|
3
|
+
|
4
|
+
def initialize x, y, initial_opacity, lifetime
|
5
|
+
@x, @y = x, y
|
6
|
+
@time = 0
|
7
|
+
@decay = Decay.new(initial_opacity, lifetime)
|
8
|
+
end
|
9
|
+
|
10
|
+
def opacity_in_time time
|
11
|
+
@time = time if time
|
12
|
+
opacity
|
13
|
+
end
|
14
|
+
|
15
|
+
def opacity
|
16
|
+
@decay.value(@time)
|
17
|
+
end
|
18
|
+
|
19
|
+
def age
|
20
|
+
@time += 1
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'RMagick'
|
2
|
+
|
3
|
+
class Visualization
|
4
|
+
attr_accessor :position_quantization_in_degrees, :circle_radius
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@map_filename = $visualization_config.map_filename
|
8
|
+
@raw_image = Magick::ImageList.new(@map_filename).first
|
9
|
+
if $visualization_config.map_width || $visualization_config.map_height
|
10
|
+
width = $visualization_config.map_width || @raw_image.columns
|
11
|
+
height = $visualization_config.map_height || @raw_image.rows
|
12
|
+
@raw_image.resize! width, height
|
13
|
+
end
|
14
|
+
new_frame
|
15
|
+
@position_quantization_in_degrees = 0
|
16
|
+
@opacity_visibility_threshold = 0.1
|
17
|
+
@circle_radius = (map_size[:width] ** 1.25) / (map_size[:width] * $visualization_config.fill_dot_scale).to_f
|
18
|
+
@points = {}
|
19
|
+
end
|
20
|
+
|
21
|
+
def map_size
|
22
|
+
@map_size ||= { :width => @frame.columns, :height => @frame.rows }
|
23
|
+
end
|
24
|
+
|
25
|
+
def scale
|
26
|
+
@scale ||= { :x => 360.0 / map_size[:width], :y => 180.0 / map_size[:height] }
|
27
|
+
end
|
28
|
+
|
29
|
+
def x_y_from_longitude_latitude longitude, latitude
|
30
|
+
[ (180 + longitude) / scale[:x], (90 - latitude) / scale[:y] ]
|
31
|
+
end
|
32
|
+
|
33
|
+
def circle_parameters center_x, center_y
|
34
|
+
[ center_x, center_y, center_x + circle_radius, center_y ]
|
35
|
+
end
|
36
|
+
|
37
|
+
def quantize_position *position
|
38
|
+
return position if @position_quantization_in_degrees == 0
|
39
|
+
position.collect{ |element| element - element.remainder(@position_quantization_in_degrees) }
|
40
|
+
end
|
41
|
+
|
42
|
+
def select_visible_points points
|
43
|
+
#points.select{ |point| point.opacity >= @opacity_visibility_threshold }
|
44
|
+
selected_points = {}
|
45
|
+
points.each{ |key, point| selected_points[key] = point if point.opacity >= @opacity_visibility_threshold }
|
46
|
+
end
|
47
|
+
|
48
|
+
def draw_positions positions_lon_lat
|
49
|
+
@draw.fill($visualization_config.fill_dot_color)
|
50
|
+
|
51
|
+
new_points = {}
|
52
|
+
positions_lon_lat.each do |longitude, latitude|
|
53
|
+
x, y = x_y_from_longitude_latitude(longitude, latitude)
|
54
|
+
x, y = quantize_position(x, y)
|
55
|
+
new_points[[x,y]] = PointInTime.new(x, y, $visualization_config.fill_dot_opacity, $visualization_config.fill_dot_lifetime)
|
56
|
+
end
|
57
|
+
|
58
|
+
@points = select_visible_points(@points)
|
59
|
+
@points.merge!(new_points)
|
60
|
+
|
61
|
+
@points.each do |key, point|
|
62
|
+
@draw.fill_opacity(point.opacity)
|
63
|
+
@draw.circle(*circle_parameters(point.x, point.y))
|
64
|
+
point.age
|
65
|
+
end
|
66
|
+
|
67
|
+
@draw.draw(@frame)
|
68
|
+
@frame
|
69
|
+
end
|
70
|
+
|
71
|
+
def display
|
72
|
+
@frame.display
|
73
|
+
end
|
74
|
+
|
75
|
+
def new_frame
|
76
|
+
@draw = Magick::Draw.new
|
77
|
+
@frame = @raw_image.clone
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
Binary file
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Decay do
|
4
|
+
it 'should be initialized with either Integers and/or Floats' do
|
5
|
+
Decay.new(10, 5 )
|
6
|
+
Decay.new(10, 5.0)
|
7
|
+
Decay.new(10.0, 5 )
|
8
|
+
Decay.new(10.0, 5.0)
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should return a value depending on the time' do
|
12
|
+
decay = Decay.new(10, 5)
|
13
|
+
|
14
|
+
decay.value(0).should == 10.0
|
15
|
+
decay.value(5).should < 5.0
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe IpLookupService do
|
4
|
+
describe 'hostname -> coordinates resolval' do
|
5
|
+
it 'should return the coordinates for an IP' do
|
6
|
+
stubbed_result = "Country: GERMANY (DE)\nCity: Berlin\n\nLatitude: 52.5\nLongitude: 13.4167\nIP: 1.1.1.1\n"
|
7
|
+
Net::HTTP.stub!(:get).and_return(stubbed_result)
|
8
|
+
|
9
|
+
IpLookupService.new.coordinates_for_host('1.1.1.1').should == [13.4167, 52.5]
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'should return the coordinates for a hostname' do
|
13
|
+
stubbed_result = "Country: GERMANY (DE)\nCity: Berlin\n\nLatitude: 52.5\nLongitude: 13.4167\nIP: 1.1.1.1\n"
|
14
|
+
Net::HTTP.stub!(:get).and_return(stubbed_result)
|
15
|
+
IPSocket.stub!(:getaddr).and_return('1.1.1.1')
|
16
|
+
|
17
|
+
IpLookupService.new.coordinates_for_host('some hostname').should == [13.4167, 52.5]
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should handle the case that no IP can be resolved for a hostname' do
|
21
|
+
stubbed_result = "Country: GERMANY (DE)\nCity: Berlin\n\nLatitude: \nLongitude: \nIP: 1.1.1.1\n"
|
22
|
+
Net::HTTP.stub!(:get).and_return(stubbed_result)
|
23
|
+
IPSocket.stub!(:getaddrinfo).and_raise(SocketError)
|
24
|
+
|
25
|
+
IpLookupService.new.coordinates_for_host('some hostname').should == [nil, nil]
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should handle the case that no coordinates can be resolved for a hostname' do
|
29
|
+
stubbed_result = "Country: GERMANY (DE)\nCity: Berlin\n\nLatitude: \nLongitude: \nIP: 1.1.1.1\n"
|
30
|
+
Net::HTTP.stub!(:get).and_return(stubbed_result)
|
31
|
+
IPSocket.stub!(:getaddr)
|
32
|
+
|
33
|
+
IpLookupService.new.coordinates_for_host('some hostname').should == [nil, nil]
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe ApacheLogVisualizer do
|
4
|
+
describe 'time format' do
|
5
|
+
it 'should return a minute based time format for times within an hour' do
|
6
|
+
now = Time.now
|
7
|
+
times = [now, now + 3599]
|
8
|
+
|
9
|
+
ApacheLogVisualizer.detect_time_format(times).should == '%b %d %Y %H:%M'
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'should return a hour based time format for times within a day' do
|
13
|
+
now = Time.now
|
14
|
+
times = [now, now + 86399]
|
15
|
+
|
16
|
+
ApacheLogVisualizer.detect_time_format(times).should == '%b %d %Y %H:00'
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'should return a day based time format for times greater than a day' do
|
20
|
+
now = Time.now
|
21
|
+
times = [now, now + 86400]
|
22
|
+
|
23
|
+
ApacheLogVisualizer.detect_time_format(times).should == '%b %d %Y'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ip-world-map
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 23
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
- 0
|
10
|
+
version: 1.0.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- "Ren\xC3\xA9 Scheibe"
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-07-26 00:00:00 +02:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: rmagick
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 57
|
30
|
+
segments:
|
31
|
+
- 2
|
32
|
+
- 13
|
33
|
+
- 1
|
34
|
+
version: 2.13.1
|
35
|
+
type: :runtime
|
36
|
+
version_requirements: *id001
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: rspec
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
hash: 23
|
46
|
+
segments:
|
47
|
+
- 2
|
48
|
+
- 6
|
49
|
+
- 0
|
50
|
+
version: 2.6.0
|
51
|
+
type: :development
|
52
|
+
version_requirements: *id002
|
53
|
+
description: A tool to generate images/videos of user locations based on Apache log files.
|
54
|
+
email: rene.scheibe@gmail.com
|
55
|
+
executables:
|
56
|
+
- ip-world-map
|
57
|
+
extensions: []
|
58
|
+
|
59
|
+
extra_rdoc_files: []
|
60
|
+
|
61
|
+
files:
|
62
|
+
- lib/ip-world-map/information_drawer.rb
|
63
|
+
- lib/ip-world-map/logfile_mock.rb
|
64
|
+
- lib/ip-world-map/decay.rb
|
65
|
+
- lib/ip-world-map/log_analyzer.rb
|
66
|
+
- lib/ip-world-map/apache_log_analyzer.rb
|
67
|
+
- lib/ip-world-map/apache_log_visualizer.rb
|
68
|
+
- lib/ip-world-map/point_in_time.rb
|
69
|
+
- lib/ip-world-map/ip_lookup_service.rb
|
70
|
+
- lib/ip-world-map/file_utils.rb
|
71
|
+
- lib/ip-world-map/visualization.rb
|
72
|
+
- lib/ip-world-map.rb
|
73
|
+
- spec/ip-world-map/decay.rb
|
74
|
+
- spec/ip-world-map/time_format.rb
|
75
|
+
- spec/ip-world-map/ip_lookup_service.rb
|
76
|
+
- spec/spec_helper.rb
|
77
|
+
- Rakefile
|
78
|
+
- resources/maps/earthmap-1920x960.tif
|
79
|
+
- bin/ip-world-map
|
80
|
+
has_rdoc: true
|
81
|
+
homepage: http://github.com/darxriggs/ip-world-map
|
82
|
+
licenses:
|
83
|
+
- GPL-2
|
84
|
+
post_install_message:
|
85
|
+
rdoc_options: []
|
86
|
+
|
87
|
+
require_paths:
|
88
|
+
- lib
|
89
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
hash: 57
|
95
|
+
segments:
|
96
|
+
- 1
|
97
|
+
- 8
|
98
|
+
- 7
|
99
|
+
version: 1.8.7
|
100
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
101
|
+
none: false
|
102
|
+
requirements:
|
103
|
+
- - ">="
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
hash: 3
|
106
|
+
segments:
|
107
|
+
- 0
|
108
|
+
version: "0"
|
109
|
+
requirements:
|
110
|
+
- ImageMagick (used by rmagick)
|
111
|
+
- ffmpeg (only for animations)
|
112
|
+
rubyforge_project:
|
113
|
+
rubygems_version: 1.3.7
|
114
|
+
signing_key:
|
115
|
+
specification_version: 3
|
116
|
+
summary: A tool to generate images/videos of user locations based on Apache log files.
|
117
|
+
test_files: []
|
118
|
+
|