cert_open_data_visualizer 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +48 -0
- data/Rakefile +1 -0
- data/bin/cert_open_data_visualizer +10 -0
- data/cert_open_data_visualizer.gemspec +24 -0
- data/lib/cert_open_data_visualizer.rb +41 -0
- data/lib/cert_open_data_visualizer/cli.rb +46 -0
- data/lib/cert_open_data_visualizer/dummy_cacher.rb +59 -0
- data/lib/cert_open_data_visualizer/formatter.rb +85 -0
- data/lib/cert_open_data_visualizer/version.rb +3 -0
- data/lib/cert_open_data_visualizer/visualize.rb +62 -0
- metadata +108 -0
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Jarmo Isotalo
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
# CertOpenDataVisualizer
|
2
|
+
|
3
|
+
|
4
|
+
## Installation
|
5
|
+
|
6
|
+
Add this line to your application's Gemfile:
|
7
|
+
|
8
|
+
gem 'cert_open_data_visualizer'
|
9
|
+
|
10
|
+
And then execute:
|
11
|
+
|
12
|
+
$ bundle
|
13
|
+
|
14
|
+
Or install it yourself as:
|
15
|
+
|
16
|
+
$ gem install cert_open_data_visualizer
|
17
|
+
|
18
|
+
## Usage
|
19
|
+
|
20
|
+
### Example
|
21
|
+
```ruby
|
22
|
+
require 'cert_open_data_visualizer'
|
23
|
+
|
24
|
+
# this will fetch and parse data
|
25
|
+
visualizer = CertDataVisualizer.new
|
26
|
+
visualizer.first_format
|
27
|
+
visualizer.second_format
|
28
|
+
|
29
|
+
# Will crear cache. After this you need explicitely redownload everything (or create new instance)
|
30
|
+
visualiser.crear_cache!
|
31
|
+
# will redownload and reparse everything if needed
|
32
|
+
visualizer.fetch!
|
33
|
+
|
34
|
+
```
|
35
|
+
|
36
|
+
If one really wants to play around with this gem, one may try using this gem in irb `irb -Ilib -rcert_open_data_visualizer`
|
37
|
+
Ofcourse one may install this gem and require it in irb.
|
38
|
+
Or just use this from any sourse code
|
39
|
+
|
40
|
+
## Contributing
|
41
|
+
|
42
|
+
1. Fork it
|
43
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
44
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
45
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
46
|
+
5. Create new Pull Request
|
47
|
+
|
48
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'cert_open_data_visualizer/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "cert_open_data_visualizer"
|
8
|
+
spec.version = CertOpenDataVisualizer::VERSION
|
9
|
+
spec.authors = ["Jarmo Isotalo"]
|
10
|
+
spec.email = ["jamo@isotalo.fi"]
|
11
|
+
spec.description = %q{Jarmo Isotalo's application project for software factory}
|
12
|
+
spec.summary = %q{Library and commandline client for processing CERT.fi open data - https://www.cert.fi/en/reports/statistics/opendata.html}
|
13
|
+
spec.homepage = "https://github.com/jamox/cert"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split("\n")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_dependency "httparty", "~> 0.12.0 "
|
22
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
23
|
+
spec.add_development_dependency "rake"
|
24
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require "cert_open_data_visualizer/version"
|
2
|
+
require "cert_open_data_visualizer/visualize"
|
3
|
+
require "cert_open_data_visualizer/dummy_cacher"
|
4
|
+
require "cert_open_data_visualizer/formatter"
|
5
|
+
require "cert_open_data_visualizer/cli"
|
6
|
+
|
7
|
+
class CertDataVisualizer
|
8
|
+
attr_accessor :formatter
|
9
|
+
def initialize
|
10
|
+
@formatter = CertOpenDataVisualizer::Formatter.new
|
11
|
+
end
|
12
|
+
|
13
|
+
def first_format
|
14
|
+
@formatter.first_format
|
15
|
+
end
|
16
|
+
|
17
|
+
def print_first_format
|
18
|
+
puts first_format
|
19
|
+
end
|
20
|
+
|
21
|
+
def second_format
|
22
|
+
@formatter.second_format
|
23
|
+
end
|
24
|
+
|
25
|
+
def print_second_format
|
26
|
+
puts second_format
|
27
|
+
end
|
28
|
+
|
29
|
+
def clean_cache!
|
30
|
+
CertOpenDataVisualizer::DummyCacher.new.clean!
|
31
|
+
end
|
32
|
+
|
33
|
+
def fetch!
|
34
|
+
@formatter.data = CertOpenDataVisualizer::Visualize.new.parse.all_data
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
module CertOpenDataVisualizer
|
40
|
+
# Your code goes here...
|
41
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module CertOpenDataVisualizer
|
2
|
+
class CLI
|
3
|
+
attr_accessor :argv
|
4
|
+
|
5
|
+
def initialize(argv)
|
6
|
+
@argv = argv
|
7
|
+
end
|
8
|
+
|
9
|
+
def start
|
10
|
+
visualizer = CertDataVisualizer.new
|
11
|
+
commands = {fetch: "fetch!",
|
12
|
+
clean: "clean_cache!",
|
13
|
+
crear: "clean_cache!",
|
14
|
+
first: "print_first_format",
|
15
|
+
second: "print_second_format",
|
16
|
+
help: "print_help"}
|
17
|
+
command = @argv[0].to_sym
|
18
|
+
exec = commands[command]
|
19
|
+
|
20
|
+
return help if exec == "print_help"
|
21
|
+
return visualizer.send(exec) if exec
|
22
|
+
puts "Invalid command #{command}"
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
def help
|
27
|
+
str <<<EOF
|
28
|
+
CertOpenDataVisualizer
|
29
|
+
Commands are:
|
30
|
+
first - prints results in format required for first print
|
31
|
+
second - prints results in format required for second print
|
32
|
+
help - shows this
|
33
|
+
clean - cleanes cache
|
34
|
+
fetch - fetches new data if neccessary
|
35
|
+
|
36
|
+
`first` and `second` may download and parse data when neccesary.
|
37
|
+
|
38
|
+
This includes a file cache, for faster results.
|
39
|
+
ie. information is calculated only when necessary, and cached for
|
40
|
+
further use. Cache may be cleared using `clean`
|
41
|
+
EOF
|
42
|
+
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module CertOpenDataVisualizer
|
2
|
+
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
class DummyCacher
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
FileUtils.mkdir_p(path)
|
9
|
+
end
|
10
|
+
|
11
|
+
# Yep, well just overwrite it if it exists
|
12
|
+
def cache_file(path)
|
13
|
+
FileUtils.mkdir_p( File.join(Dir.tmpdir, "cert"))
|
14
|
+
FileUtils.cp(path, File.join(Dir.tmpdir, "cert"))
|
15
|
+
end
|
16
|
+
|
17
|
+
def write_file_to_cache(filename, contents)
|
18
|
+
Dir.chdir(path) do
|
19
|
+
File.open(filename, "wb") { |file| file.write(contents) }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def get_from_cache(file_name)
|
24
|
+
File.join(path, file_name)
|
25
|
+
end
|
26
|
+
|
27
|
+
def file_exists?(file_name)
|
28
|
+
File.exists? get_from_cache(file_name)
|
29
|
+
end
|
30
|
+
|
31
|
+
def tmpdir_path
|
32
|
+
Dir.tmpdir
|
33
|
+
end
|
34
|
+
|
35
|
+
def unzip_file(file_name)
|
36
|
+
Dir.chdir(path) do
|
37
|
+
# Because I have found rubyzip to be buggy, we rely that your system has zip-command available
|
38
|
+
`unzip -o #{file_name}`
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def find_files_matching(matcher)
|
43
|
+
files = []
|
44
|
+
Dir.chdir(path) do
|
45
|
+
files = Dir.glob(matcher)
|
46
|
+
end
|
47
|
+
files
|
48
|
+
end
|
49
|
+
|
50
|
+
def path
|
51
|
+
File.join(Dir.tmpdir, "cert")
|
52
|
+
end
|
53
|
+
|
54
|
+
def clean!
|
55
|
+
FileUtils.rm_rf path
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
module CertOpenDataVisualizer
|
2
|
+
class Formatter
|
3
|
+
attr_accessor :data, :cacher, :visualizer
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@visualizer = Visualize.new
|
7
|
+
@cacher = @visualizer.cacher
|
8
|
+
end
|
9
|
+
|
10
|
+
# 0: date from
|
11
|
+
# 1: date to
|
12
|
+
# 2: anon AS-number
|
13
|
+
# 3: anon IP-address
|
14
|
+
# 4: IP version
|
15
|
+
# 5: maincategory
|
16
|
+
# 6: subcategory
|
17
|
+
# 7: cc
|
18
|
+
# 8: city
|
19
|
+
def first_format
|
20
|
+
if @cacher.file_exists?("first_challenge")
|
21
|
+
return File.read(@cacher.get_from_cache("first_challenge"))
|
22
|
+
else
|
23
|
+
load_data! if @data.nil?
|
24
|
+
end
|
25
|
+
incidents = count_main_incidents
|
26
|
+
incidents_str = ""
|
27
|
+
incidents.each do |incident|
|
28
|
+
incidents_str << "#{incident[0]} #{incident[1]}\n"
|
29
|
+
end
|
30
|
+
@cacher.write_file_to_cache("first_challenge", incidents_str)
|
31
|
+
incidents_str
|
32
|
+
end
|
33
|
+
|
34
|
+
def second_format
|
35
|
+
if @cacher.file_exists?("second_challenge")
|
36
|
+
return File.read(@cacher.get_from_cache("second_challenge"))
|
37
|
+
else
|
38
|
+
load_data! if @data.nil?
|
39
|
+
end
|
40
|
+
incidents = count_incidents_based_on_location
|
41
|
+
incidents_str = ""
|
42
|
+
incidents.each do |incident|
|
43
|
+
incidents_str << "#{incident[0]} #{incident[1]}\n"
|
44
|
+
end
|
45
|
+
@cacher.write_file_to_cache("second_challenge", incidents_str)
|
46
|
+
incidents_str
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
private
|
51
|
+
def count_main_incidents
|
52
|
+
incidents = {}
|
53
|
+
load_data! if @data.nil?
|
54
|
+
@data.each do |row|
|
55
|
+
main_incident = row[5]
|
56
|
+
if incidents[main_incident]
|
57
|
+
incidents[main_incident] = incidents[main_incident] + 1
|
58
|
+
else
|
59
|
+
incidents[main_incident] = 1
|
60
|
+
end
|
61
|
+
end
|
62
|
+
incidents
|
63
|
+
end
|
64
|
+
|
65
|
+
def count_incidents_based_on_location
|
66
|
+
incidents = {}
|
67
|
+
load_data! if @data.nil?
|
68
|
+
@data.each do |row|
|
69
|
+
ccode, city = row[7], row[8]
|
70
|
+
name = "#{ccode}#{city}"
|
71
|
+
if incidents[name]
|
72
|
+
incidents[name] = incidents[name] + 1
|
73
|
+
else
|
74
|
+
incidents[name] = 1
|
75
|
+
end
|
76
|
+
end
|
77
|
+
incidents
|
78
|
+
end
|
79
|
+
|
80
|
+
def load_data!
|
81
|
+
@data = @visualizer.parse.all_data
|
82
|
+
end
|
83
|
+
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
module CertOpenDataVisualizer
|
2
|
+
|
3
|
+
require 'httparty'
|
4
|
+
require 'tmpdir'
|
5
|
+
require 'csv'
|
6
|
+
class Visualize
|
7
|
+
attr_accessor :all_data, :cacher
|
8
|
+
CSV_DATA_URL = "http://pilvilinna.cert.fi/opendata/autoreporter/csv.zip"
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@cacher = DummyCacher.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def parse
|
15
|
+
maybe_download
|
16
|
+
files = extract_and_list_files
|
17
|
+
read_and_merge_files(files)
|
18
|
+
self
|
19
|
+
end
|
20
|
+
|
21
|
+
def maybe_download
|
22
|
+
if @cacher.file_exists?("cert.zip")
|
23
|
+
@cacher.get_from_cache("cert.zip")
|
24
|
+
else
|
25
|
+
download
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def download
|
30
|
+
puts "Downloading, may take a while depending on your connection"
|
31
|
+
data = HTTParty.get(CSV_DATA_URL).body
|
32
|
+
write_tmp_file(data)
|
33
|
+
end
|
34
|
+
|
35
|
+
def write_tmp_file(contents)
|
36
|
+
@cacher.write_file_to_cache "cert.zip", contents
|
37
|
+
end
|
38
|
+
|
39
|
+
def extract_and_list_files
|
40
|
+
@cacher.unzip_file("cert.zip")
|
41
|
+
@cacher.find_files_matching("*.csv")
|
42
|
+
end
|
43
|
+
|
44
|
+
def read_and_merge_files(files)
|
45
|
+
@all_data = get_csvs(files)
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
def get_csvs(files)
|
50
|
+
puts "Parsing data, it may take a while"
|
51
|
+
csvs = []
|
52
|
+
Dir.chdir (@cacher.path) do
|
53
|
+
files.each do |file|
|
54
|
+
csvs += CSV.read file, col_sep: "|"
|
55
|
+
end
|
56
|
+
csvs.reject! {|row| row[0].include?('#') || row[5].nil? || row[5] == ""}
|
57
|
+
csvs
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
metadata
ADDED
@@ -0,0 +1,108 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: cert_open_data_visualizer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.3
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Jarmo Isotalo
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-12-01 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: httparty
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 0.12.0
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.12.0
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: bundler
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '1.3'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '1.3'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: rake
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
description: Jarmo Isotalo's application project for software factory
|
63
|
+
email:
|
64
|
+
- jamo@isotalo.fi
|
65
|
+
executables:
|
66
|
+
- cert_open_data_visualizer
|
67
|
+
extensions: []
|
68
|
+
extra_rdoc_files: []
|
69
|
+
files:
|
70
|
+
- Gemfile
|
71
|
+
- LICENSE.txt
|
72
|
+
- README.md
|
73
|
+
- Rakefile
|
74
|
+
- bin/cert_open_data_visualizer
|
75
|
+
- cert_open_data_visualizer.gemspec
|
76
|
+
- lib/cert_open_data_visualizer.rb
|
77
|
+
- lib/cert_open_data_visualizer/cli.rb
|
78
|
+
- lib/cert_open_data_visualizer/dummy_cacher.rb
|
79
|
+
- lib/cert_open_data_visualizer/formatter.rb
|
80
|
+
- lib/cert_open_data_visualizer/version.rb
|
81
|
+
- lib/cert_open_data_visualizer/visualize.rb
|
82
|
+
homepage: https://github.com/jamox/cert
|
83
|
+
licenses:
|
84
|
+
- MIT
|
85
|
+
post_install_message:
|
86
|
+
rdoc_options: []
|
87
|
+
require_paths:
|
88
|
+
- lib
|
89
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ! '>='
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
96
|
+
none: false
|
97
|
+
requirements:
|
98
|
+
- - ! '>='
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: '0'
|
101
|
+
requirements: []
|
102
|
+
rubyforge_project:
|
103
|
+
rubygems_version: 1.8.23
|
104
|
+
signing_key:
|
105
|
+
specification_version: 3
|
106
|
+
summary: Library and commandline client for processing CERT.fi open data - https://www.cert.fi/en/reports/statistics/opendata.html
|
107
|
+
test_files: []
|
108
|
+
has_rdoc:
|