cert_open_data_visualizer 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in cert_open_data_visializer.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 Jarmo Isotalo
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,48 @@
1
+ # CertOpenDataVisualizer
2
+
3
+
4
+ ## Installation
5
+
6
+ Add this line to your application's Gemfile:
7
+
8
+ gem 'cert_open_data_visualizer'
9
+
10
+ And then execute:
11
+
12
+ $ bundle
13
+
14
+ Or install it yourself as:
15
+
16
+ $ gem install cert_open_data_visualizer
17
+
18
+ ## Usage
19
+
20
+ ### Example
21
+ ```ruby
22
+ require 'cert_open_data_visualizer'
23
+
24
+ # this will fetch and parse data
25
+ visualizer = CertDataVisualizer.new
26
+ visualizer.first_format
27
+ visualizer.second_format
28
+
29
+ # Will crear cache. After this you need explicitely redownload everything (or create new instance)
30
+ visualiser.crear_cache!
31
+ # will redownload and reparse everything if needed
32
+ visualizer.fetch!
33
+
34
+ ```
35
+
36
+ If one really wants to play around with this gem, one may try using this gem in irb `irb -Ilib -rcert_open_data_visualizer`
37
+ Ofcourse one may install this gem and require it in irb.
38
+ Or just use this from any sourse code
39
+
40
+ ## Contributing
41
+
42
+ 1. Fork it
43
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
44
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
45
+ 4. Push to the branch (`git push origin my-new-feature`)
46
+ 5. Create new Pull Request
47
+
48
+
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ begin
4
+ require 'cert_open_data_visualizer'
5
+ rescue LoadError
6
+ require 'rubygems'
7
+ require 'cert_open_data_visualizer'
8
+ end
9
+
10
+ CertOpenDataVisualizer::CLI.new(ARGV).start
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'cert_open_data_visualizer/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "cert_open_data_visualizer"
8
+ spec.version = CertOpenDataVisualizer::VERSION
9
+ spec.authors = ["Jarmo Isotalo"]
10
+ spec.email = ["jamo@isotalo.fi"]
11
+ spec.description = %q{Jarmo Isotalo's application project for software factory}
12
+ spec.summary = %q{Library and commandline client for processing CERT.fi open data - https://www.cert.fi/en/reports/statistics/opendata.html}
13
+ spec.homepage = "https://github.com/jamox/cert"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files`.split("\n")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_dependency "httparty", "~> 0.12.0 "
22
+ spec.add_development_dependency "bundler", "~> 1.3"
23
+ spec.add_development_dependency "rake"
24
+ end
@@ -0,0 +1,41 @@
1
+ require "cert_open_data_visualizer/version"
2
+ require "cert_open_data_visualizer/visualize"
3
+ require "cert_open_data_visualizer/dummy_cacher"
4
+ require "cert_open_data_visualizer/formatter"
5
+ require "cert_open_data_visualizer/cli"
6
+
7
+ class CertDataVisualizer
8
+ attr_accessor :formatter
9
+ def initialize
10
+ @formatter = CertOpenDataVisualizer::Formatter.new
11
+ end
12
+
13
+ def first_format
14
+ @formatter.first_format
15
+ end
16
+
17
+ def print_first_format
18
+ puts first_format
19
+ end
20
+
21
+ def second_format
22
+ @formatter.second_format
23
+ end
24
+
25
+ def print_second_format
26
+ puts second_format
27
+ end
28
+
29
+ def clean_cache!
30
+ CertOpenDataVisualizer::DummyCacher.new.clean!
31
+ end
32
+
33
+ def fetch!
34
+ @formatter.data = CertOpenDataVisualizer::Visualize.new.parse.all_data
35
+ end
36
+
37
+ end
38
+
39
+ module CertOpenDataVisualizer
40
+ # Your code goes here...
41
+ end
@@ -0,0 +1,46 @@
1
+ module CertOpenDataVisualizer
2
+ class CLI
3
+ attr_accessor :argv
4
+
5
+ def initialize(argv)
6
+ @argv = argv
7
+ end
8
+
9
+ def start
10
+ visualizer = CertDataVisualizer.new
11
+ commands = {fetch: "fetch!",
12
+ clean: "clean_cache!",
13
+ crear: "clean_cache!",
14
+ first: "print_first_format",
15
+ second: "print_second_format",
16
+ help: "print_help"}
17
+ command = @argv[0].to_sym
18
+ exec = commands[command]
19
+
20
+ return help if exec == "print_help"
21
+ return visualizer.send(exec) if exec
22
+ puts "Invalid command #{command}"
23
+ end
24
+
25
+
26
+ def help
27
+ str <<<EOF
28
+ CertOpenDataVisualizer
29
+ Commands are:
30
+ first - prints results in format required for first print
31
+ second - prints results in format required for second print
32
+ help - shows this
33
+ clean - cleanes cache
34
+ fetch - fetches new data if neccessary
35
+
36
+ `first` and `second` may download and parse data when neccesary.
37
+
38
+ This includes a file cache, for faster results.
39
+ ie. information is calculated only when necessary, and cached for
40
+ further use. Cache may be cleared using `clean`
41
+ EOF
42
+
43
+ end
44
+ end
45
+ end
46
+
@@ -0,0 +1,59 @@
1
+ module CertOpenDataVisualizer
2
+
3
+ require 'fileutils'
4
+
5
+ class DummyCacher
6
+
7
+ def initialize
8
+ FileUtils.mkdir_p(path)
9
+ end
10
+
11
+ # Yep, well just overwrite it if it exists
12
+ def cache_file(path)
13
+ FileUtils.mkdir_p( File.join(Dir.tmpdir, "cert"))
14
+ FileUtils.cp(path, File.join(Dir.tmpdir, "cert"))
15
+ end
16
+
17
+ def write_file_to_cache(filename, contents)
18
+ Dir.chdir(path) do
19
+ File.open(filename, "wb") { |file| file.write(contents) }
20
+ end
21
+ end
22
+
23
+ def get_from_cache(file_name)
24
+ File.join(path, file_name)
25
+ end
26
+
27
+ def file_exists?(file_name)
28
+ File.exists? get_from_cache(file_name)
29
+ end
30
+
31
+ def tmpdir_path
32
+ Dir.tmpdir
33
+ end
34
+
35
+ def unzip_file(file_name)
36
+ Dir.chdir(path) do
37
+ # Because I have found rubyzip to be buggy, we rely that your system has zip-command available
38
+ `unzip -o #{file_name}`
39
+ end
40
+ end
41
+
42
+ def find_files_matching(matcher)
43
+ files = []
44
+ Dir.chdir(path) do
45
+ files = Dir.glob(matcher)
46
+ end
47
+ files
48
+ end
49
+
50
+ def path
51
+ File.join(Dir.tmpdir, "cert")
52
+ end
53
+
54
+ def clean!
55
+ FileUtils.rm_rf path
56
+ end
57
+
58
+ end
59
+ end
@@ -0,0 +1,85 @@
1
+ module CertOpenDataVisualizer
2
+ class Formatter
3
+ attr_accessor :data, :cacher, :visualizer
4
+
5
+ def initialize
6
+ @visualizer = Visualize.new
7
+ @cacher = @visualizer.cacher
8
+ end
9
+
10
+ # 0: date from
11
+ # 1: date to
12
+ # 2: anon AS-number
13
+ # 3: anon IP-address
14
+ # 4: IP version
15
+ # 5: maincategory
16
+ # 6: subcategory
17
+ # 7: cc
18
+ # 8: city
19
+ def first_format
20
+ if @cacher.file_exists?("first_challenge")
21
+ return File.read(@cacher.get_from_cache("first_challenge"))
22
+ else
23
+ load_data! if @data.nil?
24
+ end
25
+ incidents = count_main_incidents
26
+ incidents_str = ""
27
+ incidents.each do |incident|
28
+ incidents_str << "#{incident[0]} #{incident[1]}\n"
29
+ end
30
+ @cacher.write_file_to_cache("first_challenge", incidents_str)
31
+ incidents_str
32
+ end
33
+
34
+ def second_format
35
+ if @cacher.file_exists?("second_challenge")
36
+ return File.read(@cacher.get_from_cache("second_challenge"))
37
+ else
38
+ load_data! if @data.nil?
39
+ end
40
+ incidents = count_incidents_based_on_location
41
+ incidents_str = ""
42
+ incidents.each do |incident|
43
+ incidents_str << "#{incident[0]} #{incident[1]}\n"
44
+ end
45
+ @cacher.write_file_to_cache("second_challenge", incidents_str)
46
+ incidents_str
47
+ end
48
+
49
+
50
+ private
51
+ def count_main_incidents
52
+ incidents = {}
53
+ load_data! if @data.nil?
54
+ @data.each do |row|
55
+ main_incident = row[5]
56
+ if incidents[main_incident]
57
+ incidents[main_incident] = incidents[main_incident] + 1
58
+ else
59
+ incidents[main_incident] = 1
60
+ end
61
+ end
62
+ incidents
63
+ end
64
+
65
+ def count_incidents_based_on_location
66
+ incidents = {}
67
+ load_data! if @data.nil?
68
+ @data.each do |row|
69
+ ccode, city = row[7], row[8]
70
+ name = "#{ccode}#{city}"
71
+ if incidents[name]
72
+ incidents[name] = incidents[name] + 1
73
+ else
74
+ incidents[name] = 1
75
+ end
76
+ end
77
+ incidents
78
+ end
79
+
80
+ def load_data!
81
+ @data = @visualizer.parse.all_data
82
+ end
83
+
84
+ end
85
+ end
@@ -0,0 +1,3 @@
1
+ module CertOpenDataVisualizer
2
+ VERSION = "0.0.3"
3
+ end
@@ -0,0 +1,62 @@
1
+ module CertOpenDataVisualizer
2
+
3
+ require 'httparty'
4
+ require 'tmpdir'
5
+ require 'csv'
6
+ class Visualize
7
+ attr_accessor :all_data, :cacher
8
+ CSV_DATA_URL = "http://pilvilinna.cert.fi/opendata/autoreporter/csv.zip"
9
+
10
+ def initialize
11
+ @cacher = DummyCacher.new
12
+ end
13
+
14
+ def parse
15
+ maybe_download
16
+ files = extract_and_list_files
17
+ read_and_merge_files(files)
18
+ self
19
+ end
20
+
21
+ def maybe_download
22
+ if @cacher.file_exists?("cert.zip")
23
+ @cacher.get_from_cache("cert.zip")
24
+ else
25
+ download
26
+ end
27
+ end
28
+
29
+ def download
30
+ puts "Downloading, may take a while depending on your connection"
31
+ data = HTTParty.get(CSV_DATA_URL).body
32
+ write_tmp_file(data)
33
+ end
34
+
35
+ def write_tmp_file(contents)
36
+ @cacher.write_file_to_cache "cert.zip", contents
37
+ end
38
+
39
+ def extract_and_list_files
40
+ @cacher.unzip_file("cert.zip")
41
+ @cacher.find_files_matching("*.csv")
42
+ end
43
+
44
+ def read_and_merge_files(files)
45
+ @all_data = get_csvs(files)
46
+ end
47
+
48
+ private
49
+ def get_csvs(files)
50
+ puts "Parsing data, it may take a while"
51
+ csvs = []
52
+ Dir.chdir (@cacher.path) do
53
+ files.each do |file|
54
+ csvs += CSV.read file, col_sep: "|"
55
+ end
56
+ csvs.reject! {|row| row[0].include?('#') || row[5].nil? || row[5] == ""}
57
+ csvs
58
+ end
59
+ end
60
+
61
+ end
62
+ end
metadata ADDED
@@ -0,0 +1,108 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: cert_open_data_visualizer
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.3
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Jarmo Isotalo
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-12-01 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: httparty
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: 0.12.0
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: 0.12.0
30
+ - !ruby/object:Gem::Dependency
31
+ name: bundler
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ~>
36
+ - !ruby/object:Gem::Version
37
+ version: '1.3'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ version: '1.3'
46
+ - !ruby/object:Gem::Dependency
47
+ name: rake
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ description: Jarmo Isotalo's application project for software factory
63
+ email:
64
+ - jamo@isotalo.fi
65
+ executables:
66
+ - cert_open_data_visualizer
67
+ extensions: []
68
+ extra_rdoc_files: []
69
+ files:
70
+ - Gemfile
71
+ - LICENSE.txt
72
+ - README.md
73
+ - Rakefile
74
+ - bin/cert_open_data_visualizer
75
+ - cert_open_data_visualizer.gemspec
76
+ - lib/cert_open_data_visualizer.rb
77
+ - lib/cert_open_data_visualizer/cli.rb
78
+ - lib/cert_open_data_visualizer/dummy_cacher.rb
79
+ - lib/cert_open_data_visualizer/formatter.rb
80
+ - lib/cert_open_data_visualizer/version.rb
81
+ - lib/cert_open_data_visualizer/visualize.rb
82
+ homepage: https://github.com/jamox/cert
83
+ licenses:
84
+ - MIT
85
+ post_install_message:
86
+ rdoc_options: []
87
+ require_paths:
88
+ - lib
89
+ required_ruby_version: !ruby/object:Gem::Requirement
90
+ none: false
91
+ requirements:
92
+ - - ! '>='
93
+ - !ruby/object:Gem::Version
94
+ version: '0'
95
+ required_rubygems_version: !ruby/object:Gem::Requirement
96
+ none: false
97
+ requirements:
98
+ - - ! '>='
99
+ - !ruby/object:Gem::Version
100
+ version: '0'
101
+ requirements: []
102
+ rubyforge_project:
103
+ rubygems_version: 1.8.23
104
+ signing_key:
105
+ specification_version: 3
106
+ summary: Library and commandline client for processing CERT.fi open data - https://www.cert.fi/en/reports/statistics/opendata.html
107
+ test_files: []
108
+ has_rdoc: