mihari 1.4.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +44 -0
  3. data/README.md +6 -7
  4. data/Rakefile +1 -0
  5. data/docker/Dockerfile +1 -1
  6. data/lib/mihari/alert_viewer.rb +3 -3
  7. data/lib/mihari/analyzers/base.rb +1 -1
  8. data/lib/mihari/analyzers/basic.rb +3 -4
  9. data/lib/mihari/analyzers/binaryedge.rb +4 -7
  10. data/lib/mihari/analyzers/censys.rb +3 -7
  11. data/lib/mihari/analyzers/circl.rb +3 -5
  12. data/lib/mihari/analyzers/crtsh.rb +2 -6
  13. data/lib/mihari/analyzers/dnpedia.rb +3 -6
  14. data/lib/mihari/analyzers/dnstwister.rb +4 -9
  15. data/lib/mihari/analyzers/free_text.rb +2 -6
  16. data/lib/mihari/analyzers/http_hash.rb +3 -11
  17. data/lib/mihari/analyzers/onyphe.rb +3 -6
  18. data/lib/mihari/analyzers/otx.rb +4 -9
  19. data/lib/mihari/analyzers/passive_dns.rb +4 -9
  20. data/lib/mihari/analyzers/passive_ssl.rb +4 -9
  21. data/lib/mihari/analyzers/passivetotal.rb +9 -14
  22. data/lib/mihari/analyzers/pulsedive.rb +7 -12
  23. data/lib/mihari/analyzers/reverse_whois.rb +4 -9
  24. data/lib/mihari/analyzers/securitytrails.rb +12 -17
  25. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +3 -7
  26. data/lib/mihari/analyzers/shodan.rb +5 -8
  27. data/lib/mihari/analyzers/spyse.rb +6 -11
  28. data/lib/mihari/analyzers/ssh_fingerprint.rb +2 -6
  29. data/lib/mihari/analyzers/urlscan.rb +4 -12
  30. data/lib/mihari/analyzers/virustotal.rb +6 -11
  31. data/lib/mihari/analyzers/zoomeye.rb +7 -11
  32. data/lib/mihari/cli.rb +7 -7
  33. data/lib/mihari/config.rb +1 -25
  34. data/lib/mihari/database.rb +1 -1
  35. data/lib/mihari/emitters/misp.rb +4 -2
  36. data/lib/mihari/emitters/slack.rb +18 -7
  37. data/lib/mihari/emitters/the_hive.rb +2 -2
  38. data/lib/mihari/errors.rb +2 -0
  39. data/lib/mihari/models/artifact.rb +1 -1
  40. data/lib/mihari/notifiers/exception_notifier.rb +5 -5
  41. data/lib/mihari/status.rb +1 -1
  42. data/lib/mihari/type_checker.rb +4 -4
  43. data/lib/mihari/version.rb +1 -1
  44. data/mihari.gemspec +17 -19
  45. metadata +15 -43
  46. data/.travis.yml +0 -13
@@ -5,12 +5,7 @@ require "virustotal"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class VirusTotal < Base
8
- attr_reader :indicator
9
- attr_reader :type
10
-
11
- attr_reader :title
12
- attr_reader :description
13
- attr_reader :tags
8
+ attr_reader :indicator, :type, :title, :description, :tags
14
9
 
15
10
  def initialize(indicator, title: nil, description: nil, tags: [])
16
11
  super()
@@ -30,7 +25,7 @@ module Mihari
30
25
  private
31
26
 
32
27
  def config_keys
33
- %w(virustotal_api_key)
28
+ %w[virustotal_api_key]
34
29
  end
35
30
 
36
31
  def api
@@ -38,7 +33,7 @@ module Mihari
38
33
  end
39
34
 
40
35
  def valid_type?
41
- %w(ip domain).include? type
36
+ %w[ip domain].include? type
42
37
  end
43
38
 
44
39
  def lookup
@@ -48,14 +43,14 @@ module Mihari
48
43
  when "ip"
49
44
  ip_lookup
50
45
  else
51
- raise InvalidInputError, "#{indicator}(type: #{type || 'unknown'}) is not supported." unless valid_type?
46
+ raise InvalidInputError, "#{indicator}(type: #{type || "unknown"}) is not supported." unless valid_type?
52
47
  end
53
48
  end
54
49
 
55
50
  def domain_lookup
56
51
  res = api.domain.resolutions(indicator)
57
52
 
58
- data = res.dig("data") || []
53
+ data = res["data"] || []
59
54
  data.map do |item|
60
55
  item.dig("attributes", "ip_address")
61
56
  end.compact.uniq
@@ -64,7 +59,7 @@ module Mihari
64
59
  def ip_lookup
65
60
  res = api.ip_address.resolutions(indicator)
66
61
 
67
- data = res.dig("data") || []
62
+ data = res["data"] || []
68
63
  data.map do |item|
69
64
  item.dig("attributes", "host_name")
70
65
  end.compact.uniq
@@ -5,11 +5,7 @@ require "zoomeye"
5
5
  module Mihari
6
6
  module Analyzers
7
7
  class ZoomEye < Base
8
- attr_reader :title
9
- attr_reader :description
10
- attr_reader :query
11
- attr_reader :tags
12
- attr_reader :type
8
+ attr_reader :title, :description, :query, :tags, :type
13
9
 
14
10
  def initialize(query, title: nil, description: nil, tags: [], type: "host")
15
11
  super()
@@ -37,11 +33,11 @@ module Mihari
37
33
  PAGE_SIZE = 10
38
34
 
39
35
  def valid_type?
40
- %w(host web).include? type
36
+ %w[host web].include? type
41
37
  end
42
38
 
43
39
  def config_keys
44
- %w(zoomeye_password zoomeye_username)
40
+ %w[zoomeye_password zoomeye_username]
45
41
  end
46
42
 
47
43
  def api
@@ -50,9 +46,9 @@ module Mihari
50
46
 
51
47
  def convert_responses(responses)
52
48
  responses.map do |res|
53
- matches = res.dig("matches") || []
49
+ matches = res["matches"] || []
54
50
  matches.map do |match|
55
- match.dig "ip"
51
+ match["ip"]
56
52
  end
57
53
  end.flatten.compact.uniq
58
54
  end
@@ -69,7 +65,7 @@ module Mihari
69
65
  res = _host_lookup(query, page: page)
70
66
  break unless res
71
67
 
72
- total = res.dig("total").to_i
68
+ total = res["total"].to_i
73
69
  responses << res
74
70
  break if total <= page * PAGE_SIZE
75
71
  end
@@ -88,7 +84,7 @@ module Mihari
88
84
  res = _web_lookup(query, page: page)
89
85
  break unless res
90
86
 
91
- total = res.dig("total").to_i
87
+ total = res["total"].to_i
92
88
  responses << res
93
89
  break if total <= page * PAGE_SIZE
94
90
  end
data/lib/mihari/cli.rb CHANGED
@@ -259,16 +259,16 @@ module Mihari
259
259
  desc "import_from_json", "Give a JSON input via STDIN"
260
260
  def import_from_json(input = nil)
261
261
  with_error_handling do
262
- json = input || STDIN.gets.chomp
262
+ json = input || $stdin.gets.chomp
263
263
  raise ArgumentError, "Input not found: please give an input in a JSON format" unless json
264
264
 
265
265
  json = parse_as_json(json)
266
266
  raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless valid_json?(json)
267
267
 
268
- title = json.dig("title")
269
- description = json.dig("description")
270
- artifacts = json.dig("artifacts")
271
- tags = json.dig("tags") || []
268
+ title = json["title"]
269
+ description = json["description"]
270
+ artifacts = json["artifacts"]
271
+ tags = json["tags"] || []
272
272
 
273
273
  basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
274
274
  basic.run
@@ -302,7 +302,7 @@ module Mihari
302
302
  no_commands do
303
303
  def with_error_handling
304
304
  yield
305
- rescue StandardError => e
305
+ rescue => e
306
306
  notifier = Notifiers::ExceptionNotifier.new
307
307
  notifier.notify e
308
308
  end
@@ -315,7 +315,7 @@ module Mihari
315
315
 
316
316
  # @return [true, false]
317
317
  def valid_json?(json)
318
- %w(title description artifacts).all? { |key| json.key? key }
318
+ %w[title description artifacts].all? { |key| json.key? key }
319
319
  end
320
320
 
321
321
  def load_configuration
data/lib/mihari/config.rb CHANGED
@@ -4,31 +4,7 @@ require "yaml"
4
4
 
5
5
  module Mihari
6
6
  class Config
7
- attr_accessor :binaryedge_api_key
8
- attr_accessor :censys_id
9
- attr_accessor :censys_secret
10
- attr_accessor :circl_passive_password
11
- attr_accessor :circl_passive_username
12
- attr_accessor :misp_api_endpoint
13
- attr_accessor :misp_api_key
14
- attr_accessor :onyphe_api_key
15
- attr_accessor :otx_api_key
16
- attr_accessor :passivetotal_api_key
17
- attr_accessor :passivetotal_username
18
- attr_accessor :pulsedive_api_key
19
- attr_accessor :securitytrails_api_key
20
- attr_accessor :shodan_api_key
21
- attr_accessor :slack_channel
22
- attr_accessor :slack_webhook_url
23
- attr_accessor :spyse_api_key
24
- attr_accessor :thehive_api_endpoint
25
- attr_accessor :thehive_api_key
26
- attr_accessor :urlscan_api_key
27
- attr_accessor :virustotal_api_key
28
- attr_accessor :zoomeye_password
29
- attr_accessor :zoomeye_username
30
-
31
- attr_accessor :database
7
+ attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_password, :zoomeye_username, :database
32
8
 
33
9
  def initialize
34
10
  load_from_env
@@ -54,7 +54,7 @@ module Mihari
54
54
 
55
55
  ActiveRecord::Migration.verbose = false
56
56
  InitialSchema.migrate(:up)
57
- rescue StandardError
57
+ rescue
58
58
  # Do nothing
59
59
  end
60
60
 
@@ -7,6 +7,8 @@ module Mihari
7
7
  module Emitters
8
8
  class MISP < Base
9
9
  def initialize
10
+ super()
11
+
10
12
  ::MISP.configure do |config|
11
13
  config.api_endpoint = Mihari.config.misp_api_endpoint
12
14
  config.api_key = Mihari.config.misp_api_key
@@ -35,7 +37,7 @@ module Mihari
35
37
  private
36
38
 
37
39
  def config_keys
38
- %w(misp_api_endpoint misp_api_key)
40
+ %w[misp_api_endpoint misp_api_key]
39
41
  end
40
42
 
41
43
  def build_attribute(artifact)
@@ -61,7 +63,7 @@ module Mihari
61
63
  ip: "ip-dst",
62
64
  mail: "email-dst",
63
65
  url: "url",
64
- domain: "domain",
66
+ domain: "domain"
65
67
  }
66
68
  return table[type] if table.key?(type)
67
69
 
@@ -19,25 +19,31 @@ module Mihari
19
19
  end
20
20
 
21
21
  def actions
22
- [vt_link, urlscan_link, censys_link].compact
22
+ [vt_link, urlscan_link, censys_link, shodan_link].compact
23
23
  end
24
24
 
25
25
  def vt_link
26
26
  return nil unless _vt_link
27
27
 
28
- { type: "button", text: "Lookup on VirusTotal", url: _vt_link, }
28
+ { type: "button", text: "VirusTotal", url: _vt_link }
29
29
  end
30
30
 
31
31
  def urlscan_link
32
32
  return nil unless _urlscan_link
33
33
 
34
- { type: "button", text: "Lookup on urlscan.io", url: _urlscan_link, }
34
+ { type: "button", text: "urlscan.io", url: _urlscan_link }
35
35
  end
36
36
 
37
37
  def censys_link
38
38
  return nil unless _censys_link
39
39
 
40
- { type: "button", text: "Lookup on Censys", url: _censys_link, }
40
+ { type: "button", text: "Censys", url: _censys_link }
41
+ end
42
+
43
+ def shodan_link
44
+ return nil unless _shodan_link
45
+
46
+ { type: "button", text: "Shodan", url: _shodan_link }
41
47
  end
42
48
 
43
49
  # @return [Array]
@@ -89,6 +95,11 @@ module Mihari
89
95
  end
90
96
  memoize :_censys_link
91
97
 
98
+ def _shodan_link
99
+ data_type == "ip" ? "https://www.shodan.io/host/#{data}" : nil
100
+ end
101
+ memoize :_shodan_link
102
+
92
103
  # @return [String]
93
104
  def sha256
94
105
  Digest::SHA256.hexdigest data
@@ -96,7 +107,7 @@ module Mihari
96
107
 
97
108
  # @return [String]
98
109
  def defanged_data
99
- @defanged_data ||= data.to_s.gsub /\./, "[.]"
110
+ @defanged_data ||= data.to_s.gsub(/\./, "[.]")
100
111
  end
101
112
  end
102
113
 
@@ -121,7 +132,7 @@ module Mihari
121
132
  [
122
133
  "*#{title}*",
123
134
  "*Desc.*: #{description}",
124
- "*Tags*: #{tags.join(', ')}",
135
+ "*Tags*: #{tags.join(', ')}"
125
136
  ].join("\n")
126
137
  end
127
138
 
@@ -137,7 +148,7 @@ module Mihari
137
148
  private
138
149
 
139
150
  def config_keys
140
- %w(slack_webhook_url)
151
+ %w[slack_webhook_url]
141
152
  end
142
153
  end
143
154
  end
@@ -17,7 +17,7 @@ module Mihari
17
17
  api.alert.create(
18
18
  title: title,
19
19
  description: description,
20
- artifacts: artifacts.map { |artifact| { data: artifact.data, data_type: artifact.data_type, message: description } },
20
+ artifacts: artifacts.map { |artifact| {data: artifact.data, data_type: artifact.data_type, message: description} },
21
21
  tags: tags,
22
22
  type: "external",
23
23
  source: "mihari"
@@ -27,7 +27,7 @@ module Mihari
27
27
  private
28
28
 
29
29
  def config_keys
30
- %w(thehive_api_endpoint thehive_api_key)
30
+ %w[thehive_api_endpoint thehive_api_key]
31
31
  end
32
32
 
33
33
  def api
data/lib/mihari/errors.rb CHANGED
@@ -2,6 +2,8 @@
2
2
 
3
3
  module Mihari
4
4
  class Error < StandardError; end
5
+
5
6
  class InvalidInputError < Error; end
7
+
6
8
  class RetryableError < Error; end
7
9
  end
@@ -6,7 +6,7 @@ class ArtifactValidator < ActiveModel::Validator
6
6
  def validate(record)
7
7
  return if record.data_type
8
8
 
9
- record.errors[:data] << "#{record.data} is not supported"
9
+ record.errors.add :data, "#{record.data} is not supported"
10
10
  end
11
11
  end
12
12
 
@@ -19,7 +19,7 @@ module Mihari
19
19
  def notify(exception)
20
20
  notify_to_stdout exception
21
21
 
22
- clean_message = exception.message.tr('`', "'")
22
+ clean_message = exception.message.tr("`", "'")
23
23
  attachments = to_attachments(exception, clean_message)
24
24
  notify_to_slack(text: clean_message, attachments: attachments) if @slack.valid?
25
25
  end
@@ -51,20 +51,20 @@ module Mihari
51
51
 
52
52
  def to_fields(clean_message, backtrace)
53
53
  fields = [
54
- { title: "Exception", value: clean_message },
55
- { title: "Hostname", value: hostname }
54
+ {title: "Exception", value: clean_message},
55
+ {title: "Hostname", value: hostname}
56
56
  ]
57
57
 
58
58
  if backtrace
59
59
  formatted_backtrace = format_backtrace(backtrace)
60
- fields << { title: "Backtrace", value: formatted_backtrace }
60
+ fields << {title: "Backtrace", value: formatted_backtrace}
61
61
  end
62
62
  fields
63
63
  end
64
64
 
65
65
  def hostname
66
66
  Socket.gethostname
67
- rescue StandardError => _e
67
+ rescue => _e
68
68
  "N/A"
69
69
  end
70
70
 
data/lib/mihari/status.rb CHANGED
@@ -36,7 +36,7 @@ module Mihari
36
36
  status = instance.configured?
37
37
  message = instance.configuration_status
38
38
 
39
- message ? { status: status, message: message } : nil
39
+ message ? {status: status, message: message} : nil
40
40
  rescue ArgumentError => _e
41
41
  nil
42
42
  end
@@ -80,22 +80,22 @@ module Mihari
80
80
 
81
81
  # @return [true, false]
82
82
  def md5?
83
- data.match? /^[A-Fa-f0-9]{32}$/
83
+ data.match?(/^[A-Fa-f0-9]{32}$/)
84
84
  end
85
85
 
86
86
  # @return [true, false]
87
87
  def sha1?
88
- data.match? /^[A-Fa-f0-9]{40}$/
88
+ data.match?(/^[A-Fa-f0-9]{40}$/)
89
89
  end
90
90
 
91
91
  # @return [true, false]
92
92
  def sha256?
93
- data.match? /^[A-Fa-f0-9]{64}$/
93
+ data.match?(/^[A-Fa-f0-9]{64}$/)
94
94
  end
95
95
 
96
96
  # @return [true, false]
97
97
  def sha512?
98
- data.match? /^[A-Fa-f0-9]{128}$/
98
+ data.match?(/^[A-Fa-f0-9]{128}$/)
99
99
  end
100
100
  end
101
101
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- VERSION = "1.4.1"
4
+ VERSION = "1.5.0"
5
5
  end
data/mihari.gemspec CHANGED
@@ -1,41 +1,39 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- lib = File.expand_path('lib', __dir__)
3
+ lib = File.expand_path("lib", __dir__)
4
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
  require "mihari/version"
6
6
 
7
7
  Gem::Specification.new do |spec|
8
- spec.name = "mihari"
9
- spec.version = Mihari::VERSION
10
- spec.authors = ["Manabu Niseki"]
11
- spec.email = ["manabu.niseki@gmail.com"]
8
+ spec.name = "mihari"
9
+ spec.version = Mihari::VERSION
10
+ spec.authors = ["Manabu Niseki"]
11
+ spec.email = ["manabu.niseki@gmail.com"]
12
12
 
13
- spec.summary = "A framework for continuous malicious hosts monitoring."
14
- spec.description = "A framework for continuous malicious hosts monitoring."
15
- spec.homepage = "https://github.com/ninoseki/mihari"
16
- spec.license = "MIT"
13
+ spec.summary = "A framework for continuous malicious hosts monitoring."
14
+ spec.description = "A framework for continuous malicious hosts monitoring."
15
+ spec.homepage = "https://github.com/ninoseki/mihari"
16
+ spec.license = "MIT"
17
17
 
18
18
  # Specify which files should be added to the gem when it is released.
19
19
  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
20
- spec.files = Dir.chdir(File.expand_path(__dir__)) do
20
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
21
21
  `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
22
22
  end
23
- spec.bindir = "exe"
24
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
23
+ spec.bindir = "exe"
24
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
25
25
  spec.require_paths = ["lib"]
26
26
 
27
- spec.add_development_dependency "bundler", "~> 2.1"
27
+ spec.add_development_dependency "bundler", "~> 2.2"
28
28
  spec.add_development_dependency "coveralls", "~> 0.8"
29
29
  spec.add_development_dependency "execjs", "~> 2.7"
30
- spec.add_development_dependency "fakefs", "~> 1.2"
31
- spec.add_development_dependency "pre-commit", "~> 0.39"
30
+ spec.add_development_dependency "fakefs", "~> 1.3"
32
31
  spec.add_development_dependency "rake", "~> 13.0"
33
32
  spec.add_development_dependency "rspec", "~> 3.10"
34
- spec.add_development_dependency "rubocop", "~> 1.6.0"
35
- spec.add_development_dependency "rubocop-performance", "~> 1.9"
33
+ spec.add_development_dependency "standard", "~> 1.0"
36
34
  spec.add_development_dependency "timecop", "~> 0.9"
37
35
  spec.add_development_dependency "vcr", "~> 6.0"
38
- spec.add_development_dependency "webmock", "~> 3.10"
36
+ spec.add_development_dependency "webmock", "~> 3.12"
39
37
 
40
38
  spec.add_dependency "active_model_serializers", "~> 0.10"
41
39
  spec.add_dependency "activerecord", "~> 6.1"
@@ -64,7 +62,7 @@ Gem::Specification.new do |spec|
64
62
  spec.add_dependency "slack-notifier", "~> 2.3"
65
63
  spec.add_dependency "spysex", "~> 0.1"
66
64
  spec.add_dependency "sqlite3", "~> 1.4"
67
- spec.add_dependency "thor", "~> 1.0"
65
+ spec.add_dependency "thor", "~> 1.1"
68
66
  spec.add_dependency "thread_safe", "~> 0.3"
69
67
  spec.add_dependency "urlscan", "~> 0.6"
70
68
  spec.add_dependency "virustotalx", "~> 1.1"