mihari 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4a0fd2207c7c92a24cbeda2fb328ecca19db331aa00132b3f6d361d6f8b4b6fd
4
- data.tar.gz: 77b42d18d1509f6a33be27d38064298b726fa3244364d468714926d2878c3e89
3
+ metadata.gz: f8c53264f15a01502a5a872c013db461b718655608941c88af53c0424261de76
4
+ data.tar.gz: 1f6146bce1c0eaacfcd688cb33794f974433f8418d7f8eb38513ef338b5914ac
5
5
  SHA512:
6
- metadata.gz: 14bbf6da4dffb24e0fb3c696bc664970a33784298b7fd09d24c9e792262b27a8eebd229fa1b0d17218e1116590acda49b56d4afff0bbc147c8c661dc545df811
7
- data.tar.gz: 308620e95164284b5370985899ed524705b64808a2ceb073af404e619901a0dc00277653e2e0eb41e83d5a9db4678577436fe28a78c5e8accb2534dd5362ae22
6
+ metadata.gz: f8e79477f488c81f84a9c594bbcfec1e50aa60b66f3e8c0f8a7e71672efc22cceb0ddbd62330b9962e091878f364c9a0475356b54b1924465cd433579530a4f9
7
+ data.tar.gz: 966b523e74b2769501a613c2b4adeda2d521791a8b6376fb6e47dc8d60fe1b41316e077cb1bc4a95a630397f398b345c5f12d56298581128714c60b2076a7de3
data/README.md CHANGED
@@ -19,7 +19,7 @@ mihari(`見張り`) is a sidekick tool for [TheHive](https://github.com/TheHive-
19
19
 
20
20
  ![img](./screenshots/eyecatch.png)
21
21
 
22
- Check this blog post for more detail: [Continuous C2 hunting with Censys, Shodan, Onyphe and TheHive](https://hackmd.io/s/SkUaSrqoE).
22
+ Check this blog post for more details: [Continuous C2 hunting with Censys, Shodan, Onyphe and TheHive](https://hackmd.io/s/SkUaSrqoE).
23
23
 
24
24
  You can use mihari without TheHive. But note that mihari depends on TheHive to manage artifacts. It means mihari might make duplications when without TheHive.
25
25
 
@@ -51,7 +51,7 @@ docker pull ninoseki/mihari
51
51
 
52
52
  ## Basic usage
53
53
 
54
- mihari supports Censys, Shodan, Onyphe, urlscan, SecurityTrails, crt.sh, CIRCL passive DNS/SSL, PassiveTotal and VirusTotal by default.
54
+ mihari supports Censys, Shodan, Onyphe, urlscan, SecurityTrails, crt.sh, CIRCL passive DNS/SSL, PassiveTotal, VirusTotal and ZoomEye by default.
55
55
 
56
56
  ```bash
57
57
  $ mihari
@@ -71,6 +71,7 @@ Commands:
71
71
  mihari status # Show the current configuration status
72
72
  mihari urlscan [QUERY] # urlscan lookup by a given query
73
73
  mihari virustotal [IP|DOMAIN] # VirusTotal resolutions lookup by a given ip or domain
74
+ mihari zoommeye [QUERY] # ZoomEye lookup by a given query
74
75
 
75
76
  ```
76
77
 
@@ -91,7 +92,7 @@ $ mihari censys '("PANDA" AND "SMAdmin" AND "layui")' --title "PANDA C2"
91
92
  "tags": []
92
93
  }
93
94
 
94
- # VirusTotal passive DNS lookup for a FAKESPY host
95
+ # VirusTotal passive DNS lookup of a FAKESPY host
95
96
  $ mihari virustotal "jppost-hi.top" --title "FAKESPY host passive DNS results"
96
97
  {
97
98
  "title": "FAKESPY host passive DNS results",
@@ -105,6 +106,9 @@ $ mihari virustotal "jppost-hi.top" --title "FAKESPY host passive DNS results"
105
106
  "tags": []
106
107
  }
107
108
 
109
+ # You can pass a "defanged" indicator as an input
110
+ $ mihari virustotal "jppost-hi[.]top" --title "FAKESPY host passive DNS results"
111
+
108
112
  # SecurityTrails domain feed lookup for finding (possibly) Apple phishing websites
109
113
  mihari securitytrails_domain_feed "apple-" --type new
110
114
  {
@@ -158,7 +162,7 @@ All configuration is done via ENV variables.
158
162
  | SLACK_CHANNEL | Slack channel name | Optional (default: `#general`) |
159
163
  | CENSYS_ID | Censys API ID | Optional |
160
164
  | CENSYS_SECRET | Censys secret | Optional |
161
- | CIRCL_PASSIVE_PASSWORD | CIRC_ passive DNS/SSL password | Optional |
165
+ | CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password | Optional |
162
166
  | CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username | Optional |
163
167
  | ONYPHE_API_KEY | Onyphe API key | Optional |
164
168
  | PASSIVETOTAL_API_KEY | PassiveTotal API key | Optional |
@@ -166,6 +170,8 @@ All configuration is done via ENV variables.
166
170
  | SECURITYTRAILS_API_KEY | SecurityTrails API key | Optional |
167
171
  | SHODAN_API_KEY | Shodan API key | Optional |
168
172
  | VIRUSTOTAL_API_KEY | VirusTotal API key | Optional |
173
+ | ZOOMEYE_USERNAMME | ZoomEye username | Optional |
174
+ | ZOOMEYE_PASSWORD | ZoomEye password | Optional |
169
175
 
170
176
  You can check the configuration status via `status` command.
171
177
 
@@ -46,6 +46,7 @@ require "mihari/analyzers/securitytrails"
46
46
  require "mihari/analyzers/shodan"
47
47
  require "mihari/analyzers/urlscan"
48
48
  require "mihari/analyzers/virustotal"
49
+ require "mihari/analyzers/zoomeye"
49
50
 
50
51
  require "mihari/notifiers/base"
51
52
  require "mihari/notifiers/slack"
@@ -9,17 +9,44 @@ module Mihari
9
9
  attr_reader :description
10
10
  attr_reader :query
11
11
  attr_reader :tags
12
+ attr_reader :type
12
13
 
13
- def initialize(query, title: nil, description: nil, tags: [])
14
+ def initialize(query, title: nil, description: nil, tags: [], type: "ipv4")
14
15
  super()
15
16
 
16
17
  @query = query
17
18
  @title = title || "Censys lookup"
18
19
  @description = description || "query = #{query}"
19
20
  @tags = tags
21
+ @type = type
20
22
  end
21
23
 
22
24
  def artifacts
25
+ case type
26
+ when "ipv4"
27
+ ipv4_lookup
28
+ when "websites"
29
+ websites_lookup
30
+ when "certificates"
31
+ certificates_lookup
32
+ else
33
+ raise TypeError, "#{type} type is not supported." unless valid_type?
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ def valid_type?
40
+ %w(ipv4 websites certificates).include? type
41
+ end
42
+
43
+ def normalize(domain)
44
+ return domain unless domain.start_with?("*.")
45
+
46
+ domain.sub("*.", "")
47
+ end
48
+
49
+ def ipv4_lookup
23
50
  ipv4s = []
24
51
 
25
52
  res = api.ipv4.search(query: query)
@@ -30,7 +57,33 @@ module Mihari
30
57
  ipv4s.flatten
31
58
  end
32
59
 
33
- private
60
+ def websites_lookup
61
+ domains = []
62
+
63
+ res = api.websites.search(query: query)
64
+ res.each_page do |page|
65
+ domains << page.map(&:domain)
66
+ end
67
+
68
+ domains.flatten
69
+ end
70
+
71
+ def certificates_lookup
72
+ domains = []
73
+
74
+ res = api.certificates.search(query: query)
75
+ res.each_page do |page|
76
+ page.each do |result|
77
+ subject_dn = result.subject_dn
78
+ names = subject_dn.scan(/CN=(.+)/).flatten.first
79
+ next unless names
80
+
81
+ domains << names.split(",").map { |domain| normalize(domain) }
82
+ end
83
+ end
84
+
85
+ domains.flatten
86
+ end
34
87
 
35
88
  def config_keys
36
89
  %w(CENSYS_ID CENSYS_SECRET)
@@ -41,7 +41,7 @@ module Mihari
41
41
  when "hash"
42
42
  passive_ssl_lookup
43
43
  else
44
- raise ArgumentError, "#{@query}(type: #{@type || 'unknown'}) is not supported."
44
+ raise TypeError, "#{@query}(type: #{@type || 'unknown'}) is not supported."
45
45
  end
46
46
  rescue ::PassiveCIRCL::Error => _e
47
47
  nil
@@ -52,7 +52,7 @@ module Mihari
52
52
  when "hash"
53
53
  ssl_lookup
54
54
  else
55
- raise ArgumentError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
55
+ raise TypeError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
56
56
  end
57
57
  rescue ::PassiveTotal::Error => _e
58
58
  nil
@@ -50,7 +50,7 @@ module Mihari
50
50
  when "mail"
51
51
  mail_lookup
52
52
  else
53
- raise ArgumentError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
53
+ raise TypeError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
54
54
  end
55
55
  rescue ::SecurityTrails::Error => _e
56
56
  nil
@@ -17,8 +17,8 @@ module Mihari
17
17
  @_regexp = regexp
18
18
  @type = type
19
19
 
20
- raise ArgumentError, "#{@_regexp} is not a valid regexp" unless regexp
21
- raise ArgumentError, "#{type} is not a valid type" unless valid_type?
20
+ raise TypeError, "#{@_regexp} is not a valid regexp" unless regexp
21
+ raise TypeError, "#{type} is not a valid type" unless valid_type?
22
22
 
23
23
  @title = title || "SecurityTrails domain feed lookup"
24
24
  @description = description || "Regexp = /#{@_regexp}/"
@@ -20,13 +20,15 @@ module Mihari
20
20
  end
21
21
 
22
22
  def artifacts
23
- result = search
24
- return [] unless result
25
-
26
- matches = result.dig("matches") || []
27
- matches.map do |match|
28
- match.dig "ip_str"
29
- end.compact
23
+ results = search
24
+ return [] unless results || results.empty?
25
+
26
+ results.map do |result|
27
+ matches = result.dig("matches") || []
28
+ matches.map do |match|
29
+ match.dig "ip_str"
30
+ end.compact
31
+ end.flatten.compact.uniq
30
32
  end
31
33
 
32
34
  private
@@ -39,11 +41,23 @@ module Mihari
39
41
  @api ||= ::Shodan::API.new
40
42
  end
41
43
 
42
- def search
43
- api.host.search(query)
44
+ def search_with_page(query, page: 1)
45
+ api.host.search(query, page: page)
44
46
  rescue ::Shodan::Error => _e
45
47
  nil
46
48
  end
49
+
50
+ def search
51
+ responses = []
52
+ (1..Float::INFINITY).each do |page|
53
+ res = search_with_page(query, page: page)
54
+ break unless res
55
+
56
+ responses << res
57
+ break if res.dig("total").to_i <= page * 100
58
+ end
59
+ responses
60
+ end
47
61
  end
48
62
  end
49
63
  end
@@ -20,7 +20,7 @@ module Mihari
20
20
  @tags = tags
21
21
  @target_type = target_type
22
22
 
23
- raise ArgumentError, "type should be url, domain or ip." unless valid_target_type?
23
+ raise TypeError, "type should be url, domain or ip." unless valid_target_type?
24
24
  end
25
25
 
26
26
  def artifacts
@@ -48,7 +48,7 @@ module Mihari
48
48
  when "ip"
49
49
  ip_lookup
50
50
  else
51
- raise ArgumentError, "#{indicator}(type: #{type || 'unknown'}) is not supported." unless valid_type?
51
+ raise TypeError, "#{indicator}(type: #{type || 'unknown'}) is not supported." unless valid_type?
52
52
  end
53
53
  rescue ::VirusTotal::Error => _e
54
54
  nil
@@ -0,0 +1,99 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "zoomeye"
4
+
5
+ module Mihari
6
+ module Analyzers
7
+ class ZoomEye < Base
8
+ attr_reader :title
9
+ attr_reader :description
10
+ attr_reader :query
11
+ attr_reader :tags
12
+ attr_reader :type
13
+
14
+ def initialize(query, title: nil, description: nil, tags: [], type: "host")
15
+ super()
16
+
17
+ @query = query
18
+ @title = title || "ZoomEye lookup"
19
+ @description = description || "query = #{query}"
20
+ @tags = tags
21
+ @type = type
22
+ end
23
+
24
+ def artifacts
25
+ case type
26
+ when "host"
27
+ host_lookup
28
+ when "web"
29
+ web_lookup
30
+ else
31
+ raise TypeError, "#{type} type is not supported." unless valid_type?
32
+ end
33
+ end
34
+
35
+ private
36
+
37
+ def valid_type?
38
+ %w(host web).include? type
39
+ end
40
+
41
+ def config_keys
42
+ %w(ZOOMEYE_USERNAME ZOOMEYE_PASSWORD)
43
+ end
44
+
45
+ def api
46
+ @api ||= ::ZoomEye::API.new
47
+ end
48
+
49
+ def convert_responses(responses)
50
+ responses.map do |res|
51
+ matches = res.dig("matches") || []
52
+ matches.map do |match|
53
+ match.dig "ip"
54
+ end
55
+ end.flatten.compact.uniq
56
+ end
57
+
58
+ def _host_lookup(query, page: 1)
59
+ api.host.search(query, page: page)
60
+ rescue ::ZoomEye::Error => _e
61
+ nil
62
+ end
63
+
64
+ def host_lookup
65
+ responses = []
66
+
67
+ res = _host_lookup(query)
68
+ total = res.dig("total").to_f
69
+ max = (total / 10.0).ceil
70
+ responses << res
71
+
72
+ (2..max).each do |page|
73
+ responses << _host_lookup(query, page: page)
74
+ end
75
+ convert_responses responses.compact
76
+ end
77
+
78
+ def _web_lookup(query, page: 1)
79
+ api.web.search(query, page: page)
80
+ rescue ::ZoomEye::Error => _e
81
+ nil
82
+ end
83
+
84
+ def web_lookup
85
+ responses = []
86
+
87
+ res = _web_lookup(query)
88
+ total = res.dig("total").to_f
89
+ max = (total / 10.0).ceil
90
+ responses << res
91
+
92
+ (2..max).each do |page|
93
+ responses << _web_lookup(query, page: page)
94
+ end
95
+ convert_responses responses.compact
96
+ end
97
+ end
98
+ end
99
+ end
@@ -9,6 +9,7 @@ module Mihari
9
9
  method_option :title, type: :string, desc: "title"
10
10
  method_option :description, type: :string, desc: "description"
11
11
  method_option :tags, type: :array, desc: "tags"
12
+ method_option :type, type: :string, desc: "type to search (ipv4 / websites / certificates)", default: "ipv4"
12
13
  def censys(query)
13
14
  with_error_handling do
14
15
  run_analyzer Analyzers::Censys, query: query, options: options
@@ -52,7 +53,7 @@ module Mihari
52
53
  method_option :tags, type: :array, desc: "tags"
53
54
  def virustotal(indiactor)
54
55
  with_error_handling do
55
- run_analyzer Analyzers::VirusTotal, query: indiactor, options: options
56
+ run_analyzer Analyzers::VirusTotal, query: refang(indiactor), options: options
56
57
  end
57
58
  end
58
59
 
@@ -62,7 +63,7 @@ module Mihari
62
63
  method_option :tags, type: :array, desc: "tags"
63
64
  def securitytrails(indiactor)
64
65
  with_error_handling do
65
- run_analyzer Analyzers::SecurityTrails, query: indiactor, options: options
66
+ run_analyzer Analyzers::SecurityTrails, query: refang(indiactor), options: options
66
67
  end
67
68
  end
68
69
  map "st" => :securitytrails
@@ -113,9 +114,20 @@ module Mihari
113
114
  method_option :title, type: :string, desc: "title"
114
115
  method_option :description, type: :string, desc: "description"
115
116
  method_option :tags, type: :array, desc: "tags"
116
- def passivetotal(query)
117
+ def passivetotal(indicator)
117
118
  with_error_handling do
118
- run_analyzer Analyzers::PassiveTotal, query: query, options: options
119
+ run_analyzer Analyzers::PassiveTotal, query: refang(indicator), options: options
120
+ end
121
+ end
122
+
123
+ desc "zoommeye [QUERY]", "ZoomEye lookup by a given query"
124
+ method_option :title, type: :string, desc: "title"
125
+ method_option :description, type: :string, desc: "description"
126
+ method_option :tags, type: :array, desc: "tags"
127
+ method_option :type, type: :string, desc: "type to search(host / web)", default: "host"
128
+ def zoomeye(query)
129
+ with_error_handling do
130
+ run_analyzer Analyzers::ZoomEye, query: query, options: options
119
131
  end
120
132
  end
121
133
 
@@ -184,6 +196,10 @@ module Mihari
184
196
  def symbolize_hash_keys(hash)
185
197
  hash.map{ |k, v| [k.to_sym, v] }.to_h
186
198
  end
199
+
200
+ def refang(indicator)
201
+ indicator.gsub("[.]", ".").gsub("(.)", ".")
202
+ end
187
203
  end
188
204
  end
189
205
  end
@@ -113,12 +113,21 @@ module Mihari
113
113
  end.flatten
114
114
  end
115
115
 
116
+ def to_text(title:, description:, tags: [])
117
+ tags = ["N/A"] if tags.empty?
118
+
119
+ [
120
+ "*#{title}*",
121
+ "*Desc.*: #{description}",
122
+ "*Tags*: #{tags.join(', ')}",
123
+ ].join("\n")
124
+ end
125
+
116
126
  def emit(title:, description:, artifacts:, tags: [])
117
127
  return if artifacts.empty?
118
128
 
119
129
  attachments = to_attachments(artifacts)
120
- tags = ["N/A"] if tags.empty?
121
- text = "#{title} (desc.: #{description} / tags: #{tags.join(', ')})"
130
+ text = to_text(title: title, description: description, tags: tags)
122
131
 
123
132
  notifier.notify(text: text, attachments: attachments)
124
133
  end
@@ -5,6 +5,7 @@ module Mihari
5
5
  class Slack < Base
6
6
  SLACK_WEBHOOK_URL_KEY = "SLACK_WEBHOOK_URL"
7
7
  SLACK_CHANNEL_KEY = "SLACK_CHANNEL"
8
+ DEFAULT_USERNAME = "mihari"
8
9
 
9
10
  def slack_channel
10
11
  ENV.fetch SLACK_CHANNEL_KEY, "#general"
@@ -22,9 +23,9 @@ module Mihari
22
23
  slack_webhook_url?
23
24
  end
24
25
 
25
- def notify(text:, attachments: [])
26
- notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
27
- notifier.post(text: text, attachments: attachments)
26
+ def notify(text:, attachments: [], mrkdwn: true)
27
+ notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
28
+ notifier.post(text: text, attachments: attachments, mrkdwn: true)
28
29
  end
29
30
  end
30
31
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- VERSION = "0.11.0"
4
+ VERSION = "0.12.0"
5
5
  end
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
28
28
  spec.add_development_dependency "coveralls", "~> 0.8"
29
29
  spec.add_development_dependency "fakefs", "~> 0.20"
30
30
  spec.add_development_dependency "rake", "~> 13.0"
31
- spec.add_development_dependency "rspec", "~> 3.8"
31
+ spec.add_development_dependency "rspec", "~> 3.9"
32
32
  spec.add_development_dependency "timecop", "~> 0.9"
33
33
  spec.add_development_dependency "vcr", "~> 5.0"
34
34
  spec.add_development_dependency "webmock", "~> 3.7"
@@ -54,4 +54,5 @@ Gem::Specification.new do |spec|
54
54
  spec.add_dependency "thor", "~> 0.20"
55
55
  spec.add_dependency "urlscan", "~> 0.4"
56
56
  spec.add_dependency "virustotalx", "~> 1.0"
57
+ spec.add_dependency "zoomeye-rb", "~> 0.1"
57
58
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mihari
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.11.0
4
+ version: 0.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-10-07 00:00:00.000000000 Z
11
+ date: 2019-10-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -72,14 +72,14 @@ dependencies:
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '3.8'
75
+ version: '3.9'
76
76
  type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '3.8'
82
+ version: '3.9'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: timecop
85
85
  requirement: !ruby/object:Gem::Requirement
@@ -416,6 +416,20 @@ dependencies:
416
416
  - - "~>"
417
417
  - !ruby/object:Gem::Version
418
418
  version: '1.0'
419
+ - !ruby/object:Gem::Dependency
420
+ name: zoomeye-rb
421
+ requirement: !ruby/object:Gem::Requirement
422
+ requirements:
423
+ - - "~>"
424
+ - !ruby/object:Gem::Version
425
+ version: '0.1'
426
+ type: :runtime
427
+ prerelease: false
428
+ version_requirements: !ruby/object:Gem::Requirement
429
+ requirements:
430
+ - - "~>"
431
+ - !ruby/object:Gem::Version
432
+ version: '0.1'
419
433
  description: A framework for continuous malicious hosts monitoring.
420
434
  email:
421
435
  - manabu.niseki@gmail.com
@@ -451,6 +465,7 @@ files:
451
465
  - lib/mihari/analyzers/shodan.rb
452
466
  - lib/mihari/analyzers/urlscan.rb
453
467
  - lib/mihari/analyzers/virustotal.rb
468
+ - lib/mihari/analyzers/zoomeye.rb
454
469
  - lib/mihari/artifact.rb
455
470
  - lib/mihari/cache.rb
456
471
  - lib/mihari/cli.rb