mihari 5.1.1 → 5.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/.gitmodules +0 -3
  3. data/.rubocop.yml +6 -0
  4. data/README.md +0 -1
  5. data/lib/mihari/analyzers/base.rb +32 -27
  6. data/lib/mihari/analyzers/binaryedge.rb +8 -2
  7. data/lib/mihari/analyzers/censys.rb +10 -61
  8. data/lib/mihari/analyzers/circl.rb +13 -19
  9. data/lib/mihari/analyzers/crtsh.rb +6 -0
  10. data/lib/mihari/analyzers/dnstwister.rb +12 -19
  11. data/lib/mihari/analyzers/feed.rb +21 -0
  12. data/lib/mihari/analyzers/greynoise.rb +5 -28
  13. data/lib/mihari/analyzers/onyphe.rb +8 -33
  14. data/lib/mihari/analyzers/otx.rb +11 -17
  15. data/lib/mihari/analyzers/passivetotal.rb +13 -19
  16. data/lib/mihari/analyzers/pulsedive.rb +3 -1
  17. data/lib/mihari/analyzers/rule.rb +0 -1
  18. data/lib/mihari/analyzers/securitytrails.rb +18 -29
  19. data/lib/mihari/analyzers/shodan.rb +13 -92
  20. data/lib/mihari/analyzers/urlscan.rb +12 -4
  21. data/lib/mihari/analyzers/virustotal.rb +4 -0
  22. data/lib/mihari/analyzers/virustotal_intelligence.rb +9 -6
  23. data/lib/mihari/analyzers/zoomeye.rb +9 -0
  24. data/lib/mihari/clients/binaryedge.rb +5 -0
  25. data/lib/mihari/clients/censys.rb +4 -4
  26. data/lib/mihari/clients/circl.rb +3 -3
  27. data/lib/mihari/clients/greynoise.rb +6 -1
  28. data/lib/mihari/clients/misp.rb +8 -1
  29. data/lib/mihari/clients/onyphe.rb +13 -1
  30. data/lib/mihari/clients/otx.rb +20 -0
  31. data/lib/mihari/clients/passivetotal.rb +6 -2
  32. data/lib/mihari/clients/publsedive.rb +18 -1
  33. data/lib/mihari/clients/securitytrails.rb +94 -0
  34. data/lib/mihari/clients/shodan.rb +14 -3
  35. data/lib/mihari/clients/the_hive.rb +6 -1
  36. data/lib/mihari/clients/urlscan.rb +3 -1
  37. data/lib/mihari/clients/virustotal.rb +9 -3
  38. data/lib/mihari/clients/zoomeye.rb +7 -1
  39. data/lib/mihari/commands/database.rb +1 -6
  40. data/lib/mihari/commands/searcher.rb +1 -2
  41. data/lib/mihari/database.rb +9 -0
  42. data/lib/mihari/http.rb +14 -18
  43. data/lib/mihari/structs/censys.rb +62 -0
  44. data/lib/mihari/structs/greynoise.rb +43 -0
  45. data/lib/mihari/structs/onyphe.rb +45 -0
  46. data/lib/mihari/structs/shodan.rb +83 -0
  47. data/lib/mihari/version.rb +1 -1
  48. data/lib/mihari/web/middleware/connection_adapter.rb +1 -3
  49. data/lib/mihari/web/public/assets/{index-63900d73.js → index-7d0fb8c4.js} +2 -2
  50. data/lib/mihari/web/public/index.html +1 -1
  51. data/lib/mihari/web/public/redoc-static.html +2 -2
  52. data/lib/mihari.rb +1 -3
  53. data/mihari.gemspec +2 -3
  54. metadata +8 -24
  55. data/lib/mihari/analyzers/dnpedia.rb +0 -33
  56. data/lib/mihari/clients/dnpedia.rb +0 -64
  57. data/lib/mihari/mixins/database.rb +0 -16
@@ -0,0 +1,94 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Clients
5
+ class SecurityTrails < Base
6
+ #
7
+ # @param [String] base_url
8
+ # @param [String, nil] api_key
9
+ # @param [Hash] headers
10
+ #
11
+ def initialize(base_url = "https://api.securitytrails.com", api_key:, headers: {})
12
+ raise(ArgumentError, "'api_key' argument is required") unless api_key
13
+
14
+ headers["apikey"] = api_key
15
+ super(base_url, headers: headers)
16
+ end
17
+
18
+ #
19
+ # @param [String] mail
20
+ #
21
+ # @return [Array<Hash>]
22
+ #
23
+ def search_by_mail(mail)
24
+ res = _post "/v1/domains/list", json: { filter: { whois_email: mail } }
25
+ res["records"] || []
26
+ end
27
+
28
+ #
29
+ # @param [String] ip
30
+ #
31
+ # @return [Array<Hash>]
32
+ #
33
+ def search_by_ip(ip)
34
+ res = _post "/v1/domains/list", json: { filter: { ipv4: ip } }
35
+ res["records"] || []
36
+ end
37
+
38
+ #
39
+ # @param [String] domain
40
+ # @param [String] type
41
+ #
42
+ # @return [Array<Hash>]
43
+ #
44
+ def get_all_dns_history(domain, type:)
45
+ first_page = get_dns_history(domain, type: type, page: 1)
46
+
47
+ pages = first_page["pages"].to_i
48
+ records = first_page["records"] || []
49
+
50
+ (2..pages).each do |page_idx|
51
+ next_page = get_dns_history(domain, type: type, page: page_idx)
52
+ records << next_page["records"]
53
+ end
54
+
55
+ records.flatten
56
+ end
57
+
58
+ private
59
+
60
+ #
61
+ # @param [String] domain
62
+ # @param [String] type
63
+ # @param [Integer] page
64
+ #
65
+ # @return [Array<Hash>]
66
+ #
67
+ def get_dns_history(domain, type:, page:)
68
+ _get "/v1/history/#{domain}/dns/#{type}", params: { page: page }
69
+ end
70
+
71
+ #
72
+ # @param [String] path
73
+ # @param [Hash, nil] params
74
+ #
75
+ # @return [Hash]
76
+ #
77
+ def _get(path, params:)
78
+ res = get(path, params: params)
79
+ JSON.parse(res.body.to_s)
80
+ end
81
+
82
+ #
83
+ # @param [String] path
84
+ # @param [Hash, nil] json
85
+ #
86
+ # @return [Hash]
87
+ #
88
+ def _post(path, json:)
89
+ res = post(path, json: json)
90
+ JSON.parse(res.body.to_s)
91
+ end
92
+ end
93
+ end
94
+ end
@@ -3,8 +3,14 @@
3
3
  module Mihari
4
4
  module Clients
5
5
  class Shodan < Base
6
+ # @return [String]
6
7
  attr_reader :api_key
7
8
 
9
+ #
10
+ # @param [String] base_url
11
+ # @param [String, nil] api_key
12
+ # @param [Hash] headers
13
+ #
8
14
  def initialize(base_url = "https://api.shodan.io", api_key:, headers: {})
9
15
  raise(ArgumentError, "'api_key' argument is required") unless api_key
10
16
 
@@ -13,8 +19,13 @@ module Mihari
13
19
  @api_key = api_key
14
20
  end
15
21
 
16
- # Search Shodan using the same query syntax as the website and use facets
17
- # to get summary information for different properties.
22
+ #
23
+ # @param [String] query
24
+ # @param [Integer] page
25
+ # @param [Boolean] minify
26
+ #
27
+ # @return [Structs::Shodan::Result]
28
+ #
18
29
  def search(query, page: 1, minify: true)
19
30
  params = {
20
31
  query: query,
@@ -23,7 +34,7 @@ module Mihari
23
34
  key: api_key
24
35
  }
25
36
  res = get("/shodan/host/search", params: params)
26
- JSON.parse(res.body.to_s)
37
+ Structs::Shodan::Result.from_dynamic! JSON.parse(res.body.to_s)
27
38
  end
28
39
  end
29
40
  end
@@ -5,7 +5,7 @@ module Mihari
5
5
  class TheHive < Base
6
6
  #
7
7
  # @param [String] base_url
8
- # @param [String] api_key
8
+ # @param [String, nil] api_key
9
9
  # @param [String, nil] api_version
10
10
  # @param [Hash] headers
11
11
  #
@@ -18,6 +18,11 @@ module Mihari
18
18
  super(base_url, headers: headers)
19
19
  end
20
20
 
21
+ #
22
+ # @param [Hash] json
23
+ #
24
+ # @return [Hash]
25
+ #
21
26
  def alert(json)
22
27
  json = json.to_camelback_keys.compact
23
28
  res = post("/alert", json: json)
@@ -5,7 +5,7 @@ module Mihari
5
5
  class UrlScan < Base
6
6
  #
7
7
  # @param [String] base_url
8
- # @param [String] api_key
8
+ # @param [String, nil] api_key
9
9
  # @param [Hash] headers
10
10
  #
11
11
  def initialize(base_url = "https://urlscan.io", api_key:, headers: {})
@@ -21,6 +21,8 @@ module Mihari
21
21
  # @param [Integer] size
22
22
  # @param [String, nil] search_after
23
23
  #
24
+ # @return [Hash]
25
+ #
24
26
  def search(q, size: 100, search_after: nil)
25
27
  params = { q: q, size: size, search_after: search_after }.compact
26
28
  res = get("/api/v1/search/", params: params)
@@ -5,8 +5,7 @@ module Mihari
5
5
  class VirusTotal < Base
6
6
  #
7
7
  # @param [String] base_url
8
- # @param [String] id
9
- # @param [String] secret
8
+ # @param [String, nil] api_key
10
9
  # @param [Hash] headers
11
10
  #
12
11
  def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {})
@@ -20,6 +19,8 @@ module Mihari
20
19
  #
21
20
  # @param [String] query
22
21
  #
22
+ # @return [Hash]
23
+ #
23
24
  def domain_search(query)
24
25
  _get("/api/v3/domains/#{query}/resolutions")
25
26
  end
@@ -27,6 +28,8 @@ module Mihari
27
28
  #
28
29
  # @param [String] query
29
30
  #
31
+ # @return [Hash]
32
+ #
30
33
  def ip_search(query)
31
34
  _get("/api/v3/ip_addresses/#{query}/resolutions")
32
35
  end
@@ -35,6 +38,8 @@ module Mihari
35
38
  # @param [String] query
36
39
  # @param [String, nil] cursor
37
40
  #
41
+ # @return [Hash]
42
+ #
38
43
  def intel_search(query, cursor: nil)
39
44
  params = { query: query, cursor: cursor }.compact
40
45
  _get("/api/v3/intelligence/search", params: params)
@@ -42,11 +47,12 @@ module Mihari
42
47
 
43
48
  private
44
49
 
45
- #
46
50
  #
47
51
  # @param [String] path
48
52
  # @param [Hash] params
49
53
  #
54
+ # @return [Hash]
55
+ #
50
56
  def _get(path, params: {})
51
57
  res = get(path, params: params)
52
58
  JSON.parse(res.body.to_s)
@@ -5,6 +5,11 @@ module Mihari
5
5
  class ZoomEye < Base
6
6
  attr_reader :api_key
7
7
 
8
+ #
9
+ # @param [String] base_url
10
+ # @param [String, nil] api_key
11
+ # @param [Hash] headers
12
+ #
8
13
  def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {})
9
14
  raise(ArgumentError, "'api_key' argument is required") unless api_key
10
15
 
@@ -52,11 +57,12 @@ module Mihari
52
57
 
53
58
  private
54
59
 
55
- #
56
60
  #
57
61
  # @param [String] path
58
62
  # @param [Hash] params
59
63
  #
64
+ # @return [Hash, nil]
65
+ #
60
66
  def _get(path, params: {})
61
67
  res = get(path, params: params)
62
68
  JSON.parse(res.body.to_s)
@@ -3,8 +3,6 @@
3
3
  module Mihari
4
4
  module Commands
5
5
  module Database
6
- include Mixins::Database
7
-
8
6
  def self.included(thor)
9
7
  thor.class_eval do
10
8
  desc "migrate", "Migrate DB schemas"
@@ -12,14 +10,11 @@ module Mihari
12
10
  #
13
11
  # @param [String] direction
14
12
  #
15
- #
16
13
  def migrate(direction = "up")
17
14
  verbose = options["verbose"]
18
15
  ActiveRecord::Migration.verbose = verbose
19
16
 
20
- with_db_connection do
21
- Mihari::Database.migrate(direction.to_sym)
22
- end
17
+ Mihari::Database.with_db_connection { Mihari::Database.migrate(direction.to_sym) }
23
18
  end
24
19
  end
25
20
  end
@@ -3,7 +3,6 @@
3
3
  module Mihari
4
4
  module Commands
5
5
  module Searcher
6
- include Mixins::Database
7
6
  include Mixins::ErrorNotification
8
7
 
9
8
  def self.included(thor)
@@ -16,7 +15,7 @@ module Mihari
16
15
  # @param [String] path_or_id
17
16
  #
18
17
  def search(path_or_id)
19
- with_db_connection do
18
+ Mihari::Database.with_db_connection do
20
19
  rule = Structs::Rule.from_path_or_id path_or_id
21
20
 
22
21
  # validate
@@ -164,6 +164,15 @@ module Mihari
164
164
 
165
165
  ActiveRecord::Base.clear_active_connections!
166
166
  end
167
+
168
+ def with_db_connection
169
+ Mihari::Database.connect
170
+ yield
171
+ rescue ActiveRecord::StatementInvalid
172
+ Mihari.logger.error("You haven't finished the DB migration! Please run 'mihari db migrate'.")
173
+ ensure
174
+ Mihari::Database.close
175
+ end
167
176
  end
168
177
  end
169
178
  end
data/lib/mihari/http.rb CHANGED
@@ -4,7 +4,7 @@ require "insensitive_hash"
4
4
 
5
5
  module Mihari
6
6
  class HTTP
7
- # @return [String]
7
+ # @return [URI]
8
8
  attr_reader :url
9
9
 
10
10
  # @return [Hash]
@@ -26,12 +26,12 @@ module Mihari
26
26
  new_url = url.deep_dup
27
27
  new_url.query = Addressable::URI.form_encode(params) unless (params || {}).empty?
28
28
 
29
- get = Net::HTTP::Get.new(new_url)
29
+ get = Net::HTTP::Get.new(new_url, headers)
30
30
  request get
31
31
  end
32
32
 
33
33
  #
34
- # Make a POST requesti
34
+ # Make a POST request
35
35
  #
36
36
  # @param [Hash, nil] params
37
37
  # @param [Hash, nil] json
@@ -43,10 +43,17 @@ module Mihari
43
43
  new_url = url.deep_dup
44
44
  new_url.query = Addressable::URI.form_encode(params) unless (params || {}).empty?
45
45
 
46
- post = Net::HTTP::Post.new(new_url)
46
+ post = Net::HTTP::Post.new(new_url, headers)
47
47
 
48
- post.body = JSON.generate(json) if json
49
- post.set_form_data(data) if data
48
+ if json
49
+ post.body = JSON.generate(json) if json
50
+ post.content_type = "application/json"
51
+ end
52
+
53
+ if data
54
+ post.set_form_data(data) if data
55
+ post.content_type = "application/x-www-form-urlencoded"
56
+ end
50
57
 
51
58
  request post
52
59
  end
@@ -65,10 +72,6 @@ module Mihari
65
72
 
66
73
  private
67
74
 
68
- def content_type
69
- headers["content-type"] || "application/json"
70
- end
71
-
72
75
  #
73
76
  # Get options for HTTP request
74
77
  #
@@ -89,16 +92,9 @@ module Mihari
89
92
  #
90
93
  def request(req)
91
94
  Net::HTTP.start(url.host, url.port, https_options) do |http|
92
- # set headers
93
- headers.each do |k, v|
94
- req[k] = v
95
- end
96
-
97
95
  res = http.request(req)
98
-
99
96
  unless res.is_a?(Net::HTTPSuccess)
100
- code = res.code.to_i
101
- raise UnsuccessfulStatusCodeError, "Unsuccessful response code returned: #{code}"
97
+ raise UnsuccessfulStatusCodeError, "Unsuccessful response code returned: #{res.code}"
102
98
  end
103
99
 
104
100
  res
@@ -4,8 +4,17 @@ module Mihari
4
4
  module Structs
5
5
  module Censys
6
6
  class AutonomousSystem < Dry::Struct
7
+ include Mixins::AutonomousSystem
8
+
7
9
  attribute :asn, Types::Int
8
10
 
11
+ #
12
+ # @return [Mihari::AutonomousSystem]
13
+ #
14
+ def to_as
15
+ Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
16
+ end
17
+
9
18
  def self.from_dynamic!(d)
10
19
  d = Types::Hash[d]
11
20
  new(
@@ -18,6 +27,20 @@ module Mihari
18
27
  attribute :country, Types::String.optional
19
28
  attribute :country_code, Types::String.optional
20
29
 
30
+ #
31
+ # @return [Mihari::Geolocation] <description>
32
+ #
33
+ def to_geolocation
34
+ # sometimes Censys overlooks country
35
+ # then set geolocation as nil
36
+ return nil if country.nil?
37
+
38
+ Mihari::Geolocation.new(
39
+ country: country,
40
+ country_code: country_code
41
+ )
42
+ end
43
+
21
44
  def self.from_dynamic!(d)
22
45
  d = Types::Hash[d]
23
46
  new(
@@ -30,6 +53,13 @@ module Mihari
30
53
  class Service < Dry::Struct
31
54
  attribute :port, Types::Integer
32
55
 
56
+ #
57
+ # @return [Mihari::Port]
58
+ #
59
+ def to_port
60
+ Port.new(port: port)
61
+ end
62
+
33
63
  def self.from_dynamic!(d)
34
64
  d = Types::Hash[d]
35
65
  new(
@@ -45,6 +75,29 @@ module Mihari
45
75
  attribute :metadata, Types::Hash
46
76
  attribute :services, Types.Array(Service)
47
77
 
78
+ #
79
+ # @return [Array<Mihari::Port>]
80
+ #
81
+ def to_ports
82
+ services.map(&:to_port)
83
+ end
84
+
85
+ #
86
+ # @param [String] source
87
+ #
88
+ # @return [Mihari::Artifact]
89
+ #
90
+ def to_artifact(source = "Censys")
91
+ Artifact.new(
92
+ data: ip,
93
+ source: source,
94
+ metadata: metadata,
95
+ autonomous_system: autonomous_system.to_as,
96
+ geolocation: location.to_geolocation,
97
+ ports: to_ports
98
+ )
99
+ end
100
+
48
101
  def self.from_dynamic!(d)
49
102
  d = Types::Hash[d]
50
103
  new(
@@ -76,6 +129,15 @@ module Mihari
76
129
  attribute :hits, Types.Array(Hit)
77
130
  attribute :links, Links
78
131
 
132
+ #
133
+ # @param [String] source
134
+ #
135
+ # @return [Array<Mihari::Artifact>]
136
+ #
137
+ def to_artifacts(source = "Censys")
138
+ hits.map { |hit| hit.to_artifact(source) }
139
+ end
140
+
79
141
  def self.from_dynamic!(d)
80
142
  d = Types::Hash[d]
81
143
  new(
@@ -4,10 +4,29 @@ module Mihari
4
4
  module Structs
5
5
  module GreyNoise
6
6
  class Metadata < Dry::Struct
7
+ include Mixins::AutonomousSystem
8
+
7
9
  attribute :country, Types::String
8
10
  attribute :country_code, Types::String
9
11
  attribute :asn, Types::String
10
12
 
13
+ #
14
+ # @return [Mihari::AutonomousSystem]
15
+ #
16
+ def to_as
17
+ Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
18
+ end
19
+
20
+ #
21
+ # @return [Mihari::Geolocation]
22
+ #
23
+ def to_geolocation
24
+ Mihari::Geolocation.new(
25
+ country: country,
26
+ country_code: country_code
27
+ )
28
+ end
29
+
11
30
  def self.from_dynamic!(d)
12
31
  d = Types::Hash[d]
13
32
  new(
@@ -23,6 +42,21 @@ module Mihari
23
42
  attribute :metadata, Metadata
24
43
  attribute :metadata_, Types::Hash
25
44
 
45
+ #
46
+ # @param [String] source
47
+ #
48
+ # @return [Mihari::Artifact]
49
+ #
50
+ def to_artifact(source = "GreyNoise")
51
+ Mihari::Artifact.new(
52
+ data: ip,
53
+ source: source,
54
+ metadata: metadata_,
55
+ autonomous_system: metadata.to_as,
56
+ geolocation: metadata.to_geolocation
57
+ )
58
+ end
59
+
26
60
  def self.from_dynamic!(d)
27
61
  d = Types::Hash[d]
28
62
  new(
@@ -40,6 +74,15 @@ module Mihari
40
74
  attribute :message, Types::String
41
75
  attribute :query, Types::String
42
76
 
77
+ #
78
+ # @param [String] source
79
+ #
80
+ # @return [Array<Mihari::Artifact>]
81
+ #
82
+ def to_artifacts(source = "GreyNoise")
83
+ data.map { |datum| datum.to_artifact(source) }
84
+ end
85
+
43
86
  def self.from_dynamic!(d)
44
87
  d = Types::Hash[d]
45
88
  new(
@@ -4,11 +4,47 @@ module Mihari
4
4
  module Structs
5
5
  module Onyphe
6
6
  class Result < Dry::Struct
7
+ include Mixins::AutonomousSystem
8
+
7
9
  attribute :asn, Types::String
8
10
  attribute :country_code, Types::String.optional
9
11
  attribute :ip, Types::String
10
12
  attribute :metadata, Types::Hash
11
13
 
14
+ #
15
+ # @param [String] source
16
+ #
17
+ # @return [Mihari::Artifact]
18
+ #
19
+ def to_artifact(source = "Onyphe")
20
+ Mihari::Artifact.new(
21
+ data: ip,
22
+ source: source,
23
+ metadata: metadata,
24
+ autonomous_system: to_as,
25
+ geolocation: to_geolocation
26
+ )
27
+ end
28
+
29
+ #
30
+ # @return [Mihari::Geolocation, nil]
31
+ #
32
+ def to_geolocation
33
+ return nil if country_code.nil?
34
+
35
+ Mihari::Geolocation.new(
36
+ country: NormalizeCountry(country_code, to: :short),
37
+ country_code: country_code
38
+ )
39
+ end
40
+
41
+ #
42
+ # @return [Mihari::AutonomousSystem]
43
+ #
44
+ def to_as
45
+ Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
46
+ end
47
+
12
48
  def self.from_dynamic!(d)
13
49
  d = Types::Hash[d]
14
50
  new(
@@ -30,6 +66,15 @@ module Mihari
30
66
  attribute :status, Types::String
31
67
  attribute :total, Types::Int
32
68
 
69
+ #
70
+ # @param [String] source
71
+ #
72
+ # @return [Array<Mihari::Artifact>]
73
+ #
74
+ def to_artifacts(source = "Onyphe")
75
+ results.map { |result| result.to_artifact(source) }
76
+ end
77
+
33
78
  def self.from_dynamic!(d)
34
79
  d = Types::Hash[d]
35
80
  new(