mihari 5.1.1 → 5.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/.gitmodules +0 -3
  3. data/.rubocop.yml +6 -0
  4. data/README.md +0 -1
  5. data/lib/mihari/analyzers/base.rb +32 -27
  6. data/lib/mihari/analyzers/binaryedge.rb +8 -2
  7. data/lib/mihari/analyzers/censys.rb +10 -61
  8. data/lib/mihari/analyzers/circl.rb +13 -19
  9. data/lib/mihari/analyzers/crtsh.rb +6 -0
  10. data/lib/mihari/analyzers/dnstwister.rb +12 -19
  11. data/lib/mihari/analyzers/feed.rb +21 -0
  12. data/lib/mihari/analyzers/greynoise.rb +5 -28
  13. data/lib/mihari/analyzers/onyphe.rb +8 -33
  14. data/lib/mihari/analyzers/otx.rb +11 -17
  15. data/lib/mihari/analyzers/passivetotal.rb +13 -19
  16. data/lib/mihari/analyzers/pulsedive.rb +3 -1
  17. data/lib/mihari/analyzers/rule.rb +0 -1
  18. data/lib/mihari/analyzers/securitytrails.rb +18 -29
  19. data/lib/mihari/analyzers/shodan.rb +13 -92
  20. data/lib/mihari/analyzers/urlscan.rb +12 -4
  21. data/lib/mihari/analyzers/virustotal.rb +4 -0
  22. data/lib/mihari/analyzers/virustotal_intelligence.rb +9 -6
  23. data/lib/mihari/analyzers/zoomeye.rb +9 -0
  24. data/lib/mihari/clients/binaryedge.rb +5 -0
  25. data/lib/mihari/clients/censys.rb +4 -4
  26. data/lib/mihari/clients/circl.rb +3 -3
  27. data/lib/mihari/clients/greynoise.rb +6 -1
  28. data/lib/mihari/clients/misp.rb +8 -1
  29. data/lib/mihari/clients/onyphe.rb +13 -1
  30. data/lib/mihari/clients/otx.rb +20 -0
  31. data/lib/mihari/clients/passivetotal.rb +6 -2
  32. data/lib/mihari/clients/publsedive.rb +18 -1
  33. data/lib/mihari/clients/securitytrails.rb +94 -0
  34. data/lib/mihari/clients/shodan.rb +14 -3
  35. data/lib/mihari/clients/the_hive.rb +6 -1
  36. data/lib/mihari/clients/urlscan.rb +3 -1
  37. data/lib/mihari/clients/virustotal.rb +9 -3
  38. data/lib/mihari/clients/zoomeye.rb +7 -1
  39. data/lib/mihari/commands/database.rb +1 -6
  40. data/lib/mihari/commands/searcher.rb +1 -2
  41. data/lib/mihari/database.rb +9 -0
  42. data/lib/mihari/http.rb +14 -18
  43. data/lib/mihari/structs/censys.rb +62 -0
  44. data/lib/mihari/structs/greynoise.rb +43 -0
  45. data/lib/mihari/structs/onyphe.rb +45 -0
  46. data/lib/mihari/structs/shodan.rb +83 -0
  47. data/lib/mihari/version.rb +1 -1
  48. data/lib/mihari/web/middleware/connection_adapter.rb +1 -3
  49. data/lib/mihari/web/public/assets/{index-63900d73.js → index-7d0fb8c4.js} +2 -2
  50. data/lib/mihari/web/public/index.html +1 -1
  51. data/lib/mihari/web/public/redoc-static.html +2 -2
  52. data/lib/mihari.rb +1 -3
  53. data/mihari.gemspec +2 -3
  54. metadata +8 -24
  55. data/lib/mihari/analyzers/dnpedia.rb +0 -33
  56. data/lib/mihari/clients/dnpedia.rb +0 -64
  57. data/lib/mihari/mixins/database.rb +0 -16
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "securitytrails"
4
-
5
3
  module Mihari
6
4
  module Analyzers
7
5
  class SecurityTrails < Base
@@ -15,6 +13,9 @@ module Mihari
15
13
  # @return [String, nil]
16
14
  attr_reader :api_key
17
15
 
16
+ # @return [String]
17
+ attr_reader :query
18
+
18
19
  def initialize(*args, **kwargs)
19
20
  super
20
21
 
@@ -25,7 +26,16 @@ module Mihari
25
26
  end
26
27
 
27
28
  def artifacts
28
- search || []
29
+ case type
30
+ when "domain"
31
+ domain_search
32
+ when "ip"
33
+ ip_search
34
+ when "mail"
35
+ mail_search
36
+ else
37
+ raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
38
+ end
29
39
  end
30
40
 
31
41
  private
@@ -34,8 +44,8 @@ module Mihari
34
44
  %w[securitytrails_api_key]
35
45
  end
36
46
 
37
- def api
38
- @api ||= ::SecurityTrails::API.new(api_key)
47
+ def client
48
+ @client ||= Clients::SecurityTrails.new(api_key: api_key)
39
49
  end
40
50
 
41
51
  #
@@ -47,32 +57,13 @@ module Mihari
47
57
  %w[ip domain mail].include? type
48
58
  end
49
59
 
50
- #
51
- # IP/domain/mail search
52
- #
53
- # @return [Array<String>, Array<Mihari::Artifact>]
54
- #
55
- def search
56
- case type
57
- when "domain"
58
- domain_search
59
- when "ip"
60
- ip_search
61
- when "mail"
62
- mail_search
63
- else
64
- raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
65
- end
66
- end
67
-
68
60
  #
69
61
  # Domain search
70
62
  #
71
63
  # @return [Array<String>]
72
64
  #
73
65
  def domain_search
74
- result = api.history.get_all_dns_history(query, type: "a")
75
- records = result["records"] || []
66
+ records = client.get_all_dns_history(query, type: "a")
76
67
  records.map do |record|
77
68
  (record["values"] || []).map { |value| value["ip"] }
78
69
  end.flatten.compact.uniq
@@ -84,8 +75,7 @@ module Mihari
84
75
  # @return [Array<Mihari::Artifact>]
85
76
  #
86
77
  def ip_search
87
- result = api.domains.search(filter: { ipv4: query })
88
- records = result["records"] || []
78
+ records = client.search_by_ip(query)
89
79
  records.filter_map do |record|
90
80
  data = record["hostname"]
91
81
  Artifact.new(data: data, source: source, metadata: record)
@@ -98,8 +88,7 @@ module Mihari
98
88
  # @return [Array<String>]
99
89
  #
100
90
  def mail_search
101
- result = api.domains.search(filter: { whois_email: query })
102
- records = result["records"] || []
91
+ records = client.search_by_mail(query)
103
92
  records.filter_map do |record|
104
93
  data = record["hostname"]
105
94
  Artifact.new(data: data, source: source, metadata: record)
@@ -10,6 +10,12 @@ module Mihari
10
10
  # @return [String, nil]
11
11
  attr_reader :api_key
12
12
 
13
+ # @return [Integer]
14
+ attr_reader :interval
15
+
16
+ # @return [String]
17
+ attr_reader :query
18
+
13
19
  def initialize(*args, **kwargs)
14
20
  super(*args, **kwargs)
15
21
 
@@ -18,13 +24,9 @@ module Mihari
18
24
 
19
25
  def artifacts
20
26
  results = search
21
- return [] unless results || results.empty?
22
-
23
- results = results.map { |result| Structs::Shodan::Result.from_dynamic!(result) }
24
- matches = results.map { |result| result.matches || [] }.flatten
27
+ return [] if results.empty?
25
28
 
26
- uniq_matches = matches.uniq(&:ip_str)
27
- uniq_matches.map { |match| build_artifact(match, matches) }
29
+ results.map { |result| result.to_artifacts(source) }.flatten.uniq(&:data)
28
30
  end
29
31
 
30
32
  private
@@ -42,29 +44,25 @@ module Mihari
42
44
  #
43
45
  # Search with pagination
44
46
  #
45
- # @param [String] query
46
47
  # @param [Integer] page
47
48
  #
48
- # @return [Hash]
49
+ # @return [Structs::Shodan::Result]
49
50
  #
50
- def search_with_page(query, page: 1)
51
+ def search_with_page(page: 1)
51
52
  client.search(query, page: page)
52
53
  end
53
54
 
54
55
  #
55
56
  # Search
56
57
  #
57
- # @return [Array<Hash>]
58
+ # @return [Array<Structs::Shodan::Result>]
58
59
  #
59
60
  def search
60
61
  responses = []
61
62
  (1..Float::INFINITY).each do |page|
62
- res = search_with_page(query, page: page)
63
-
64
- break unless res
65
-
63
+ res = search_with_page(page: page)
66
64
  responses << res
67
- break if res["total"].to_i <= page * PAGE_SIZE
65
+ break if res.total <= page * PAGE_SIZE
68
66
 
69
67
  # sleep #{interval} seconds to avoid the rate limitation (if it is set)
70
68
  sleep interval
@@ -75,83 +73,6 @@ module Mihari
75
73
  end
76
74
  responses
77
75
  end
78
-
79
- #
80
- # Collect metadata from matches
81
- #
82
- # @param [Array<Structs::Shodan::Match>] matches
83
- # @param [String] ip
84
- #
85
- # @return [Array<Hash>]
86
- #
87
- def collect_metadata_by_ip(matches, ip)
88
- matches.select { |match| match.ip_str == ip }.map(&:metadata)
89
- end
90
-
91
- #
92
- # Collect ports from matches
93
- #
94
- # @param [Array<Structs::Shodan::Match>] matches
95
- # @param [String] ip
96
- #
97
- # @return [Array<String>]
98
- #
99
- def collect_ports_by_ip(matches, ip)
100
- matches.select { |match| match.ip_str == ip }.map(&:port)
101
- end
102
-
103
- #
104
- # Collect hostnames from matches
105
- #
106
- # @param [Array<Structs::Shodan::Match>] matches
107
- # @param [String] ip
108
- #
109
- # @return [Array<String>]
110
- #
111
- def collect_hostnames_by_ip(matches, ip)
112
- matches.select { |match| match.ip_str == ip }.map(&:hostnames).flatten.uniq
113
- end
114
-
115
- #
116
- # Build an artifact from a Shodan search API response
117
- #
118
- # @param [Structs::Shodan::Match] match
119
- # @param [Array<Structs::Shodan::Match>] matches
120
- #
121
- # @return [Artifact]
122
- #
123
- def build_artifact(match, matches)
124
- as = nil
125
- as = AutonomousSystem.new(asn: normalize_asn(match.asn)) unless match.asn.nil?
126
-
127
- geolocation = nil
128
- if !match.location.country_name.nil? && !match.location.country_code.nil?
129
- geolocation = Geolocation.new(
130
- country: match.location.country_name,
131
- country_code: match.location.country_code
132
- )
133
- end
134
-
135
- metadata = collect_metadata_by_ip(matches, match.ip_str)
136
-
137
- ports = collect_ports_by_ip(matches, match.ip_str).map do |port|
138
- Port.new(port: port)
139
- end
140
-
141
- reverse_dns_names = collect_hostnames_by_ip(matches, match.ip_str).map do |name|
142
- ReverseDnsName.new(name: name)
143
- end
144
-
145
- Artifact.new(
146
- data: match.ip_str,
147
- source: source,
148
- metadata: metadata,
149
- autonomous_system: as,
150
- geolocation: geolocation,
151
- ports: ports,
152
- reverse_dns_names: reverse_dns_names
153
- )
154
- end
155
76
  end
156
77
  end
157
78
  end
@@ -15,12 +15,20 @@ module Mihari
15
15
  # @return [String, nil]
16
16
  attr_reader :api_key
17
17
 
18
+ # @return [String]
19
+ attr_reader :query
20
+
21
+ # @return [Integer]
22
+ attr_reader :interval
23
+
24
+ # @return [String]
25
+ attr_reader :allowed_data_types
26
+
18
27
  def initialize(*args, **kwargs)
19
28
  super
20
29
 
21
- unless valid_alllowed_data_types?
22
- raise InvalidInputError,
23
- "allowed_data_types should be any of url, domain and ip."
30
+ unless valid_allowed_data_types?
31
+ raise InvalidInputError, "allowed_data_types should be any of url, domain and ip."
24
32
  end
25
33
 
26
34
  @api_key = kwargs[:api_key] || Mihari.config.urlscan_api_key
@@ -88,7 +96,7 @@ module Mihari
88
96
  #
89
97
  # @return [Boolean]
90
98
  #
91
- def valid_alllowed_data_types?
99
+ def valid_allowed_data_types?
92
100
  allowed_data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
93
101
  end
94
102
  end
@@ -7,11 +7,15 @@ module Mihari
7
7
 
8
8
  param :query
9
9
 
10
+ # @return [String]
10
11
  attr_reader :type
11
12
 
12
13
  # @return [String, nil]
13
14
  attr_reader :api_key
14
15
 
16
+ # @return [String]
17
+ attr_reader :query
18
+
15
19
  def initialize(*args, **kwargs)
16
20
  super(*args, **kwargs)
17
21
 
@@ -10,6 +10,12 @@ module Mihari
10
10
  # @return [String, nil]
11
11
  attr_reader :api_key
12
12
 
13
+ # @return [String]
14
+ attr_reader :query
15
+
16
+ # @return [Integer]
17
+ attr_reader :interval
18
+
13
19
  def initialize(*args, **kwargs)
14
20
  super
15
21
 
@@ -19,7 +25,7 @@ module Mihari
19
25
  end
20
26
 
21
27
  def artifacts
22
- responses = search_witgh_cursor
28
+ responses = search_with_cursor
23
29
  responses.map do |response|
24
30
  response.data.map do |datum|
25
31
  Artifact.new(data: datum.value, source: source, metadata: datum.metadata)
@@ -47,19 +53,16 @@ module Mihari
47
53
  #
48
54
  # @return [Array<Structs::VirusTotalIntelligence::Response>]
49
55
  #
50
- def search_witgh_cursor
56
+ def search_with_cursor
51
57
  cursor = nil
52
58
  responses = []
53
59
 
54
60
  loop do
55
- response = Structs::VirusTotalIntelligence::Response.from_dynamic!(client.intel_search(query,
56
- cursor: cursor))
61
+ response = Structs::VirusTotalIntelligence::Response.from_dynamic!(client.intel_search(query, cursor: cursor))
57
62
  responses << response
58
-
59
63
  break if response.meta.cursor.nil?
60
64
 
61
65
  cursor = response.meta.cursor
62
-
63
66
  # sleep #{interval} seconds to avoid the rate limitation (if it is set)
64
67
  sleep interval
65
68
  end
@@ -12,6 +12,15 @@ module Mihari
12
12
  # @return [String, nil]
13
13
  attr_reader :api_key
14
14
 
15
+ # @return [String]
16
+ attr_reader :query
17
+
18
+ # @return [String]
19
+ attr_reader :type
20
+
21
+ # @return [Integer]
22
+ attr_reader :interval
23
+
15
24
  def initialize(*args, **kwargs)
16
25
  super(*args, **kwargs)
17
26
 
@@ -3,6 +3,11 @@
3
3
  module Mihari
4
4
  module Clients
5
5
  class BinaryEdge < Base
6
+ #
7
+ # @param [String] base_url
8
+ # @param [String, nil] api_key
9
+ # @param [Hash] headers
10
+ #
6
11
  def initialize(base_url = "https://api.binaryedge.io/v2", api_key:, headers: {})
7
12
  raise(ArgumentError, "'api_key' argument is required") unless api_key
8
13
 
@@ -7,8 +7,8 @@ module Mihari
7
7
  class Censys < Base
8
8
  #
9
9
  # @param [String] base_url
10
- # @param [String] id
11
- # @param [String] secret
10
+ # @param [String, nil] id
11
+ # @param [String, nil] secret
12
12
  # @param [Hash] headers
13
13
  #
14
14
  def initialize(base_url = "https://search.censys.io", id:, secret:, headers: {})
@@ -30,12 +30,12 @@ module Mihari
30
30
  # @params [Integer, nil] per_page the number of results to be returned for each page.
31
31
  # @params [Integer, nil] cursor the cursor of the desired result set.
32
32
  #
33
- # @return [Hash]
33
+ # @return [Structs::Censys::Response]
34
34
  #
35
35
  def search(query, per_page: nil, cursor: nil)
36
36
  params = { q: query, per_page: per_page, cursor: cursor }.compact
37
37
  res = get("/api/v2/hosts/search", params: params)
38
- JSON.parse(res.body.to_s)
38
+ Structs::Censys::Response.from_dynamic! JSON.parse(res.body.to_s)
39
39
  end
40
40
  end
41
41
  end
@@ -7,8 +7,8 @@ module Mihari
7
7
  class CIRCL < Base
8
8
  #
9
9
  # @param [String] base_url
10
- # @param [String] username
11
- # @param [String] password
10
+ # @param [String, nil] username
11
+ # @param [String, nil] password
12
12
  # @param [Hash] headers
13
13
  #
14
14
  def initialize(base_url = "https://www.circl.lu", username:, password:, headers: {})
@@ -43,7 +43,7 @@ module Mihari
43
43
  #
44
44
  #
45
45
  # @param [String] path
46
- # @param [Array<Hash>] params
46
+ # @param [Hash] params
47
47
  #
48
48
  def _get(path, params: {})
49
49
  res = get(path, params: params)
@@ -3,6 +3,11 @@
3
3
  module Mihari
4
4
  module Clients
5
5
  class GreyNoise < Base
6
+ #
7
+ # @param [String] base_url
8
+ # @param [String, nil] api_key
9
+ # @param [Hash] headers
10
+ #
6
11
  def initialize(base_url = "https://api.greynoise.io", api_key:, headers: {})
7
12
  raise(ArgumentError, "'api_key' argument is required") unless api_key
8
13
 
@@ -22,7 +27,7 @@ module Mihari
22
27
  def gnql_search(query, size: nil, scroll: nil)
23
28
  params = { query: query, size: size, scroll: scroll }.compact
24
29
  res = get("/v2/experimental/gnql", params: params)
25
- JSON.parse res.body.to_s
30
+ Structs::GreyNoise::Response.from_dynamic! JSON.parse(res.body.to_s)
26
31
  end
27
32
  end
28
33
  end
@@ -5,16 +5,23 @@ module Mihari
5
5
  class MISP < Base
6
6
  #
7
7
  # @param [String] base_url
8
- # @param [String] api_key
8
+ # @param [String, nil] api_key
9
9
  # @param [Hash] headers
10
10
  #
11
11
  def initialize(base_url, api_key:, headers: {})
12
12
  raise(ArgumentError, "'api_key' argument is required") unless api_key
13
13
 
14
14
  headers["authorization"] = api_key
15
+ headers["accept"] = "application/json"
16
+
15
17
  super(base_url, headers: headers)
16
18
  end
17
19
 
20
+ #
21
+ # @param [Hash] payload
22
+ #
23
+ # @return [Hash]
24
+ #
18
25
  def create_event(payload)
19
26
  res = post("/events/add", json: payload)
20
27
  JSON.parse(res.body.to_s)
@@ -3,8 +3,14 @@
3
3
  module Mihari
4
4
  module Clients
5
5
  class Onyphe < Base
6
+ # @return [String]
6
7
  attr_reader :api_key
7
8
 
9
+ #
10
+ # @param [String] base_url
11
+ # @param [String, nil] api_key
12
+ # @param [Hash] headers
13
+ #
8
14
  def initialize(base_url = "https://www.onyphe.io", api_key:, headers: {})
9
15
  raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
10
16
 
@@ -13,10 +19,16 @@ module Mihari
13
19
  @api_key = api_key
14
20
  end
15
21
 
22
+ #
23
+ # @param [String] query
24
+ # @param [Integer] page
25
+ #
26
+ # @return [Hash]
27
+ #
16
28
  def datascan(query, page: 1)
17
29
  params = { page: page, apikey: api_key }
18
30
  res = get("/api/v2/simple/datascan/#{query}", params: params)
19
- JSON.parse(res.body.to_s)
31
+ Structs::Onyphe::Response.from_dynamic! JSON.parse(res.body.to_s)
20
32
  end
21
33
  end
22
34
  end
@@ -3,6 +3,11 @@
3
3
  module Mihari
4
4
  module Clients
5
5
  class OTX < Base
6
+ #
7
+ # @param [String] base_url
8
+ # @param [String, nil] api_key
9
+ # @param [Hash] headers
10
+ #
6
11
  def initialize(base_url = "https://otx.alienvault.com", api_key:, headers: {})
7
12
  raise(ArgumentError, "'api_key' argument is required") unless api_key
8
13
 
@@ -10,16 +15,31 @@ module Mihari
10
15
  super(base_url, headers: headers)
11
16
  end
12
17
 
18
+ #
19
+ # @param [String] ip
20
+ #
21
+ # @return [Hash]
22
+ #
13
23
  def query_by_ip(ip)
14
24
  _get "/api/v1/indicators/IPv4/#{ip}/passive_dns"
15
25
  end
16
26
 
27
+ #
28
+ # @param [String] domain
29
+ #
30
+ # @return [Hash]
31
+ #
17
32
  def query_by_domain(domain)
18
33
  _get "/api/v1/indicators/domain/#{domain}/passive_dns"
19
34
  end
20
35
 
21
36
  private
22
37
 
38
+ #
39
+ # @param [String] path
40
+ #
41
+ # @return [Hash]
42
+ #
23
43
  def _get(path)
24
44
  res = get(path)
25
45
  JSON.parse(res.body.to_s)
@@ -7,8 +7,8 @@ module Mihari
7
7
  class PassiveTotal < Base
8
8
  #
9
9
  # @param [String] base_url
10
- # @param [String] username
11
- # @param [String] api_key
10
+ # @param [String, nil] username
11
+ # @param [String, nil] api_key
12
12
  # @param [Hash] headers
13
13
  #
14
14
  def initialize(base_url = "https://api.passivetotal.org", username:, api_key:, headers: {})
@@ -31,6 +31,8 @@ module Mihari
31
31
  #
32
32
  # @param [String] query
33
33
  #
34
+ # @return [Hash]
35
+ #
34
36
  def passive_dns_search(query)
35
37
  params = { query: query }
36
38
  _get("/v2/dns/passive/unique", params: params)
@@ -56,6 +58,8 @@ module Mihari
56
58
  # @param [String] path
57
59
  # @param [Hash] params
58
60
  #
61
+ # @return [Hash]
62
+ #
59
63
  def _get(path, params: {})
60
64
  res = get(path, params: params)
61
65
  JSON.parse(res.body.to_s)
@@ -3,8 +3,14 @@
3
3
  module Mihari
4
4
  module Clients
5
5
  class PulseDive < Base
6
+ # @return [String]
6
7
  attr_reader :api_key
7
8
 
9
+ #
10
+ # @param [String] base_url
11
+ # @param [String, nil] api_key
12
+ # @param [Hash] headers
13
+ #
8
14
  def initialize(base_url = "https://pulsedive.com", api_key:, headers: {})
9
15
  super(base_url, headers: headers)
10
16
 
@@ -13,21 +19,32 @@ module Mihari
13
19
  raise(ArgumentError, "'api_key' argument is required") unless api_key
14
20
  end
15
21
 
22
+ #
23
+ # @param [String] indicator_id
24
+ #
25
+ # @return [Hash]
26
+ #
16
27
  def get_indicator(ip_or_domain)
17
28
  _get "/api/info.php", params: { indicator: ip_or_domain }
18
29
  end
19
30
 
31
+ #
32
+ # @param [String] indicator_id
33
+ #
34
+ # @return [Hash]
35
+ #
20
36
  def get_properties(indicator_id)
21
37
  _get "/api/info.php", params: { iid: indicator_id, get: "properties" }
22
38
  end
23
39
 
24
40
  private
25
41
 
26
- #
27
42
  #
28
43
  # @param [String] path
29
44
  # @param [Hash] params
30
45
  #
46
+ # @return [Hash]
47
+ #
31
48
  def _get(path, params: {})
32
49
  params["key"] = api_key
33
50