mihari 8.0.2 → 8.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d3d454e8ac560ec2d1b7463dfdc686e69fc033da82e318f9479ba14f72194c49
4
- data.tar.gz: d2f191e78eb2180a9062d885fd527f5e55c70fa70a5e9f8f910b15a50d57d6a2
3
+ metadata.gz: d0f4a9365970c476890029a56b880a81383fcb3c8c900e123542ec8c0bd52756
4
+ data.tar.gz: 323805a72e47dacb248a3f4737ebc5f1890350875cc881a4e52619a5e1d09509
5
5
  SHA512:
6
- metadata.gz: 723186046c14e4532c3662280264a51fb7a5966173d7010c50d6537a8b96304b6ff7f7f24d9686d1773fe8f13ceaba09d7d33c0265e286b58c9d18f3849645b8
7
- data.tar.gz: 8a9f7001f3edb2a4035178c810ff5a12cf2cf913533f853e778c8d15ef4e74edfaef4d1b9061ab406d824406e7abd9eba94adb2e45da6c9b963a5d029123d322
6
+ metadata.gz: ec6bb62f4d03438cc83052ca4b53950bdd13e22abab9435e92aeb993b6809c139a53f02b299f2706512aa67f72370086f2433573165b256dab1973331929b7ac
7
+ data.tar.gz: 976fe873b455de92f917bde4de403531e104d3378820364a4f7ddf9274b13298e474891c774df6008c1d4e8099be391f73723cc8ea9f73839a9087fc66eb915d
@@ -12,29 +12,29 @@ module Mihari
12
12
  attr_reader :api_key
13
13
 
14
14
  # @return [Array<String>]
15
- attr_reader :allowed_data_types
15
+ attr_reader :data_types
16
16
 
17
17
  #
18
18
  # @param [String] query
19
19
  # @param [Hash, nil] options
20
20
  # @param [String, nil] api_key
21
- # @param [Array<String>] allowed_data_types
21
+ # @param [Array<String>] data_types
22
22
  #
23
- def initialize(query, options: nil, api_key: nil, allowed_data_types: SUPPORTED_DATA_TYPES)
23
+ def initialize(query, options: nil, api_key: nil, data_types: SUPPORTED_DATA_TYPES)
24
24
  super(query, options:)
25
25
 
26
26
  @api_key = api_key || Mihari.config.urlscan_api_key
27
- @allowed_data_types = allowed_data_types
27
+ @data_types = data_types
28
28
 
29
- return if valid_allowed_data_types?
29
+ return if valid_data_types?
30
30
 
31
- raise ValueError, "allowed_data_types should be any of url, domain and ip."
31
+ raise ValueError, "data_types should be any of url, domain and ip."
32
32
  end
33
33
 
34
34
  def artifacts
35
35
  # @type [Array<Mihari::Models::Artifact>]
36
36
  artifacts = client.search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
37
- artifacts.select { |artifact| allowed_data_types.include? artifact.data_type }
37
+ artifacts.select { |artifact| data_types.include? artifact.data_type }
38
38
  end
39
39
 
40
40
  private
@@ -52,8 +52,8 @@ module Mihari
52
52
  #
53
53
  # @return [Boolean]
54
54
  #
55
- def valid_allowed_data_types?
56
- allowed_data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
55
+ def valid_data_types?
56
+ data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
57
57
  end
58
58
  end
59
59
  end
@@ -6,51 +6,39 @@ module Mihari
6
6
  # ZoomEye analyzer
7
7
  #
8
8
  class ZoomEye < Base
9
+ SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
10
+
9
11
  # @return [String, nil]
10
12
  attr_reader :api_key
11
13
 
12
- # @return [String]
13
- attr_reader :type
14
+ # @return [Array<String>]
15
+ attr_reader :data_types
14
16
 
15
17
  #
16
18
  # @param [String] query
17
19
  # @param [Hash, nil] options
18
20
  # @param [String, nil] api_key
19
- # @param [String] type
21
+ # @param [Array<String>] data_types
20
22
  #
21
- def initialize(query, options: nil, api_key: nil, type: "host")
23
+ def initialize(query, options: nil, api_key: nil, data_types: SUPPORTED_DATA_TYPES)
22
24
  super(query, options:)
23
25
 
24
- @type = type
25
26
  @api_key = api_key || Mihari.config.zoomeye_api_key
27
+ @data_types = data_types
28
+
29
+ return if valid_data_types?
30
+
31
+ raise ValueError, "data_types should be any of url, domain and ip."
26
32
  end
27
33
 
28
34
  def artifacts
29
- case type
30
- when "host"
31
- client.host_search_with_pagination(query).map do |res|
32
- convert(res)
33
- end.flatten
34
- when "web"
35
- client.web_search_with_pagination(query).map do |res|
36
- convert(res)
37
- end.flatten
38
- else
39
- raise ValueError, "#{type} type is not supported." unless valid_type?
40
- end
35
+ # @type [Array<Mihari::Models::Artifact>]
36
+ artifacts = client.search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
37
+ artifacts.select { |artifact| data_types.include? artifact.data_type }
41
38
  end
42
39
 
43
40
  private
44
41
 
45
- #
46
- # Check whether a type is valid or not
47
- #
48
- # @return [Boolean]
49
- #
50
- def valid_type?
51
- %w[host web].include? type
52
- end
53
-
54
42
  def client
55
43
  Clients::ZoomEye.new(
56
44
  api_key:,
@@ -60,23 +48,12 @@ module Mihari
60
48
  end
61
49
 
62
50
  #
63
- # Convert responses into an array of String
51
+ # Check whether a data type is valid or not
64
52
  #
65
- # @param [Hash] res
66
- #
67
- # @return [Array<Mihari::Models::Artifact>]
53
+ # @return [Boolean]
68
54
  #
69
- def convert(res)
70
- matches = res["matches"] || []
71
- matches.map do |match|
72
- data = match["ip"]
73
-
74
- if data.is_a?(Array)
75
- data.map { |d| Models::Artifact.new(data: d, metadata: match) }
76
- else
77
- Models::Artifact.new(data:, metadata: match)
78
- end
79
- end.flatten
55
+ def valid_data_types?
56
+ data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
80
57
  end
81
58
  end
82
59
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "base64"
4
+
3
5
  module Mihari
4
6
  module Clients
5
7
  #
@@ -18,7 +20,7 @@ module Mihari
18
20
  # @param [Integer, nil] timeout
19
21
  #
20
22
  def initialize(
21
- base_url = "https://api.zoomeye.org",
23
+ base_url = "https://api.zoomeye.ai",
22
24
  api_key:,
23
25
  headers: {},
24
26
  pagination_interval: Mihari.config.pagination_interval,
@@ -38,83 +40,36 @@ module Mihari
38
40
  end
39
41
 
40
42
  #
41
- # Search the Host devices
42
- #
43
- # @param [String] query Query string
44
- # @param [Integer, nil] page The page number to paging(default:1)
45
- # @param [String, nil] facets A comma-separated list of properties to get summary information on query
46
- #
47
- # @return [Hash]
48
- #
49
- def host_search(query, page: nil, facets: nil)
50
- params = {
51
- query:,
52
- page:,
53
- facets:
54
- }.compact
55
- get_json "/host/search", params:
56
- end
57
-
58
- #
59
- # @param [String] query
60
- # @param [String, nil] facets
61
- # @param [Integer] pagination_limit
62
- #
63
- # @return [Enumerable<Hash>]
64
- #
65
- def host_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
66
- Enumerator.new do |y|
67
- (1..pagination_limit).each do |page|
68
- res = host_search(query, facets:, page:)
69
-
70
- break if res.nil?
71
-
72
- y.yield res
73
-
74
- total = res["total"].to_i
75
- break if total <= page * PAGE_SIZE
76
-
77
- sleep_pagination_interval
78
- end
79
- end
80
- end
81
-
82
- #
83
- # Search the Web technologies
43
+ # Search
84
44
  #
85
45
  # @param [String] query Query string
86
46
  # @param [Integer, nil] page The page number to paging(default:1)
87
- # @param [String, nil] facets A comma-separated list of properties to get summary information on query
88
47
  #
89
- # @return [Hash]
48
+ # @return [Structs::ZoomEye::Response]
90
49
  #
91
- def web_search(query, page: nil, facets: nil)
92
- params = {
93
- query:,
94
- page:,
95
- facets:
96
- }.compact
97
- get_json "/web/search", params:
50
+ # @param [Object, nil] facets
51
+ def search(query, page: nil)
52
+ qbase64 = Base64.urlsafe_encode64(query)
53
+ json = {qbase64:, page:}.compact
54
+ Structs::ZoomEye::Response.from_dynamic! post_json("/v2/search", json:)
98
55
  end
99
56
 
100
57
  #
101
58
  # @param [String] query
102
- # @param [String, nil] facets
103
59
  # @param [Integer] pagination_limit
104
60
  #
105
- # @return [Enumerable<Hash>]
61
+ # @return [Enumerable<Structs::ZoomEye::Response>]
106
62
  #
107
- def web_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
63
+ def search_with_pagination(query, pagination_limit: Mihari.config.pagination_limit)
108
64
  Enumerator.new do |y|
109
65
  (1..pagination_limit).each do |page|
110
- res = web_search(query, facets:, page:)
66
+ res = search(query, page:)
111
67
 
112
68
  break if res.nil?
113
69
 
114
70
  y.yield res
115
71
 
116
- total = res["total"].to_i
117
- break if total <= page * PAGE_SIZE
72
+ break if res.total <= page * PAGE_SIZE
118
73
 
119
74
  sleep_pagination_interval
120
75
  end
@@ -14,7 +14,6 @@ module Mihari
14
14
  Mihari::Analyzers::GreyNoise.keys,
15
15
  Mihari::Analyzers::Onyphe.keys,
16
16
  Mihari::Analyzers::Shodan.keys,
17
- Mihari::Analyzers::Urlscan.keys,
18
17
  Mihari::Analyzers::Validin.keys,
19
18
  Mihari::Analyzers::VirusTotalIntelligence.keys
20
19
  ].each do |keys|
@@ -81,10 +80,17 @@ module Mihari
81
80
  optional(:options).hash(AnalyzerOptions)
82
81
  end
83
82
 
83
+ Urlscan = Dry::Schema.Params do
84
+ required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Urlscan.keys))
85
+ required(:query).filled(:string)
86
+ optional(:data_types).filled(array[Types::NetworkDataTypes]).default(Types::NetworkDataTypes.values)
87
+ optional(:options).hash(AnalyzerPaginationOptions)
88
+ end
89
+
84
90
  ZoomEye = Dry::Schema.Params do
85
91
  required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::ZoomEye.keys))
86
92
  required(:query).filled(:string)
87
- required(:type).value(Types::String.enum("host", "web"))
93
+ optional(:data_types).filled(array[Types::NetworkDataTypes]).default(Types::NetworkDataTypes.values)
88
94
  optional(:options).hash(AnalyzerPaginationOptions)
89
95
  end
90
96
 
@@ -25,7 +25,7 @@ module Mihari
25
25
  optional(:emitters).array { Emitter }.default(DEFAULT_EMITTERS)
26
26
  optional(:enrichers).array { Enricher }.default(DEFAULT_ENRICHERS)
27
27
 
28
- optional(:data_types).filled(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
28
+ optional(:data_types).filled(array[Types::DataTypes]).default(Types::DataTypes.values)
29
29
 
30
30
  optional(:falsepositives).array { filled(:string) }.default([])
31
31
 
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mihari
4
+ module Structs
5
+ module ZoomEye
6
+ class Datanum < Dry::Struct
7
+ # @!attribute [r] ip
8
+ # @return [String]
9
+ attribute :ip, Types::String.optional
10
+
11
+ # @!attribute [r] domain
12
+ # @return [String, nil]
13
+ attribute :domain, Types::String.optional
14
+
15
+ # @!attribute [r] url
16
+ # @return [String]
17
+ attribute :url, Types::String.optional
18
+
19
+ # @!attribute [r] metadata
20
+ # @return [Hash]
21
+ attribute :metadata, Types::Hash
22
+
23
+ class << self
24
+ #
25
+ # @param [Hash] d
26
+ #
27
+ def from_dynamic!(d)
28
+ d = Types::Hash[d]
29
+ new(
30
+ domain: d["domain"],
31
+ ip: d["ip"],
32
+ url: d["url"],
33
+ metadata: d
34
+ )
35
+ end
36
+ end
37
+
38
+ def artifacts
39
+ values = [url, domain, ip].compact
40
+ values.map { |value| Mihari::Models::Artifact.new(data: value, metadata:) }
41
+ end
42
+ end
43
+
44
+ class Response < Dry::Struct
45
+ # @!attribute [r] data
46
+ # @return [Array<Datanum>]
47
+ attribute :data, Types.Array(Datanum)
48
+
49
+ # @!attribute [r] total
50
+ # @return [Integer]
51
+ attribute :total, Types::Int
52
+
53
+ #
54
+ # @return [Array<Mihari::Models::Artifact>]
55
+ #
56
+ def artifacts
57
+ data.map(&:artifacts).flatten
58
+ end
59
+
60
+ class << self
61
+ #
62
+ # @param [Hash] d
63
+ #
64
+ def from_dynamic!(d)
65
+ d = Types::Hash[d]
66
+ new(
67
+ data: d.fetch("data").map { |x| Datanum.from_dynamic!(x) },
68
+ total: d.fetch("total")
69
+ )
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
data/lib/mihari/types.rb CHANGED
@@ -15,7 +15,10 @@ module Mihari
15
15
  Double = Strict::Float | Strict::Integer
16
16
  DateTime = Strict::DateTime
17
17
 
18
- DataTypes = Types::String.enum("hash", "ip", "domain", "url", "mail")
18
+ NetworkDataTypes = Types::String.enum("ip", "domain", "url")
19
+ DataTypes = Types::String.enum(
20
+ *[NetworkDataTypes.values, "hash", "mail"].flatten
21
+ )
19
22
 
20
23
  HTTPRequestMethods = Types::String.enum("GET", "POST")
21
24
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- VERSION = "8.0.2"
4
+ VERSION = "8.1.0"
5
5
  end