mihari 8.0.2 → 8.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/mihari/analyzers/urlscan.rb +9 -9
- data/lib/mihari/analyzers/zoomeye.rb +18 -41
- data/lib/mihari/clients/zoomeye.rb +14 -59
- data/lib/mihari/schemas/analyzer.rb +8 -2
- data/lib/mihari/schemas/rule.rb +1 -1
- data/lib/mihari/structs/zoomeye.rb +75 -0
- data/lib/mihari/types.rb +4 -1
- data/lib/mihari/version.rb +1 -1
- data/lib/mihari/web/public/assets/index-sQr_iTap.js +1594 -0
- data/lib/mihari/web/public/index.html +1 -1
- data/lib/mihari/web/public/redoc-static.html +412 -388
- data/lib/mihari.rb +1 -0
- data/mihari.gemspec +28 -28
- metadata +54 -53
- data/lib/mihari/web/public/assets/index-BO6YgRrl.js +0 -1586
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d0f4a9365970c476890029a56b880a81383fcb3c8c900e123542ec8c0bd52756
|
4
|
+
data.tar.gz: 323805a72e47dacb248a3f4737ebc5f1890350875cc881a4e52619a5e1d09509
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ec6bb62f4d03438cc83052ca4b53950bdd13e22abab9435e92aeb993b6809c139a53f02b299f2706512aa67f72370086f2433573165b256dab1973331929b7ac
|
7
|
+
data.tar.gz: 976fe873b455de92f917bde4de403531e104d3378820364a4f7ddf9274b13298e474891c774df6008c1d4e8099be391f73723cc8ea9f73839a9087fc66eb915d
|
@@ -12,29 +12,29 @@ module Mihari
|
|
12
12
|
attr_reader :api_key
|
13
13
|
|
14
14
|
# @return [Array<String>]
|
15
|
-
attr_reader :
|
15
|
+
attr_reader :data_types
|
16
16
|
|
17
17
|
#
|
18
18
|
# @param [String] query
|
19
19
|
# @param [Hash, nil] options
|
20
20
|
# @param [String, nil] api_key
|
21
|
-
# @param [Array<String>]
|
21
|
+
# @param [Array<String>] data_types
|
22
22
|
#
|
23
|
-
def initialize(query, options: nil, api_key: nil,
|
23
|
+
def initialize(query, options: nil, api_key: nil, data_types: SUPPORTED_DATA_TYPES)
|
24
24
|
super(query, options:)
|
25
25
|
|
26
26
|
@api_key = api_key || Mihari.config.urlscan_api_key
|
27
|
-
@
|
27
|
+
@data_types = data_types
|
28
28
|
|
29
|
-
return if
|
29
|
+
return if valid_data_types?
|
30
30
|
|
31
|
-
raise ValueError, "
|
31
|
+
raise ValueError, "data_types should be any of url, domain and ip."
|
32
32
|
end
|
33
33
|
|
34
34
|
def artifacts
|
35
35
|
# @type [Array<Mihari::Models::Artifact>]
|
36
36
|
artifacts = client.search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
|
37
|
-
artifacts.select { |artifact|
|
37
|
+
artifacts.select { |artifact| data_types.include? artifact.data_type }
|
38
38
|
end
|
39
39
|
|
40
40
|
private
|
@@ -52,8 +52,8 @@ module Mihari
|
|
52
52
|
#
|
53
53
|
# @return [Boolean]
|
54
54
|
#
|
55
|
-
def
|
56
|
-
|
55
|
+
def valid_data_types?
|
56
|
+
data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
|
57
57
|
end
|
58
58
|
end
|
59
59
|
end
|
@@ -6,51 +6,39 @@ module Mihari
|
|
6
6
|
# ZoomEye analyzer
|
7
7
|
#
|
8
8
|
class ZoomEye < Base
|
9
|
+
SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
|
10
|
+
|
9
11
|
# @return [String, nil]
|
10
12
|
attr_reader :api_key
|
11
13
|
|
12
|
-
# @return [String]
|
13
|
-
attr_reader :
|
14
|
+
# @return [Array<String>]
|
15
|
+
attr_reader :data_types
|
14
16
|
|
15
17
|
#
|
16
18
|
# @param [String] query
|
17
19
|
# @param [Hash, nil] options
|
18
20
|
# @param [String, nil] api_key
|
19
|
-
# @param [String]
|
21
|
+
# @param [Array<String>] data_types
|
20
22
|
#
|
21
|
-
def initialize(query, options: nil, api_key: nil,
|
23
|
+
def initialize(query, options: nil, api_key: nil, data_types: SUPPORTED_DATA_TYPES)
|
22
24
|
super(query, options:)
|
23
25
|
|
24
|
-
@type = type
|
25
26
|
@api_key = api_key || Mihari.config.zoomeye_api_key
|
27
|
+
@data_types = data_types
|
28
|
+
|
29
|
+
return if valid_data_types?
|
30
|
+
|
31
|
+
raise ValueError, "data_types should be any of url, domain and ip."
|
26
32
|
end
|
27
33
|
|
28
34
|
def artifacts
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
convert(res)
|
33
|
-
end.flatten
|
34
|
-
when "web"
|
35
|
-
client.web_search_with_pagination(query).map do |res|
|
36
|
-
convert(res)
|
37
|
-
end.flatten
|
38
|
-
else
|
39
|
-
raise ValueError, "#{type} type is not supported." unless valid_type?
|
40
|
-
end
|
35
|
+
# @type [Array<Mihari::Models::Artifact>]
|
36
|
+
artifacts = client.search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
|
37
|
+
artifacts.select { |artifact| data_types.include? artifact.data_type }
|
41
38
|
end
|
42
39
|
|
43
40
|
private
|
44
41
|
|
45
|
-
#
|
46
|
-
# Check whether a type is valid or not
|
47
|
-
#
|
48
|
-
# @return [Boolean]
|
49
|
-
#
|
50
|
-
def valid_type?
|
51
|
-
%w[host web].include? type
|
52
|
-
end
|
53
|
-
|
54
42
|
def client
|
55
43
|
Clients::ZoomEye.new(
|
56
44
|
api_key:,
|
@@ -60,23 +48,12 @@ module Mihari
|
|
60
48
|
end
|
61
49
|
|
62
50
|
#
|
63
|
-
#
|
51
|
+
# Check whether a data type is valid or not
|
64
52
|
#
|
65
|
-
# @
|
66
|
-
#
|
67
|
-
# @return [Array<Mihari::Models::Artifact>]
|
53
|
+
# @return [Boolean]
|
68
54
|
#
|
69
|
-
def
|
70
|
-
|
71
|
-
matches.map do |match|
|
72
|
-
data = match["ip"]
|
73
|
-
|
74
|
-
if data.is_a?(Array)
|
75
|
-
data.map { |d| Models::Artifact.new(data: d, metadata: match) }
|
76
|
-
else
|
77
|
-
Models::Artifact.new(data:, metadata: match)
|
78
|
-
end
|
79
|
-
end.flatten
|
55
|
+
def valid_data_types?
|
56
|
+
data_types.all? { |type| SUPPORTED_DATA_TYPES.include? type }
|
80
57
|
end
|
81
58
|
end
|
82
59
|
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "base64"
|
4
|
+
|
3
5
|
module Mihari
|
4
6
|
module Clients
|
5
7
|
#
|
@@ -18,7 +20,7 @@ module Mihari
|
|
18
20
|
# @param [Integer, nil] timeout
|
19
21
|
#
|
20
22
|
def initialize(
|
21
|
-
base_url = "https://api.zoomeye.
|
23
|
+
base_url = "https://api.zoomeye.ai",
|
22
24
|
api_key:,
|
23
25
|
headers: {},
|
24
26
|
pagination_interval: Mihari.config.pagination_interval,
|
@@ -38,83 +40,36 @@ module Mihari
|
|
38
40
|
end
|
39
41
|
|
40
42
|
#
|
41
|
-
# Search
|
42
|
-
#
|
43
|
-
# @param [String] query Query string
|
44
|
-
# @param [Integer, nil] page The page number to paging(default:1)
|
45
|
-
# @param [String, nil] facets A comma-separated list of properties to get summary information on query
|
46
|
-
#
|
47
|
-
# @return [Hash]
|
48
|
-
#
|
49
|
-
def host_search(query, page: nil, facets: nil)
|
50
|
-
params = {
|
51
|
-
query:,
|
52
|
-
page:,
|
53
|
-
facets:
|
54
|
-
}.compact
|
55
|
-
get_json "/host/search", params:
|
56
|
-
end
|
57
|
-
|
58
|
-
#
|
59
|
-
# @param [String] query
|
60
|
-
# @param [String, nil] facets
|
61
|
-
# @param [Integer] pagination_limit
|
62
|
-
#
|
63
|
-
# @return [Enumerable<Hash>]
|
64
|
-
#
|
65
|
-
def host_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
|
66
|
-
Enumerator.new do |y|
|
67
|
-
(1..pagination_limit).each do |page|
|
68
|
-
res = host_search(query, facets:, page:)
|
69
|
-
|
70
|
-
break if res.nil?
|
71
|
-
|
72
|
-
y.yield res
|
73
|
-
|
74
|
-
total = res["total"].to_i
|
75
|
-
break if total <= page * PAGE_SIZE
|
76
|
-
|
77
|
-
sleep_pagination_interval
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
#
|
83
|
-
# Search the Web technologies
|
43
|
+
# Search
|
84
44
|
#
|
85
45
|
# @param [String] query Query string
|
86
46
|
# @param [Integer, nil] page The page number to paging(default:1)
|
87
|
-
# @param [String, nil] facets A comma-separated list of properties to get summary information on query
|
88
47
|
#
|
89
|
-
# @return [
|
48
|
+
# @return [Structs::ZoomEye::Response]
|
90
49
|
#
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
}.compact
|
97
|
-
get_json "/web/search", params:
|
50
|
+
# @param [Object, nil] facets
|
51
|
+
def search(query, page: nil)
|
52
|
+
qbase64 = Base64.urlsafe_encode64(query)
|
53
|
+
json = {qbase64:, page:}.compact
|
54
|
+
Structs::ZoomEye::Response.from_dynamic! post_json("/v2/search", json:)
|
98
55
|
end
|
99
56
|
|
100
57
|
#
|
101
58
|
# @param [String] query
|
102
|
-
# @param [String, nil] facets
|
103
59
|
# @param [Integer] pagination_limit
|
104
60
|
#
|
105
|
-
# @return [Enumerable<
|
61
|
+
# @return [Enumerable<Structs::ZoomEye::Response>]
|
106
62
|
#
|
107
|
-
def
|
63
|
+
def search_with_pagination(query, pagination_limit: Mihari.config.pagination_limit)
|
108
64
|
Enumerator.new do |y|
|
109
65
|
(1..pagination_limit).each do |page|
|
110
|
-
res =
|
66
|
+
res = search(query, page:)
|
111
67
|
|
112
68
|
break if res.nil?
|
113
69
|
|
114
70
|
y.yield res
|
115
71
|
|
116
|
-
|
117
|
-
break if total <= page * PAGE_SIZE
|
72
|
+
break if res.total <= page * PAGE_SIZE
|
118
73
|
|
119
74
|
sleep_pagination_interval
|
120
75
|
end
|
@@ -14,7 +14,6 @@ module Mihari
|
|
14
14
|
Mihari::Analyzers::GreyNoise.keys,
|
15
15
|
Mihari::Analyzers::Onyphe.keys,
|
16
16
|
Mihari::Analyzers::Shodan.keys,
|
17
|
-
Mihari::Analyzers::Urlscan.keys,
|
18
17
|
Mihari::Analyzers::Validin.keys,
|
19
18
|
Mihari::Analyzers::VirusTotalIntelligence.keys
|
20
19
|
].each do |keys|
|
@@ -81,10 +80,17 @@ module Mihari
|
|
81
80
|
optional(:options).hash(AnalyzerOptions)
|
82
81
|
end
|
83
82
|
|
83
|
+
Urlscan = Dry::Schema.Params do
|
84
|
+
required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Urlscan.keys))
|
85
|
+
required(:query).filled(:string)
|
86
|
+
optional(:data_types).filled(array[Types::NetworkDataTypes]).default(Types::NetworkDataTypes.values)
|
87
|
+
optional(:options).hash(AnalyzerPaginationOptions)
|
88
|
+
end
|
89
|
+
|
84
90
|
ZoomEye = Dry::Schema.Params do
|
85
91
|
required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::ZoomEye.keys))
|
86
92
|
required(:query).filled(:string)
|
87
|
-
|
93
|
+
optional(:data_types).filled(array[Types::NetworkDataTypes]).default(Types::NetworkDataTypes.values)
|
88
94
|
optional(:options).hash(AnalyzerPaginationOptions)
|
89
95
|
end
|
90
96
|
|
data/lib/mihari/schemas/rule.rb
CHANGED
@@ -25,7 +25,7 @@ module Mihari
|
|
25
25
|
optional(:emitters).array { Emitter }.default(DEFAULT_EMITTERS)
|
26
26
|
optional(:enrichers).array { Enricher }.default(DEFAULT_ENRICHERS)
|
27
27
|
|
28
|
-
optional(:data_types).filled(array[Types::DataTypes]).default(
|
28
|
+
optional(:data_types).filled(array[Types::DataTypes]).default(Types::DataTypes.values)
|
29
29
|
|
30
30
|
optional(:falsepositives).array { filled(:string) }.default([])
|
31
31
|
|
@@ -0,0 +1,75 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Mihari
|
4
|
+
module Structs
|
5
|
+
module ZoomEye
|
6
|
+
class Datanum < Dry::Struct
|
7
|
+
# @!attribute [r] ip
|
8
|
+
# @return [String]
|
9
|
+
attribute :ip, Types::String.optional
|
10
|
+
|
11
|
+
# @!attribute [r] domain
|
12
|
+
# @return [String, nil]
|
13
|
+
attribute :domain, Types::String.optional
|
14
|
+
|
15
|
+
# @!attribute [r] url
|
16
|
+
# @return [String]
|
17
|
+
attribute :url, Types::String.optional
|
18
|
+
|
19
|
+
# @!attribute [r] metadata
|
20
|
+
# @return [Hash]
|
21
|
+
attribute :metadata, Types::Hash
|
22
|
+
|
23
|
+
class << self
|
24
|
+
#
|
25
|
+
# @param [Hash] d
|
26
|
+
#
|
27
|
+
def from_dynamic!(d)
|
28
|
+
d = Types::Hash[d]
|
29
|
+
new(
|
30
|
+
domain: d["domain"],
|
31
|
+
ip: d["ip"],
|
32
|
+
url: d["url"],
|
33
|
+
metadata: d
|
34
|
+
)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def artifacts
|
39
|
+
values = [url, domain, ip].compact
|
40
|
+
values.map { |value| Mihari::Models::Artifact.new(data: value, metadata:) }
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
class Response < Dry::Struct
|
45
|
+
# @!attribute [r] data
|
46
|
+
# @return [Array<Datanum>]
|
47
|
+
attribute :data, Types.Array(Datanum)
|
48
|
+
|
49
|
+
# @!attribute [r] total
|
50
|
+
# @return [Integer]
|
51
|
+
attribute :total, Types::Int
|
52
|
+
|
53
|
+
#
|
54
|
+
# @return [Array<Mihari::Models::Artifact>]
|
55
|
+
#
|
56
|
+
def artifacts
|
57
|
+
data.map(&:artifacts).flatten
|
58
|
+
end
|
59
|
+
|
60
|
+
class << self
|
61
|
+
#
|
62
|
+
# @param [Hash] d
|
63
|
+
#
|
64
|
+
def from_dynamic!(d)
|
65
|
+
d = Types::Hash[d]
|
66
|
+
new(
|
67
|
+
data: d.fetch("data").map { |x| Datanum.from_dynamic!(x) },
|
68
|
+
total: d.fetch("total")
|
69
|
+
)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
data/lib/mihari/types.rb
CHANGED
@@ -15,7 +15,10 @@ module Mihari
|
|
15
15
|
Double = Strict::Float | Strict::Integer
|
16
16
|
DateTime = Strict::DateTime
|
17
17
|
|
18
|
-
|
18
|
+
NetworkDataTypes = Types::String.enum("ip", "domain", "url")
|
19
|
+
DataTypes = Types::String.enum(
|
20
|
+
*[NetworkDataTypes.values, "hash", "mail"].flatten
|
21
|
+
)
|
19
22
|
|
20
23
|
HTTPRequestMethods = Types::String.enum("GET", "POST")
|
21
24
|
end
|
data/lib/mihari/version.rb
CHANGED