mihari 7.0.0 → 7.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Dockerfile +3 -2
- data/docker-compose.yml +1 -2
- data/lib/mihari/analyzers/crtsh.rb +7 -2
- data/lib/mihari/clients/crtsh.rb +1 -1
- data/lib/mihari/models/artifact.rb +17 -11
- data/lib/mihari/models/autonomous_system.rb +0 -16
- data/lib/mihari/models/cpe.rb +0 -16
- data/lib/mihari/models/dns.rb +0 -16
- data/lib/mihari/models/geolocation.rb +0 -19
- data/lib/mihari/models/reverse_dns.rb +0 -18
- data/lib/mihari/models/whois.rb +0 -14
- data/lib/mihari/schemas/analyzer.rb +1 -0
- data/lib/mihari/services/builders.rb +133 -16
- data/lib/mihari/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bc368e612c78350f6b66f4fdcf815a63f4a292e185a2348dce8110b7d17e1692
|
4
|
+
data.tar.gz: b3e9939de4ee9488bb6b48793cf5ebe196f50e92eb8f6c1eadb3ed0b201fd2c0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 217918561a441c15d4c55e12fa90792230dee66186ad12db8629c30ec848a2a4711eb154d1a008f153abadb3d9cc97ad43ce574d175d167873f3b7c44f0de011
|
7
|
+
data.tar.gz: 2d16db3598a3d5918499ce7119f7e69a548f0891dd6f9c5ee5764508244486c6ba255e190f65cca79559db888936b7975db86eadefb7e548b476fb3925316f32
|
data/Dockerfile
CHANGED
@@ -2,10 +2,11 @@ FROM ruby:3.2.2-alpine3.19
|
|
2
2
|
|
3
3
|
ARG MIHARI_VERSION=0.0.0
|
4
4
|
|
5
|
-
RUN apk --no-cache add
|
5
|
+
RUN apk --no-cache add build-base ruby-dev libpq-dev && \
|
6
|
+
echo 'gem: --no-document' >> /usr/local/etc/gemrc && \
|
6
7
|
gem install pg && \
|
7
8
|
gem install mihari -v ${MIHARI_VERSION} && \
|
8
|
-
apk del --purge
|
9
|
+
apk del --purge build-base ruby-dev && \
|
9
10
|
rm -rf /usr/local/bundle/cache/*
|
10
11
|
|
11
12
|
ENTRYPOINT ["mihari"]
|
data/docker-compose.yml
CHANGED
@@ -47,8 +47,7 @@ services:
|
|
47
47
|
image: ghcr.io/ninoseki/mihari:latest
|
48
48
|
environment:
|
49
49
|
- DATABASE_URL=${DATABASE_URL:-postgresql://${POSTGRES_USER:-user}:${POSTGRES_PASSWORD:-password}@database:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-mihari}}
|
50
|
-
-
|
51
|
-
- USE_SIDEKIQ=true
|
50
|
+
- SIDEKIQ_REDIS_URL=${REDIS_URL:-redis://redis:${REDIS_PORT:-6379}}
|
52
51
|
env_file:
|
53
52
|
- .env
|
54
53
|
entrypoint: ["mihari", "sidekiq"]
|
@@ -9,20 +9,25 @@ module Mihari
|
|
9
9
|
# @return [Boolean]
|
10
10
|
attr_reader :exclude_expired
|
11
11
|
|
12
|
+
# @return [String, nil]
|
13
|
+
attr_reader :match
|
14
|
+
|
12
15
|
#
|
13
16
|
# @param [String] query
|
14
17
|
# @param [Hash, nil] options
|
15
18
|
# @param [Bool] exclude_expired
|
19
|
+
# @param [String, nil] match
|
16
20
|
#
|
17
|
-
def initialize(query, options: nil, exclude_expired: true)
|
21
|
+
def initialize(query, options: nil, exclude_expired: true, match: nil)
|
18
22
|
super(query, options: options)
|
19
23
|
|
20
24
|
@exclude_expired = exclude_expired
|
25
|
+
@match = match
|
21
26
|
end
|
22
27
|
|
23
28
|
def artifacts
|
24
29
|
exclude = exclude_expired ? "expired" : nil
|
25
|
-
client.search(query, exclude: exclude).map do |result|
|
30
|
+
client.search(query, exclude: exclude, match: match).map do |result|
|
26
31
|
values = result["name_value"].to_s.lines.map(&:chomp).reject { |value| value.starts_with?("*.") }
|
27
32
|
values.map { |value| Models::Artifact.new(data: value, metadata: result) }
|
28
33
|
end.flatten
|
data/lib/mihari/clients/crtsh.rb
CHANGED
@@ -19,7 +19,7 @@ module Mihari
|
|
19
19
|
# Search crt.sh by a given identity
|
20
20
|
#
|
21
21
|
# @param [String] identity
|
22
|
-
# @param [String, nil] match "=", "ILIKE", "LIKE", "single", "any" or nil
|
22
|
+
# @param [String, nil] match "=", "ILIKE", "LIKE", "single", "any", "FTS" or nil
|
23
23
|
# @param [String, nil] exclude "expired" or nil
|
24
24
|
#
|
25
25
|
# @return [Array<Hash>]
|
@@ -87,7 +87,7 @@ module Mihari
|
|
87
87
|
def enrich_whois(enricher = Enrichers::Whois.new)
|
88
88
|
return unless can_enrich_whois?
|
89
89
|
|
90
|
-
self.whois_record =
|
90
|
+
self.whois_record = Services::WhoisRecordBuilder.call(domain, enricher: enricher)
|
91
91
|
end
|
92
92
|
|
93
93
|
#
|
@@ -98,7 +98,7 @@ module Mihari
|
|
98
98
|
def enrich_dns(enricher = Enrichers::GooglePublicDNS.new)
|
99
99
|
return unless can_enrich_dns?
|
100
100
|
|
101
|
-
self.dns_records =
|
101
|
+
self.dns_records = Services::DnsRecordBuilder.call(domain, enricher: enricher)
|
102
102
|
end
|
103
103
|
|
104
104
|
#
|
@@ -109,7 +109,7 @@ module Mihari
|
|
109
109
|
def enrich_reverse_dns(enricher = Enrichers::Shodan.new)
|
110
110
|
return unless can_enrich_reverse_dns?
|
111
111
|
|
112
|
-
self.reverse_dns_names =
|
112
|
+
self.reverse_dns_names = Services::ReverseDnsNameBuilder.call(data, enricher: enricher)
|
113
113
|
end
|
114
114
|
|
115
115
|
#
|
@@ -120,7 +120,7 @@ module Mihari
|
|
120
120
|
def enrich_geolocation(enricher = Enrichers::MMDB.new)
|
121
121
|
return unless can_enrich_geolocation?
|
122
122
|
|
123
|
-
self.geolocation =
|
123
|
+
self.geolocation = Services::GeolocationBuilder.call(data, enricher: enricher)
|
124
124
|
end
|
125
125
|
|
126
126
|
#
|
@@ -131,7 +131,7 @@ module Mihari
|
|
131
131
|
def enrich_autonomous_system(enricher = Enrichers::MMDB.new)
|
132
132
|
return unless can_enrich_autonomous_system?
|
133
133
|
|
134
|
-
self.autonomous_system =
|
134
|
+
self.autonomous_system = Services::AutonomousSystemBuilder.call(data, enricher: enricher)
|
135
135
|
end
|
136
136
|
|
137
137
|
#
|
@@ -142,7 +142,7 @@ module Mihari
|
|
142
142
|
def enrich_ports(enricher = Enrichers::Shodan.new)
|
143
143
|
return unless can_enrich_ports?
|
144
144
|
|
145
|
-
self.ports =
|
145
|
+
self.ports = Services::PortBuilder.call(data, enricher: enricher)
|
146
146
|
end
|
147
147
|
|
148
148
|
#
|
@@ -153,7 +153,7 @@ module Mihari
|
|
153
153
|
def enrich_cpes(enricher = Enrichers::Shodan.new)
|
154
154
|
return unless can_enrich_cpes?
|
155
155
|
|
156
|
-
self.cpes =
|
156
|
+
self.cpes = Services::CPEBuilder.call(data, enricher: enricher)
|
157
157
|
end
|
158
158
|
|
159
159
|
#
|
@@ -225,10 +225,16 @@ module Mihari
|
|
225
225
|
@shodan ||= Enrichers::Shodan.new
|
226
226
|
end
|
227
227
|
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
228
|
+
#
|
229
|
+
# @return [String, nil]
|
230
|
+
#
|
231
|
+
def domain
|
232
|
+
case data_type
|
233
|
+
when "domain"
|
234
|
+
data
|
235
|
+
when "url"
|
236
|
+
Addressable::URI.parse(data).host
|
237
|
+
end
|
232
238
|
end
|
233
239
|
|
234
240
|
def can_enrich_whois?
|
@@ -7,22 +7,6 @@ module Mihari
|
|
7
7
|
#
|
8
8
|
class AutonomousSystem < ActiveRecord::Base
|
9
9
|
belongs_to :artifact
|
10
|
-
|
11
|
-
class << self
|
12
|
-
#
|
13
|
-
# Build AS
|
14
|
-
#
|
15
|
-
# @param [String] ip
|
16
|
-
# @param [Mihari::Enrichers::MMDB] enricher
|
17
|
-
#
|
18
|
-
# @return [Mihari::AutonomousSystem, nil]
|
19
|
-
#
|
20
|
-
def build_by_ip(ip, enricher: Enrichers::MMDB.new)
|
21
|
-
enricher.result(ip).fmap do |res|
|
22
|
-
new(asn: res.asn) if res.asn
|
23
|
-
end.value_or nil
|
24
|
-
end
|
25
|
-
end
|
26
10
|
end
|
27
11
|
end
|
28
12
|
end
|
data/lib/mihari/models/cpe.rb
CHANGED
@@ -7,22 +7,6 @@ module Mihari
|
|
7
7
|
#
|
8
8
|
class CPE < ActiveRecord::Base
|
9
9
|
belongs_to :artifact
|
10
|
-
|
11
|
-
class << self
|
12
|
-
#
|
13
|
-
# Build CPEs
|
14
|
-
#
|
15
|
-
# @param [String] ip
|
16
|
-
# @param [Mihari::Enrichers::Shodan] enricher
|
17
|
-
#
|
18
|
-
# @return [Array<Mihari::CPE>]
|
19
|
-
#
|
20
|
-
def build_by_ip(ip, enricher: Enrichers::Shodan.new)
|
21
|
-
enricher.result(ip).fmap do |res|
|
22
|
-
(res&.cpes || []).map { |cpe| new(cpe: cpe) }
|
23
|
-
end.value_or []
|
24
|
-
end
|
25
|
-
end
|
26
10
|
end
|
27
11
|
end
|
28
12
|
end
|
data/lib/mihari/models/dns.rb
CHANGED
@@ -7,22 +7,6 @@ module Mihari
|
|
7
7
|
#
|
8
8
|
class DnsRecord < ActiveRecord::Base
|
9
9
|
belongs_to :artifact
|
10
|
-
|
11
|
-
class << self
|
12
|
-
#
|
13
|
-
# Build DNS records
|
14
|
-
#
|
15
|
-
# @param [String] domain
|
16
|
-
# @param [Mihari::Enrichers::Shodan] enricher
|
17
|
-
#
|
18
|
-
# @return [Array<Mihari::Models::DnsRecord>]
|
19
|
-
#
|
20
|
-
def build_by_domain(domain, enricher: Enrichers::GooglePublicDNS.new)
|
21
|
-
enricher.result(domain).fmap do |res|
|
22
|
-
res.answers.map { |answer| new(resource: answer.resource_type, value: answer.data) }
|
23
|
-
end.value_or([])
|
24
|
-
end
|
25
|
-
end
|
26
10
|
end
|
27
11
|
end
|
28
12
|
end
|
@@ -9,25 +9,6 @@ module Mihari
|
|
9
9
|
#
|
10
10
|
class Geolocation < ActiveRecord::Base
|
11
11
|
belongs_to :artifact
|
12
|
-
|
13
|
-
class << self
|
14
|
-
#
|
15
|
-
# Build Geolocation
|
16
|
-
#
|
17
|
-
# @param [String] ip
|
18
|
-
# @param [Mihari::Enrichers::MMDB] enricher
|
19
|
-
#
|
20
|
-
# @return [Mihari::Geolocation, nil]
|
21
|
-
#
|
22
|
-
def build_by_ip(ip, enricher: Enrichers::MMDB.new)
|
23
|
-
enricher.result(ip).fmap do |res|
|
24
|
-
if res.country_code
|
25
|
-
new(country: NormalizeCountry(res.country_code, to: :short),
|
26
|
-
country_code: res.country_code)
|
27
|
-
end
|
28
|
-
end.value_or nil
|
29
|
-
end
|
30
|
-
end
|
31
12
|
end
|
32
13
|
end
|
33
14
|
end
|
@@ -7,24 +7,6 @@ module Mihari
|
|
7
7
|
#
|
8
8
|
class ReverseDnsName < ActiveRecord::Base
|
9
9
|
belongs_to :artifact
|
10
|
-
|
11
|
-
class << self
|
12
|
-
include Dry::Monads[:result]
|
13
|
-
|
14
|
-
#
|
15
|
-
# Build reverse DNS names
|
16
|
-
#
|
17
|
-
# @param [String] ip
|
18
|
-
# @param [Mihari::Enrichers::Shodan] enricher
|
19
|
-
#
|
20
|
-
# @return [Array<Mihari::Models::ReverseDnsName>]
|
21
|
-
#
|
22
|
-
def build_by_ip(ip, enricher: Enrichers::Shodan.new)
|
23
|
-
enricher.result(ip).fmap do |res|
|
24
|
-
(res&.hostnames || []).map { |name| new(name: name) }
|
25
|
-
end.value_or []
|
26
|
-
end
|
27
|
-
end
|
28
10
|
end
|
29
11
|
end
|
30
12
|
end
|
data/lib/mihari/models/whois.rb
CHANGED
@@ -7,20 +7,6 @@ module Mihari
|
|
7
7
|
#
|
8
8
|
class WhoisRecord < ActiveRecord::Base
|
9
9
|
belongs_to :artifact
|
10
|
-
|
11
|
-
class << self
|
12
|
-
#
|
13
|
-
# Build whois record
|
14
|
-
#
|
15
|
-
# @param [String] domain
|
16
|
-
# @param [Mihari::Enrichers::Whois] enricher
|
17
|
-
#
|
18
|
-
# @return [WhoisRecord, nil]
|
19
|
-
#
|
20
|
-
def build_by_domain(domain, enricher: Enrichers::Whois.new)
|
21
|
-
enricher.result(domain).value_or nil
|
22
|
-
end
|
23
|
-
end
|
24
10
|
end
|
25
11
|
end
|
26
12
|
end
|
@@ -91,6 +91,7 @@ module Mihari
|
|
91
91
|
required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Crtsh.class_keys))
|
92
92
|
required(:query).value(:string)
|
93
93
|
optional(:exclude_expired).value(:bool).default(true)
|
94
|
+
optional(:match).value(Types::String.enum("=", "ILIKE", "LIKE", "single", "any", "FTS")).default(nil)
|
94
95
|
optional(:options).hash(AnalyzerOptions)
|
95
96
|
end
|
96
97
|
|
@@ -6,35 +6,152 @@ module Mihari
|
|
6
6
|
# Rule builder
|
7
7
|
#
|
8
8
|
class RuleBuilder < Service
|
9
|
-
# @return [String]
|
10
|
-
attr_reader :path_or_id
|
11
|
-
|
12
9
|
#
|
13
10
|
# @param [String] path_or_id
|
14
11
|
#
|
15
|
-
# @return [
|
12
|
+
# @return [Mihari::Rule]
|
16
13
|
#
|
17
|
-
def
|
18
|
-
|
19
|
-
return
|
14
|
+
def call(path_or_id)
|
15
|
+
res = Try { Rule.from_model Mihari::Models::Rule.find(path_or_id) }
|
16
|
+
return res.value! if res.value?
|
20
17
|
|
21
18
|
raise ArgumentError, "#{path_or_id} not found" unless Pathname(path_or_id).exist?
|
22
19
|
|
23
|
-
|
24
|
-
ERB.new(File.read(path_or_id)).result,
|
25
|
-
permitted_classes: [Date, Symbol]
|
26
|
-
)
|
20
|
+
Rule.from_yaml ERB.new(File.read(path_or_id)).result
|
27
21
|
end
|
22
|
+
end
|
28
23
|
|
24
|
+
#
|
25
|
+
# Autonomous system builder
|
26
|
+
#
|
27
|
+
class AutonomousSystemBuilder < Service
|
29
28
|
#
|
30
|
-
# @param [String]
|
29
|
+
# @param [String] ip
|
30
|
+
# @param [Mihari::Enrichers::MMDB] enricher
|
31
31
|
#
|
32
|
-
# @return [Mihari::
|
32
|
+
# @return [Mihari::Models::AutonomousSystem, nil]
|
33
33
|
#
|
34
|
-
def call(
|
35
|
-
|
34
|
+
def call(ip, enricher: Enrichers::MMDB.new)
|
35
|
+
enricher.result(ip).fmap do |res|
|
36
|
+
Models::AutonomousSystem.new(asn: res.asn) if res.asn
|
37
|
+
end.value_or nil
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
#
|
42
|
+
# CPE builder
|
43
|
+
#
|
44
|
+
class CPEBuilder < Service
|
45
|
+
#
|
46
|
+
# Build CPEs
|
47
|
+
#
|
48
|
+
# @param [String] ip
|
49
|
+
# @param [Mihari::Enrichers::Shodan] enricher
|
50
|
+
#
|
51
|
+
# @return [Array<Mihari::Models::CPE>]
|
52
|
+
#
|
53
|
+
def call(ip, enricher: Enrichers::Shodan.new)
|
54
|
+
enricher.result(ip).fmap do |res|
|
55
|
+
(res&.cpes || []).map { |cpe| Models::CPE.new(cpe: cpe) }
|
56
|
+
end.value_or []
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
#
|
61
|
+
# DNS record builder
|
62
|
+
#
|
63
|
+
class DnsRecordBuilder < Service
|
64
|
+
#
|
65
|
+
# Build DNS records
|
66
|
+
#
|
67
|
+
# @param [String] domain
|
68
|
+
# @param [Mihari::Enrichers::Shodan] enricher
|
69
|
+
#
|
70
|
+
# @return [Array<Mihari::Models::DnsRecord>]
|
71
|
+
#
|
72
|
+
def call(domain, enricher: Enrichers::GooglePublicDNS.new)
|
73
|
+
enricher.result(domain).fmap do |res|
|
74
|
+
res.answers.map { |answer| Models::DnsRecord.new(resource: answer.resource_type, value: answer.data) }
|
75
|
+
end.value_or []
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
#
|
80
|
+
# Geolocation builder
|
81
|
+
#
|
82
|
+
class GeolocationBuilder < Service
|
83
|
+
#
|
84
|
+
# Build Geolocation
|
85
|
+
#
|
86
|
+
# @param [String] ip
|
87
|
+
# @param [Mihari::Enrichers::MMDB] enricher
|
88
|
+
#
|
89
|
+
# @return [Mihari::Models::Geolocation, nil]
|
90
|
+
#
|
91
|
+
def call(ip, enricher: Enrichers::MMDB.new)
|
92
|
+
enricher.result(ip).fmap do |res|
|
93
|
+
if res.country_code
|
94
|
+
Models::Geolocation.new(
|
95
|
+
country: NormalizeCountry(res.country_code, to: :short),
|
96
|
+
country_code: res.country_code
|
97
|
+
)
|
98
|
+
end
|
99
|
+
end.value_or nil
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
#
|
104
|
+
# Port builder
|
105
|
+
#
|
106
|
+
class PortBuilder < Service
|
107
|
+
#
|
108
|
+
# Build ports
|
109
|
+
#
|
110
|
+
# @param [String] ip
|
111
|
+
# @param [Mihari::Enrichers::Shodan] enricher
|
112
|
+
#
|
113
|
+
# @return [Array<Mihari::Models::Port>]
|
114
|
+
#
|
115
|
+
def call(ip, enricher: Enrichers::Shodan.new)
|
116
|
+
enricher.result(ip).fmap do |res|
|
117
|
+
(res&.ports || []).map { |port| Models::Port.new(port: port) }
|
118
|
+
end.value_or []
|
119
|
+
end
|
120
|
+
end
|
36
121
|
|
37
|
-
|
122
|
+
#
|
123
|
+
# Reverse DNS name builder
|
124
|
+
#
|
125
|
+
class ReverseDnsNameBuilder < Service
|
126
|
+
#
|
127
|
+
# Build reverse DNS names
|
128
|
+
#
|
129
|
+
# @param [String] ip
|
130
|
+
# @param [Mihari::Enrichers::Shodan] enricher
|
131
|
+
#
|
132
|
+
# @return [Array<Mihari::Models::ReverseDnsName>]
|
133
|
+
#
|
134
|
+
def call(ip, enricher: Enrichers::Shodan.new)
|
135
|
+
enricher.result(ip).fmap do |res|
|
136
|
+
(res&.hostnames || []).map { |name| Models::ReverseDnsName.new(name: name) }
|
137
|
+
end.value_or []
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
#
|
142
|
+
# Whois record builder
|
143
|
+
#
|
144
|
+
class WhoisRecordBuilder < Service
|
145
|
+
#
|
146
|
+
# Build whois record
|
147
|
+
#
|
148
|
+
# @param [String] domain
|
149
|
+
# @param [Mihari::Enrichers::Whois] enricher
|
150
|
+
#
|
151
|
+
# @return [Mihari::Models::WhoisRecord, nil]
|
152
|
+
#
|
153
|
+
def call(domain, enricher: Enrichers::Whois.new)
|
154
|
+
enricher.result(domain).value_or nil
|
38
155
|
end
|
39
156
|
end
|
40
157
|
end
|
data/lib/mihari/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mihari
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 7.0.
|
4
|
+
version: 7.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Manabu Niseki
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-01-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: better_errors
|