mihari 3.5.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/config.ru +1 -0
- data/lib/mihari/analyzers/base.rb +34 -6
- data/lib/mihari/analyzers/censys.rb +37 -9
- data/lib/mihari/analyzers/onyphe.rb +34 -9
- data/lib/mihari/analyzers/shodan.rb +26 -5
- data/lib/mihari/{constraints.rb → constants.rb} +0 -0
- data/lib/mihari/database.rb +42 -3
- data/lib/mihari/models/alert.rb +8 -4
- data/lib/mihari/models/artifact.rb +55 -0
- data/lib/mihari/models/autonomous_system.rb +9 -0
- data/lib/mihari/models/dns.rb +53 -0
- data/lib/mihari/models/geolocation.rb +9 -0
- data/lib/mihari/models/reverse_dns.rb +24 -0
- data/lib/mihari/models/whois.rb +119 -0
- data/lib/mihari/schemas/rule.rb +2 -15
- data/lib/mihari/serializers/alert.rb +6 -4
- data/lib/mihari/serializers/artifact.rb +11 -2
- data/lib/mihari/serializers/autonomous_system.rb +9 -0
- data/lib/mihari/serializers/dns.rb +11 -0
- data/lib/mihari/serializers/geolocation.rb +11 -0
- data/lib/mihari/serializers/reverse_dns.rb +11 -0
- data/lib/mihari/serializers/tag.rb +4 -2
- data/lib/mihari/serializers/whois.rb +11 -0
- data/lib/mihari/structs/censys.rb +92 -0
- data/lib/mihari/structs/onyphe.rb +47 -0
- data/lib/mihari/structs/shodan.rb +53 -0
- data/lib/mihari/types.rb +21 -0
- data/lib/mihari/version.rb +1 -1
- data/lib/mihari/web/controllers/artifacts_controller.rb +26 -7
- data/lib/mihari/web/controllers/sources_controller.rb +2 -2
- data/lib/mihari/web/public/index.html +1 -1
- data/lib/mihari/web/public/redoc-static.html +2 -2
- data/lib/mihari/web/public/static/js/app.8e3e5150.js +36 -0
- data/lib/mihari/web/public/static/js/app.8e3e5150.js.map +1 -0
- data/lib/mihari.rb +24 -4
- data/mihari.gemspec +7 -1
- metadata +106 -6
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_record"
|
4
|
+
require "resolv"
|
5
|
+
|
6
|
+
module Mihari
|
7
|
+
class DnsRecord < ActiveRecord::Base
|
8
|
+
class << self
|
9
|
+
#
|
10
|
+
# Build DNS records
|
11
|
+
#
|
12
|
+
# @param [String] domain
|
13
|
+
#
|
14
|
+
# @return [Array<Mihari::DnsRecord>]
|
15
|
+
#
|
16
|
+
def build_by_domain(domain)
|
17
|
+
resource_types = [
|
18
|
+
Resolv::DNS::Resource::IN::A,
|
19
|
+
Resolv::DNS::Resource::IN::AAAA,
|
20
|
+
Resolv::DNS::Resource::IN::CNAME,
|
21
|
+
Resolv::DNS::Resource::IN::TXT,
|
22
|
+
Resolv::DNS::Resource::IN::NS
|
23
|
+
]
|
24
|
+
|
25
|
+
resource_types.map do |resource_type|
|
26
|
+
get_values domain, resource_type
|
27
|
+
rescue Resolv::ResolvError
|
28
|
+
nil
|
29
|
+
end.flatten.compact
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def get_values(domain, resource_type)
|
35
|
+
resources = Resolv::DNS.new.getresources(domain, resource_type)
|
36
|
+
resource_name = resource_type.to_s.split("::").last
|
37
|
+
|
38
|
+
resources.map do |resource|
|
39
|
+
# A, AAAA
|
40
|
+
if resource.respond_to?(:address)
|
41
|
+
new(resource: resource_name, value: resource.address.to_s)
|
42
|
+
# CNAME, NS
|
43
|
+
elsif resource.respond_to?(:name)
|
44
|
+
new(resource: resource_name, value: resource.name.to_s)
|
45
|
+
# TXT
|
46
|
+
elsif resource.respond_to?(:data)
|
47
|
+
new(resource: resource_name, value: resource.data.to_s)
|
48
|
+
end
|
49
|
+
end.compact
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_record"
|
4
|
+
require "resolv"
|
5
|
+
|
6
|
+
module Mihari
|
7
|
+
class ReverseDnsName < ActiveRecord::Base
|
8
|
+
class << self
|
9
|
+
#
|
10
|
+
# Build reverse DNS names
|
11
|
+
#
|
12
|
+
# @param [String] ip
|
13
|
+
#
|
14
|
+
# @return [Array<Mihari::ReverseDnsName>]
|
15
|
+
#
|
16
|
+
def build_by_ip(ip)
|
17
|
+
names = Resolv.getnames(ip)
|
18
|
+
names.map { |name| new(name: name) }
|
19
|
+
rescue Resolv::ResolvError
|
20
|
+
[]
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_record"
|
4
|
+
require "whois-parser"
|
5
|
+
require "public_suffix"
|
6
|
+
|
7
|
+
module Mihari
|
8
|
+
class WhoisRecord < ActiveRecord::Base
|
9
|
+
has_one :artifact, dependent: :destroy
|
10
|
+
|
11
|
+
@memo = {}
|
12
|
+
|
13
|
+
class << self
|
14
|
+
#
|
15
|
+
# Build whois record
|
16
|
+
#
|
17
|
+
# @param [Stinrg] domain
|
18
|
+
#
|
19
|
+
# @return [WhoisRecord, nil]
|
20
|
+
#
|
21
|
+
def build_by_domain(domain)
|
22
|
+
domain = PublicSuffix.domain(domain)
|
23
|
+
|
24
|
+
# check memo
|
25
|
+
if @memo.key?(domain)
|
26
|
+
whois_record = @memo[domain]
|
27
|
+
# return clone of the record
|
28
|
+
return whois_record.dup
|
29
|
+
end
|
30
|
+
|
31
|
+
record = Whois.whois(domain)
|
32
|
+
parser = record.parser
|
33
|
+
|
34
|
+
return nil if parser.available?
|
35
|
+
|
36
|
+
whois_record = new(
|
37
|
+
domain: domain,
|
38
|
+
created_on: get_created_on(parser),
|
39
|
+
updated_on: get_updated_on(parser),
|
40
|
+
expires_on: get_expires_on(parser),
|
41
|
+
registrar: get_registrar(parser),
|
42
|
+
contacts: get_contacts(parser)
|
43
|
+
)
|
44
|
+
# set memo
|
45
|
+
@memo[domain] = whois_record
|
46
|
+
whois_record
|
47
|
+
rescue Whois::Error, Whois::ParserError, Timeout::Error
|
48
|
+
nil
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
#
|
54
|
+
# Get created_on
|
55
|
+
#
|
56
|
+
# @param [::Whois::Parser:] parser
|
57
|
+
#
|
58
|
+
# @return [Date, nil]
|
59
|
+
#
|
60
|
+
def get_created_on(parser)
|
61
|
+
parser.created_on
|
62
|
+
rescue ::Whois::AttributeNotImplemented
|
63
|
+
nil
|
64
|
+
end
|
65
|
+
|
66
|
+
#
|
67
|
+
# Get updated_on
|
68
|
+
#
|
69
|
+
# @param [::Whois::Parser:] parser
|
70
|
+
#
|
71
|
+
# @return [Date, nil]
|
72
|
+
#
|
73
|
+
def get_updated_on(parser)
|
74
|
+
parser.updated_on
|
75
|
+
rescue ::Whois::AttributeNotImplemented
|
76
|
+
nil
|
77
|
+
end
|
78
|
+
|
79
|
+
#
|
80
|
+
# Get expires_on
|
81
|
+
#
|
82
|
+
# @param [::Whois::Parser:] parser
|
83
|
+
#
|
84
|
+
# @return [Date, nil]
|
85
|
+
#
|
86
|
+
def get_expires_on(parser)
|
87
|
+
parser.expires_on
|
88
|
+
rescue ::Whois::AttributeNotImplemented
|
89
|
+
nil
|
90
|
+
end
|
91
|
+
|
92
|
+
#
|
93
|
+
# Get registrar
|
94
|
+
#
|
95
|
+
# @param [::Whois::Parser:] parser
|
96
|
+
#
|
97
|
+
# @return [Hash, nil]
|
98
|
+
#
|
99
|
+
def get_registrar(parser)
|
100
|
+
parser.registrar&.to_h
|
101
|
+
rescue ::Whois::AttributeNotImplemented
|
102
|
+
nil
|
103
|
+
end
|
104
|
+
|
105
|
+
#
|
106
|
+
# Get contacts
|
107
|
+
#
|
108
|
+
# @param [::Whois::Parser:] parser
|
109
|
+
#
|
110
|
+
# @return [Array[Hash], nil]
|
111
|
+
#
|
112
|
+
def get_contacts(parser)
|
113
|
+
parser.contacts.map(&:to_h)
|
114
|
+
rescue ::Whois::AttributeNotImplemented
|
115
|
+
nil
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
data/lib/mihari/schemas/rule.rb
CHANGED
@@ -2,26 +2,13 @@
|
|
2
2
|
|
3
3
|
require "dry/schema"
|
4
4
|
require "dry/validation"
|
5
|
-
require "dry/types"
|
6
5
|
|
7
6
|
require "mihari/schemas/macros"
|
8
7
|
|
9
8
|
module Mihari
|
10
|
-
module Types
|
11
|
-
include Dry.Types()
|
12
|
-
end
|
13
|
-
|
14
|
-
DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
|
15
|
-
|
16
|
-
AnalyzerTypes = Types::String.enum(
|
17
|
-
"binaryedge", "censys", "circl", "dnpedia", "dnstwister",
|
18
|
-
"onyphe", "otx", "passivetotal", "pulsedive", "securitytrails",
|
19
|
-
"shodan", "virustotal"
|
20
|
-
)
|
21
|
-
|
22
9
|
module Schemas
|
23
10
|
Analyzer = Dry::Schema.Params do
|
24
|
-
required(:analyzer).value(AnalyzerTypes)
|
11
|
+
required(:analyzer).value(Types::AnalyzerTypes)
|
25
12
|
required(:query).value(:string)
|
26
13
|
end
|
27
14
|
|
@@ -62,7 +49,7 @@ module Mihari
|
|
62
49
|
|
63
50
|
required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh }
|
64
51
|
|
65
|
-
optional(:allowed_data_types).value(array[DataTypes]).default(ALLOWED_DATA_TYPES)
|
52
|
+
optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
|
66
53
|
optional(:disallowed_data_values).value(array[:string]).default([])
|
67
54
|
|
68
55
|
optional(:ignore_old_artifacts).value(:bool).default(false)
|
@@ -3,10 +3,12 @@
|
|
3
3
|
require "active_model_serializers"
|
4
4
|
|
5
5
|
module Mihari
|
6
|
-
|
7
|
-
|
6
|
+
module Serializers
|
7
|
+
class AlertSerializer < ActiveModel::Serializer
|
8
|
+
attributes :id, :title, :description, :source, :created_at
|
8
9
|
|
9
|
-
|
10
|
-
|
10
|
+
has_many :artifacts, serializer: ArtifactSerializer
|
11
|
+
has_many :tags, through: :taggings, serializer: TagSerializer
|
12
|
+
end
|
11
13
|
end
|
12
14
|
end
|
@@ -3,7 +3,16 @@
|
|
3
3
|
require "active_model_serializers"
|
4
4
|
|
5
5
|
module Mihari
|
6
|
-
|
7
|
-
|
6
|
+
module Serializers
|
7
|
+
class ArtifactSerializer < ActiveModel::Serializer
|
8
|
+
attributes :id, :data, :data_type, :source
|
9
|
+
|
10
|
+
has_one :autonomous_system, serializer: AutonomousSystemSerializer
|
11
|
+
has_one :geolocation, serializer: GeolocationSerializer
|
12
|
+
has_one :whois_record, serializer: WhoisRecordSerializer
|
13
|
+
|
14
|
+
has_many :dns_records, serializer: DnsRecordSerializer
|
15
|
+
has_many :reverse_dns_names, serializer: ReverseDnsNameSerializer
|
16
|
+
end
|
8
17
|
end
|
9
18
|
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_model_serializers"
|
4
|
+
|
5
|
+
module Mihari
|
6
|
+
module Serializers
|
7
|
+
class WhoisRecordSerializer < ActiveModel::Serializer
|
8
|
+
attributes :domain, :created_on, :updated_on, :expires_on, :registrar, :contacts
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require "json"
|
2
|
+
require "dry/struct"
|
3
|
+
|
4
|
+
module Mihari
|
5
|
+
module Structs
|
6
|
+
module Censys
|
7
|
+
class AutonomousSystem < Dry::Struct
|
8
|
+
attribute :asn, Types::Int
|
9
|
+
|
10
|
+
def self.from_dynamic!(d)
|
11
|
+
d = Types::Hash[d]
|
12
|
+
new(
|
13
|
+
asn: d.fetch("asn")
|
14
|
+
)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
class Location < Dry::Struct
|
19
|
+
attribute :country, Types::String.optional
|
20
|
+
attribute :country_code, Types::String.optional
|
21
|
+
|
22
|
+
def self.from_dynamic!(d)
|
23
|
+
d = Types::Hash[d]
|
24
|
+
new(
|
25
|
+
country: d["country"],
|
26
|
+
country_code: d["country_code"]
|
27
|
+
)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class Hit < Dry::Struct
|
32
|
+
attribute :ip, Types::String
|
33
|
+
attribute :location, Location
|
34
|
+
attribute :autonomous_system, AutonomousSystem
|
35
|
+
|
36
|
+
def self.from_dynamic!(d)
|
37
|
+
d = Types::Hash[d]
|
38
|
+
new(
|
39
|
+
ip: d.fetch("ip"),
|
40
|
+
location: Location.from_dynamic!(d.fetch("location")),
|
41
|
+
autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system"))
|
42
|
+
)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
class Links < Dry::Struct
|
47
|
+
attribute :next, Types::String
|
48
|
+
attribute :prev, Types::String
|
49
|
+
|
50
|
+
def self.from_dynamic!(d)
|
51
|
+
d = Types::Hash[d]
|
52
|
+
new(
|
53
|
+
next: d.fetch("next"),
|
54
|
+
prev: d.fetch("prev")
|
55
|
+
)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
class Result < Dry::Struct
|
60
|
+
attribute :query, Types::String
|
61
|
+
attribute :total, Types::Int
|
62
|
+
attribute :hits, Types.Array(Hit)
|
63
|
+
attribute :links, Links
|
64
|
+
|
65
|
+
def self.from_dynamic!(d)
|
66
|
+
d = Types::Hash[d]
|
67
|
+
new(
|
68
|
+
query: d.fetch("query"),
|
69
|
+
total: d.fetch("total"),
|
70
|
+
hits: d.fetch("hits", []).map { |x| Hit.from_dynamic!(x) },
|
71
|
+
links: Links.from_dynamic!(d.fetch("links"))
|
72
|
+
)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
class Response < Dry::Struct
|
77
|
+
attribute :code, Types::Int
|
78
|
+
attribute :status, Types::String
|
79
|
+
attribute :result, Result
|
80
|
+
|
81
|
+
def self.from_dynamic!(d)
|
82
|
+
d = Types::Hash[d]
|
83
|
+
new(
|
84
|
+
code: d.fetch("code"),
|
85
|
+
status: d.fetch("status"),
|
86
|
+
result: Result.from_dynamic!(d.fetch("result"))
|
87
|
+
)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|