mihari 3.5.0 → 3.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/config.ru +1 -0
  3. data/lib/mihari/analyzers/base.rb +34 -6
  4. data/lib/mihari/analyzers/censys.rb +37 -9
  5. data/lib/mihari/analyzers/onyphe.rb +34 -9
  6. data/lib/mihari/analyzers/shodan.rb +26 -5
  7. data/lib/mihari/{constraints.rb → constants.rb} +0 -0
  8. data/lib/mihari/database.rb +42 -3
  9. data/lib/mihari/models/alert.rb +8 -4
  10. data/lib/mihari/models/artifact.rb +55 -0
  11. data/lib/mihari/models/autonomous_system.rb +9 -0
  12. data/lib/mihari/models/dns.rb +53 -0
  13. data/lib/mihari/models/geolocation.rb +9 -0
  14. data/lib/mihari/models/reverse_dns.rb +24 -0
  15. data/lib/mihari/models/whois.rb +119 -0
  16. data/lib/mihari/schemas/rule.rb +2 -15
  17. data/lib/mihari/serializers/alert.rb +6 -4
  18. data/lib/mihari/serializers/artifact.rb +11 -2
  19. data/lib/mihari/serializers/autonomous_system.rb +9 -0
  20. data/lib/mihari/serializers/dns.rb +11 -0
  21. data/lib/mihari/serializers/geolocation.rb +11 -0
  22. data/lib/mihari/serializers/reverse_dns.rb +11 -0
  23. data/lib/mihari/serializers/tag.rb +4 -2
  24. data/lib/mihari/serializers/whois.rb +11 -0
  25. data/lib/mihari/structs/censys.rb +92 -0
  26. data/lib/mihari/structs/onyphe.rb +47 -0
  27. data/lib/mihari/structs/shodan.rb +53 -0
  28. data/lib/mihari/types.rb +21 -0
  29. data/lib/mihari/version.rb +1 -1
  30. data/lib/mihari/web/controllers/artifacts_controller.rb +26 -7
  31. data/lib/mihari/web/controllers/sources_controller.rb +2 -2
  32. data/lib/mihari/web/public/index.html +1 -1
  33. data/lib/mihari/web/public/redoc-static.html +2 -2
  34. data/lib/mihari/web/public/static/js/app.8e3e5150.js +36 -0
  35. data/lib/mihari/web/public/static/js/app.8e3e5150.js.map +1 -0
  36. data/lib/mihari.rb +24 -4
  37. data/mihari.gemspec +7 -1
  38. metadata +106 -6
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ module Mihari
6
+ class AutonomousSystem < ActiveRecord::Base
7
+ has_one :artifact, dependent: :destroy
8
+ end
9
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+ require "resolv"
5
+
6
+ module Mihari
7
+ class DnsRecord < ActiveRecord::Base
8
+ class << self
9
+ #
10
+ # Build DNS records
11
+ #
12
+ # @param [String] domain
13
+ #
14
+ # @return [Array<Mihari::DnsRecord>]
15
+ #
16
+ def build_by_domain(domain)
17
+ resource_types = [
18
+ Resolv::DNS::Resource::IN::A,
19
+ Resolv::DNS::Resource::IN::AAAA,
20
+ Resolv::DNS::Resource::IN::CNAME,
21
+ Resolv::DNS::Resource::IN::TXT,
22
+ Resolv::DNS::Resource::IN::NS
23
+ ]
24
+
25
+ resource_types.map do |resource_type|
26
+ get_values domain, resource_type
27
+ rescue Resolv::ResolvError
28
+ nil
29
+ end.flatten.compact
30
+ end
31
+
32
+ private
33
+
34
+ def get_values(domain, resource_type)
35
+ resources = Resolv::DNS.new.getresources(domain, resource_type)
36
+ resource_name = resource_type.to_s.split("::").last
37
+
38
+ resources.map do |resource|
39
+ # A, AAAA
40
+ if resource.respond_to?(:address)
41
+ new(resource: resource_name, value: resource.address.to_s)
42
+ # CNAME, NS
43
+ elsif resource.respond_to?(:name)
44
+ new(resource: resource_name, value: resource.name.to_s)
45
+ # TXT
46
+ elsif resource.respond_to?(:data)
47
+ new(resource: resource_name, value: resource.data.to_s)
48
+ end
49
+ end.compact
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ module Mihari
6
+ class Geolocation < ActiveRecord::Base
7
+ has_one :artifact, dependent: :destroy
8
+ end
9
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+ require "resolv"
5
+
6
+ module Mihari
7
+ class ReverseDnsName < ActiveRecord::Base
8
+ class << self
9
+ #
10
+ # Build reverse DNS names
11
+ #
12
+ # @param [String] ip
13
+ #
14
+ # @return [Array<Mihari::ReverseDnsName>]
15
+ #
16
+ def build_by_ip(ip)
17
+ names = Resolv.getnames(ip)
18
+ names.map { |name| new(name: name) }
19
+ rescue Resolv::ResolvError
20
+ []
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,119 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+ require "whois-parser"
5
+ require "public_suffix"
6
+
7
+ module Mihari
8
+ class WhoisRecord < ActiveRecord::Base
9
+ has_one :artifact, dependent: :destroy
10
+
11
+ @memo = {}
12
+
13
+ class << self
14
+ #
15
+ # Build whois record
16
+ #
17
+ # @param [Stinrg] domain
18
+ #
19
+ # @return [WhoisRecord, nil]
20
+ #
21
+ def build_by_domain(domain)
22
+ domain = PublicSuffix.domain(domain)
23
+
24
+ # check memo
25
+ if @memo.key?(domain)
26
+ whois_record = @memo[domain]
27
+ # return clone of the record
28
+ return whois_record.dup
29
+ end
30
+
31
+ record = Whois.whois(domain)
32
+ parser = record.parser
33
+
34
+ return nil if parser.available?
35
+
36
+ whois_record = new(
37
+ domain: domain,
38
+ created_on: get_created_on(parser),
39
+ updated_on: get_updated_on(parser),
40
+ expires_on: get_expires_on(parser),
41
+ registrar: get_registrar(parser),
42
+ contacts: get_contacts(parser)
43
+ )
44
+ # set memo
45
+ @memo[domain] = whois_record
46
+ whois_record
47
+ rescue Whois::Error, Whois::ParserError, Timeout::Error
48
+ nil
49
+ end
50
+
51
+ private
52
+
53
+ #
54
+ # Get created_on
55
+ #
56
+ # @param [::Whois::Parser:] parser
57
+ #
58
+ # @return [Date, nil]
59
+ #
60
+ def get_created_on(parser)
61
+ parser.created_on
62
+ rescue ::Whois::AttributeNotImplemented
63
+ nil
64
+ end
65
+
66
+ #
67
+ # Get updated_on
68
+ #
69
+ # @param [::Whois::Parser:] parser
70
+ #
71
+ # @return [Date, nil]
72
+ #
73
+ def get_updated_on(parser)
74
+ parser.updated_on
75
+ rescue ::Whois::AttributeNotImplemented
76
+ nil
77
+ end
78
+
79
+ #
80
+ # Get expires_on
81
+ #
82
+ # @param [::Whois::Parser:] parser
83
+ #
84
+ # @return [Date, nil]
85
+ #
86
+ def get_expires_on(parser)
87
+ parser.expires_on
88
+ rescue ::Whois::AttributeNotImplemented
89
+ nil
90
+ end
91
+
92
+ #
93
+ # Get registrar
94
+ #
95
+ # @param [::Whois::Parser:] parser
96
+ #
97
+ # @return [Hash, nil]
98
+ #
99
+ def get_registrar(parser)
100
+ parser.registrar&.to_h
101
+ rescue ::Whois::AttributeNotImplemented
102
+ nil
103
+ end
104
+
105
+ #
106
+ # Get contacts
107
+ #
108
+ # @param [::Whois::Parser:] parser
109
+ #
110
+ # @return [Array[Hash], nil]
111
+ #
112
+ def get_contacts(parser)
113
+ parser.contacts.map(&:to_h)
114
+ rescue ::Whois::AttributeNotImplemented
115
+ nil
116
+ end
117
+ end
118
+ end
119
+ end
@@ -2,26 +2,13 @@
2
2
 
3
3
  require "dry/schema"
4
4
  require "dry/validation"
5
- require "dry/types"
6
5
 
7
6
  require "mihari/schemas/macros"
8
7
 
9
8
  module Mihari
10
- module Types
11
- include Dry.Types()
12
- end
13
-
14
- DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
15
-
16
- AnalyzerTypes = Types::String.enum(
17
- "binaryedge", "censys", "circl", "dnpedia", "dnstwister",
18
- "onyphe", "otx", "passivetotal", "pulsedive", "securitytrails",
19
- "shodan", "virustotal"
20
- )
21
-
22
9
  module Schemas
23
10
  Analyzer = Dry::Schema.Params do
24
- required(:analyzer).value(AnalyzerTypes)
11
+ required(:analyzer).value(Types::AnalyzerTypes)
25
12
  required(:query).value(:string)
26
13
  end
27
14
 
@@ -62,7 +49,7 @@ module Mihari
62
49
 
63
50
  required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh }
64
51
 
65
- optional(:allowed_data_types).value(array[DataTypes]).default(ALLOWED_DATA_TYPES)
52
+ optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
66
53
  optional(:disallowed_data_values).value(array[:string]).default([])
67
54
 
68
55
  optional(:ignore_old_artifacts).value(:bool).default(false)
@@ -3,10 +3,12 @@
3
3
  require "active_model_serializers"
4
4
 
5
5
  module Mihari
6
- class AlertSerializer < ActiveModel::Serializer
7
- attributes :id, :title, :description, :source, :created_at
6
+ module Serializers
7
+ class AlertSerializer < ActiveModel::Serializer
8
+ attributes :id, :title, :description, :source, :created_at
8
9
 
9
- has_many :artifacts
10
- has_many :tags, through: :taggings
10
+ has_many :artifacts, serializer: ArtifactSerializer
11
+ has_many :tags, through: :taggings, serializer: TagSerializer
12
+ end
11
13
  end
12
14
  end
@@ -3,7 +3,16 @@
3
3
  require "active_model_serializers"
4
4
 
5
5
  module Mihari
6
- class ArtifactSerializer < ActiveModel::Serializer
7
- attributes :id, :data, :data_type, :source
6
+ module Serializers
7
+ class ArtifactSerializer < ActiveModel::Serializer
8
+ attributes :id, :data, :data_type, :source
9
+
10
+ has_one :autonomous_system, serializer: AutonomousSystemSerializer
11
+ has_one :geolocation, serializer: GeolocationSerializer
12
+ has_one :whois_record, serializer: WhoisRecordSerializer
13
+
14
+ has_many :dns_records, serializer: DnsRecordSerializer
15
+ has_many :reverse_dns_names, serializer: ReverseDnsNameSerializer
16
+ end
8
17
  end
9
18
  end
@@ -0,0 +1,9 @@
1
+ require "active_model_serializers"
2
+
3
+ module Mihari
4
+ module Serializers
5
+ class AutonomousSystemSerializer < ActiveModel::Serializer
6
+ attributes :asn
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_model_serializers"
4
+
5
+ module Mihari
6
+ module Serializers
7
+ class DnsRecordSerializer < ActiveModel::Serializer
8
+ attributes :resource, :value
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_model_serializers"
4
+
5
+ module Mihari
6
+ module Serializers
7
+ class GeolocationSerializer < ActiveModel::Serializer
8
+ attributes :country, :country_code
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_model_serializers"
4
+
5
+ module Mihari
6
+ module Serializers
7
+ class ReverseDnsNameSerializer < ActiveModel::Serializer
8
+ attributes :name
9
+ end
10
+ end
11
+ end
@@ -3,7 +3,9 @@
3
3
  require "active_model_serializers"
4
4
 
5
5
  module Mihari
6
- class TagSerializer < ActiveModel::Serializer
7
- attributes :id, :name
6
+ module Serializers
7
+ class TagSerializer < ActiveModel::Serializer
8
+ attributes :id, :name
9
+ end
8
10
  end
9
11
  end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_model_serializers"
4
+
5
+ module Mihari
6
+ module Serializers
7
+ class WhoisRecordSerializer < ActiveModel::Serializer
8
+ attributes :domain, :created_on, :updated_on, :expires_on, :registrar, :contacts
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,92 @@
1
+ require "json"
2
+ require "dry/struct"
3
+
4
+ module Mihari
5
+ module Structs
6
+ module Censys
7
+ class AutonomousSystem < Dry::Struct
8
+ attribute :asn, Types::Int
9
+
10
+ def self.from_dynamic!(d)
11
+ d = Types::Hash[d]
12
+ new(
13
+ asn: d.fetch("asn")
14
+ )
15
+ end
16
+ end
17
+
18
+ class Location < Dry::Struct
19
+ attribute :country, Types::String.optional
20
+ attribute :country_code, Types::String.optional
21
+
22
+ def self.from_dynamic!(d)
23
+ d = Types::Hash[d]
24
+ new(
25
+ country: d["country"],
26
+ country_code: d["country_code"]
27
+ )
28
+ end
29
+ end
30
+
31
+ class Hit < Dry::Struct
32
+ attribute :ip, Types::String
33
+ attribute :location, Location
34
+ attribute :autonomous_system, AutonomousSystem
35
+
36
+ def self.from_dynamic!(d)
37
+ d = Types::Hash[d]
38
+ new(
39
+ ip: d.fetch("ip"),
40
+ location: Location.from_dynamic!(d.fetch("location")),
41
+ autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system"))
42
+ )
43
+ end
44
+ end
45
+
46
+ class Links < Dry::Struct
47
+ attribute :next, Types::String
48
+ attribute :prev, Types::String
49
+
50
+ def self.from_dynamic!(d)
51
+ d = Types::Hash[d]
52
+ new(
53
+ next: d.fetch("next"),
54
+ prev: d.fetch("prev")
55
+ )
56
+ end
57
+ end
58
+
59
+ class Result < Dry::Struct
60
+ attribute :query, Types::String
61
+ attribute :total, Types::Int
62
+ attribute :hits, Types.Array(Hit)
63
+ attribute :links, Links
64
+
65
+ def self.from_dynamic!(d)
66
+ d = Types::Hash[d]
67
+ new(
68
+ query: d.fetch("query"),
69
+ total: d.fetch("total"),
70
+ hits: d.fetch("hits", []).map { |x| Hit.from_dynamic!(x) },
71
+ links: Links.from_dynamic!(d.fetch("links"))
72
+ )
73
+ end
74
+ end
75
+
76
+ class Response < Dry::Struct
77
+ attribute :code, Types::Int
78
+ attribute :status, Types::String
79
+ attribute :result, Result
80
+
81
+ def self.from_dynamic!(d)
82
+ d = Types::Hash[d]
83
+ new(
84
+ code: d.fetch("code"),
85
+ status: d.fetch("status"),
86
+ result: Result.from_dynamic!(d.fetch("result"))
87
+ )
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end