mihari 5.1.0 → 5.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.gitmodules +0 -3
  3. data/.rubocop.yml +6 -0
  4. data/README.md +0 -1
  5. data/lib/mihari/analyzers/base.rb +32 -27
  6. data/lib/mihari/analyzers/binaryedge.rb +17 -9
  7. data/lib/mihari/analyzers/censys.rb +10 -54
  8. data/lib/mihari/analyzers/circl.rb +7 -6
  9. data/lib/mihari/analyzers/crtsh.rb +12 -7
  10. data/lib/mihari/analyzers/dnstwister.rb +7 -7
  11. data/lib/mihari/analyzers/feed.rb +33 -10
  12. data/lib/mihari/analyzers/greynoise.rb +8 -33
  13. data/lib/mihari/analyzers/onyphe.rb +10 -36
  14. data/lib/mihari/analyzers/otx.rb +4 -3
  15. data/lib/mihari/analyzers/passivetotal.rb +8 -7
  16. data/lib/mihari/analyzers/pulsedive.rb +8 -7
  17. data/lib/mihari/analyzers/rule.rb +0 -1
  18. data/lib/mihari/analyzers/securitytrails.rb +8 -10
  19. data/lib/mihari/analyzers/shodan.rb +16 -90
  20. data/lib/mihari/analyzers/urlscan.rb +16 -6
  21. data/lib/mihari/analyzers/virustotal.rb +8 -6
  22. data/lib/mihari/analyzers/virustotal_intelligence.rb +12 -7
  23. data/lib/mihari/analyzers/zoomeye.rb +13 -10
  24. data/lib/mihari/clients/base.rb +53 -0
  25. data/lib/mihari/clients/binaryedge.rb +38 -0
  26. data/lib/mihari/clients/censys.rb +42 -0
  27. data/lib/mihari/clients/circl.rb +59 -0
  28. data/lib/mihari/clients/crtsh.rb +31 -0
  29. data/lib/mihari/clients/dnstwister.rb +40 -0
  30. data/lib/mihari/clients/greynoise.rb +34 -0
  31. data/lib/mihari/clients/misp.rb +29 -0
  32. data/lib/mihari/clients/onyphe.rb +35 -0
  33. data/lib/mihari/clients/otx.rb +49 -0
  34. data/lib/mihari/clients/passivetotal.rb +69 -0
  35. data/lib/mihari/clients/publsedive.rb +56 -0
  36. data/lib/mihari/clients/securitytrails.rb +94 -0
  37. data/lib/mihari/clients/shodan.rb +41 -0
  38. data/lib/mihari/clients/the_hive.rb +33 -0
  39. data/lib/mihari/clients/urlscan.rb +33 -0
  40. data/lib/mihari/clients/virustotal.rb +62 -0
  41. data/lib/mihari/clients/zoomeye.rb +74 -0
  42. data/lib/mihari/commands/database.rb +1 -6
  43. data/lib/mihari/commands/searcher.rb +1 -2
  44. data/lib/mihari/database.rb +9 -0
  45. data/lib/mihari/emitters/misp.rb +13 -20
  46. data/lib/mihari/emitters/the_hive.rb +3 -5
  47. data/lib/mihari/emitters/webhook.rb +2 -2
  48. data/lib/mihari/feed/reader.rb +14 -11
  49. data/lib/mihari/http.rb +29 -21
  50. data/lib/mihari/mixins/retriable.rb +3 -1
  51. data/lib/mihari/schemas/analyzer.rb +5 -4
  52. data/lib/mihari/structs/censys.rb +62 -0
  53. data/lib/mihari/structs/greynoise.rb +43 -0
  54. data/lib/mihari/structs/onyphe.rb +45 -0
  55. data/lib/mihari/structs/shodan.rb +83 -0
  56. data/lib/mihari/version.rb +1 -1
  57. data/lib/mihari/web/middleware/connection_adapter.rb +1 -3
  58. data/lib/mihari/web/public/assets/{index-63900d73.js → index-7d0fb8c4.js} +2 -2
  59. data/lib/mihari/web/public/index.html +1 -1
  60. data/lib/mihari/web/public/redoc-static.html +2 -2
  61. data/lib/mihari.rb +21 -2
  62. data/mihari.gemspec +15 -23
  63. metadata +55 -264
  64. data/lib/mihari/analyzers/clients/otx.rb +0 -36
  65. data/lib/mihari/analyzers/dnpedia.rb +0 -37
  66. data/lib/mihari/mixins/database.rb +0 -16
@@ -7,6 +7,18 @@ module Mihari
7
7
  attribute :country_code, Types::String.optional
8
8
  attribute :country_name, Types::String.optional
9
9
 
10
+ #
11
+ # @return [Mihari::Geolocation, nil]
12
+ #
13
+ def to_geolocation
14
+ return nil if country_name.nil? && country_code.nil?
15
+
16
+ Mihari::Geolocation.new(
17
+ country: country_name,
18
+ country_code: country_code
19
+ )
20
+ end
21
+
10
22
  def self.from_dynamic!(d)
11
23
  d = Types::Hash[d]
12
24
  new(
@@ -17,6 +29,8 @@ module Mihari
17
29
  end
18
30
 
19
31
  class Match < Dry::Struct
32
+ include Mixins::AutonomousSystem
33
+
20
34
  attribute :asn, Types::String.optional
21
35
  attribute :hostnames, Types.Array(Types::String)
22
36
  attribute :location, Location
@@ -25,6 +39,15 @@ module Mihari
25
39
  attribute :port, Types::Integer
26
40
  attribute :metadata, Types::Hash
27
41
 
42
+ #
43
+ # @return [Mihari::AutonomousSystem, nil]
44
+ #
45
+ def to_asn
46
+ return nil if asn.nil?
47
+
48
+ Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
49
+ end
50
+
28
51
  def self.from_dynamic!(d)
29
52
  d = Types::Hash[d]
30
53
 
@@ -51,6 +74,66 @@ module Mihari
51
74
  attribute :matches, Types.Array(Match)
52
75
  attribute :total, Types::Int
53
76
 
77
+ #
78
+ # Collect metadata from matches
79
+ #
80
+ # @param [String] ip
81
+ #
82
+ # @return [Array<Hash>]
83
+ #
84
+ def collect_metadata_by_ip(ip)
85
+ matches.select { |match| match.ip_str == ip }.map(&:metadata)
86
+ end
87
+
88
+ #
89
+ # Collect ports from matches
90
+ #
91
+ # @param [String] ip
92
+ #
93
+ # @return [Array<String>]
94
+ #
95
+ def collect_ports_by_ip(ip)
96
+ matches.select { |match| match.ip_str == ip }.map(&:port)
97
+ end
98
+
99
+ #
100
+ # Collect hostnames from matches
101
+ #
102
+ # @param [String] ip
103
+ #
104
+ # @return [Array<String>]
105
+ #
106
+ def collect_hostnames_by_ip(ip)
107
+ matches.select { |match| match.ip_str == ip }.map(&:hostnames).flatten.uniq
108
+ end
109
+
110
+ #
111
+ # @param [Source] source
112
+ #
113
+ # @return [Array<Mihari::Artifact>]
114
+ #
115
+ def to_artifacts(source = "Shodan")
116
+ matches.map do |match|
117
+ metadata = collect_metadata_by_ip(match.ip_str)
118
+ ports = collect_ports_by_ip(match.ip_str).map do |port|
119
+ Mihari::Port.new(port: port)
120
+ end
121
+ reverse_dns_names = collect_hostnames_by_ip(match.ip_str).map do |name|
122
+ Mihari::ReverseDnsName.new(name: name)
123
+ end
124
+
125
+ Mihari::Artifact.new(
126
+ data: match.ip_str,
127
+ source: source,
128
+ metadata: metadata,
129
+ autonomous_system: match.to_asn,
130
+ geolocation: match.location.to_geolocation,
131
+ ports: ports,
132
+ reverse_dns_names: reverse_dns_names
133
+ )
134
+ end
135
+ end
136
+
54
137
  def self.from_dynamic!(d)
55
138
  d = Types::Hash[d]
56
139
  new(
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- VERSION = "5.1.0"
4
+ VERSION = "5.1.2"
5
5
  end
@@ -1,14 +1,12 @@
1
1
  module Mihari
2
2
  module Middleware
3
3
  class ConnectionAdapter
4
- include Mixins::Database
5
-
6
4
  def initialize(app)
7
5
  @app = app
8
6
  end
9
7
 
10
8
  def call(env)
11
- with_db_connection do
9
+ Mihari::Database.with_db_connection do
12
10
  status, headers, body = @app.call(env)
13
11
 
14
12
  [status, headers, body]