mihari 2.4.0 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +7 -0
  3. data/.overcommit.yml +12 -0
  4. data/README.md +1 -9
  5. data/exe/mihari +1 -1
  6. data/lib/mihari.rb +88 -15
  7. data/lib/mihari/analyzers/base.rb +49 -8
  8. data/lib/mihari/analyzers/basic.rb +1 -2
  9. data/lib/mihari/analyzers/binaryedge.rb +7 -13
  10. data/lib/mihari/analyzers/censys.rb +26 -63
  11. data/lib/mihari/analyzers/circl.rb +20 -17
  12. data/lib/mihari/analyzers/crtsh.rb +6 -13
  13. data/lib/mihari/analyzers/dnpedia.rb +6 -12
  14. data/lib/mihari/analyzers/dnstwister.rb +13 -10
  15. data/lib/mihari/analyzers/onyphe.rb +6 -12
  16. data/lib/mihari/analyzers/otx.rb +22 -19
  17. data/lib/mihari/analyzers/passivetotal.rb +22 -21
  18. data/lib/mihari/analyzers/pulsedive.rb +16 -13
  19. data/lib/mihari/analyzers/rule.rb +99 -0
  20. data/lib/mihari/analyzers/securitytrails.rb +22 -19
  21. data/lib/mihari/analyzers/shodan.rb +7 -13
  22. data/lib/mihari/analyzers/spyse.rb +12 -19
  23. data/lib/mihari/analyzers/urlscan.rb +22 -27
  24. data/lib/mihari/analyzers/virustotal.rb +25 -22
  25. data/lib/mihari/analyzers/zoomeye.rb +14 -20
  26. data/lib/mihari/cli/analyzer.rb +44 -0
  27. data/lib/mihari/cli/base.rb +27 -0
  28. data/lib/mihari/cli/init.rb +13 -0
  29. data/lib/mihari/cli/main.rb +30 -0
  30. data/lib/mihari/cli/mixins/utils.rb +88 -0
  31. data/lib/mihari/cli/validator.rb +11 -0
  32. data/lib/mihari/commands/binaryedge.rb +1 -1
  33. data/lib/mihari/commands/censys.rb +1 -1
  34. data/lib/mihari/commands/circl.rb +2 -2
  35. data/lib/mihari/commands/crtsh.rb +1 -1
  36. data/lib/mihari/commands/dnpedia.rb +1 -1
  37. data/lib/mihari/commands/dnstwister.rb +2 -2
  38. data/lib/mihari/commands/init.rb +46 -0
  39. data/lib/mihari/commands/json.rb +1 -1
  40. data/lib/mihari/commands/onyphe.rb +1 -1
  41. data/lib/mihari/commands/otx.rb +2 -2
  42. data/lib/mihari/commands/passivetotal.rb +2 -2
  43. data/lib/mihari/commands/pulsedive.rb +2 -2
  44. data/lib/mihari/commands/search.rb +77 -0
  45. data/lib/mihari/commands/securitytrails.rb +2 -2
  46. data/lib/mihari/commands/shodan.rb +1 -1
  47. data/lib/mihari/commands/spyse.rb +1 -1
  48. data/lib/mihari/commands/urlscan.rb +2 -2
  49. data/lib/mihari/commands/validator.rb +38 -0
  50. data/lib/mihari/commands/virustotal.rb +2 -2
  51. data/lib/mihari/commands/zoomeye.rb +1 -1
  52. data/lib/mihari/constraints.rb +5 -0
  53. data/lib/mihari/database.rb +13 -2
  54. data/lib/mihari/emitters/base.rb +2 -2
  55. data/lib/mihari/emitters/database.rb +1 -1
  56. data/lib/mihari/emitters/misp.rb +1 -1
  57. data/lib/mihari/emitters/slack.rb +5 -6
  58. data/lib/mihari/emitters/the_hive.rb +1 -1
  59. data/lib/mihari/emitters/webhook.rb +2 -9
  60. data/lib/mihari/mixins/configurable.rb +38 -0
  61. data/lib/mihari/mixins/configuration.rb +85 -0
  62. data/lib/mihari/mixins/hash.rb +20 -0
  63. data/lib/mihari/mixins/refang.rb +21 -0
  64. data/lib/mihari/mixins/retriable.rb +27 -0
  65. data/lib/mihari/mixins/rule.rb +79 -0
  66. data/lib/mihari/models/alert.rb +28 -1
  67. data/lib/mihari/models/artifact.rb +10 -0
  68. data/lib/mihari/notifiers/base.rb +9 -1
  69. data/lib/mihari/notifiers/exception_notifier.rb +50 -0
  70. data/lib/mihari/notifiers/slack.rb +29 -0
  71. data/lib/mihari/schemas/configuration.rb +42 -0
  72. data/lib/mihari/schemas/macros.rb +17 -0
  73. data/lib/mihari/schemas/rule.rb +72 -0
  74. data/lib/mihari/serializers/artifact.rb +1 -1
  75. data/lib/mihari/status.rb +14 -0
  76. data/lib/mihari/templates/rule.yml.erb +19 -0
  77. data/lib/mihari/type_checker.rb +8 -3
  78. data/lib/mihari/version.rb +1 -1
  79. data/lib/mihari/web/controllers/base_controller.rb +1 -1
  80. data/lib/mihari/web/public/index.html +1 -21
  81. data/lib/mihari/web/public/redoc-static.html +2 -2
  82. data/lib/mihari/web/public/static/js/app.ab213f7c.js +12 -0
  83. data/lib/mihari/web/public/static/js/app.ab213f7c.js.map +1 -0
  84. data/mihari.gemspec +12 -5
  85. metadata +123 -50
  86. data/.rubocop.yml +0 -161
  87. data/lib/mihari/analyzers/free_text.rb +0 -48
  88. data/lib/mihari/analyzers/http_hash.rb +0 -100
  89. data/lib/mihari/analyzers/passive_dns.rb +0 -59
  90. data/lib/mihari/analyzers/passive_ssl.rb +0 -55
  91. data/lib/mihari/analyzers/reverse_whois.rb +0 -55
  92. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +0 -59
  93. data/lib/mihari/analyzers/ssh_fingerprint.rb +0 -58
  94. data/lib/mihari/cli.rb +0 -126
  95. data/lib/mihari/commands/config.rb +0 -27
  96. data/lib/mihari/commands/free_text.rb +0 -21
  97. data/lib/mihari/commands/http_hash.rb +0 -25
  98. data/lib/mihari/commands/passive_dns.rb +0 -21
  99. data/lib/mihari/commands/passive_ssl.rb +0 -21
  100. data/lib/mihari/commands/reverse_whois.rb +0 -21
  101. data/lib/mihari/commands/securitytrails_domain_feed.rb +0 -23
  102. data/lib/mihari/commands/ssh_fingerprint.rb +0 -21
  103. data/lib/mihari/config.rb +0 -85
  104. data/lib/mihari/configurable.rb +0 -21
  105. data/lib/mihari/html.rb +0 -43
  106. data/lib/mihari/retriable.rb +0 -17
data/.rubocop.yml DELETED
@@ -1,161 +0,0 @@
1
- # Relaxed.Ruby.Style
2
- ## Version 2.5
3
-
4
- require:
5
- - rubocop-performance
6
-
7
- AllCops:
8
- NewCops: enable
9
-
10
- Style/Alias:
11
- Enabled: false
12
- StyleGuide: https://relaxed.ruby.style/#stylealias
13
-
14
- Style/AsciiComments:
15
- Enabled: false
16
- StyleGuide: https://relaxed.ruby.style/#styleasciicomments
17
-
18
- Style/BeginBlock:
19
- Enabled: false
20
- StyleGuide: https://relaxed.ruby.style/#stylebeginblock
21
-
22
- Style/BlockDelimiters:
23
- Enabled: false
24
- StyleGuide: https://relaxed.ruby.style/#styleblockdelimiters
25
-
26
- Style/CommentAnnotation:
27
- Enabled: false
28
- StyleGuide: https://relaxed.ruby.style/#stylecommentannotation
29
-
30
- Style/Documentation:
31
- Enabled: false
32
- StyleGuide: https://relaxed.ruby.style/#styledocumentation
33
-
34
- Layout/DotPosition:
35
- Enabled: false
36
- StyleGuide: https://relaxed.ruby.style/#layoutdotposition
37
-
38
- Style/DoubleNegation:
39
- Enabled: false
40
- StyleGuide: https://relaxed.ruby.style/#styledoublenegation
41
-
42
- Style/EndBlock:
43
- Enabled: false
44
- StyleGuide: https://relaxed.ruby.style/#styleendblock
45
-
46
- Style/FormatString:
47
- Enabled: false
48
- StyleGuide: https://relaxed.ruby.style/#styleformatstring
49
-
50
- Style/IfUnlessModifier:
51
- Enabled: false
52
- StyleGuide: https://relaxed.ruby.style/#styleifunlessmodifier
53
-
54
- Style/Lambda:
55
- Enabled: false
56
- StyleGuide: https://relaxed.ruby.style/#stylelambda
57
-
58
- Style/ModuleFunction:
59
- Enabled: false
60
- StyleGuide: https://relaxed.ruby.style/#stylemodulefunction
61
-
62
- Style/MultilineBlockChain:
63
- Enabled: false
64
- StyleGuide: https://relaxed.ruby.style/#stylemultilineblockchain
65
-
66
- Style/NegatedIf:
67
- Enabled: false
68
- StyleGuide: https://relaxed.ruby.style/#stylenegatedif
69
-
70
- Style/NegatedWhile:
71
- Enabled: false
72
- StyleGuide: https://relaxed.ruby.style/#stylenegatedwhile
73
-
74
- Style/NumericPredicate:
75
- Enabled: false
76
- StyleGuide: https://relaxed.ruby.style/#stylenumericpredicate
77
-
78
- Style/ParallelAssignment:
79
- Enabled: false
80
- StyleGuide: https://relaxed.ruby.style/#styleparallelassignment
81
-
82
- Style/PercentLiteralDelimiters:
83
- Enabled: false
84
- StyleGuide: https://relaxed.ruby.style/#stylepercentliteraldelimiters
85
-
86
- Style/PerlBackrefs:
87
- Enabled: false
88
- StyleGuide: https://relaxed.ruby.style/#styleperlbackrefs
89
-
90
- Style/Semicolon:
91
- Enabled: false
92
- StyleGuide: https://relaxed.ruby.style/#stylesemicolon
93
-
94
- Style/SignalException:
95
- Enabled: false
96
- StyleGuide: https://relaxed.ruby.style/#stylesignalexception
97
-
98
- Style/SingleLineBlockParams:
99
- Enabled: false
100
- StyleGuide: https://relaxed.ruby.style/#stylesinglelineblockparams
101
-
102
- Style/SingleLineMethods:
103
- Enabled: false
104
- StyleGuide: https://relaxed.ruby.style/#stylesinglelinemethods
105
-
106
- Layout/SpaceBeforeBlockBraces:
107
- Enabled: false
108
- StyleGuide: https://relaxed.ruby.style/#layoutspacebeforeblockbraces
109
-
110
- Layout/SpaceInsideParens:
111
- Enabled: false
112
- StyleGuide: https://relaxed.ruby.style/#layoutspaceinsideparens
113
-
114
- Style/SpecialGlobalVars:
115
- Enabled: false
116
- StyleGuide: https://relaxed.ruby.style/#stylespecialglobalvars
117
-
118
- Style/StringLiterals:
119
- Enabled: false
120
- StyleGuide: https://relaxed.ruby.style/#stylestringliterals
121
-
122
- Style/TrailingCommaInArguments:
123
- Enabled: false
124
- StyleGuide: https://relaxed.ruby.style/#styletrailingcommainarguments
125
-
126
- Style/TrailingCommaInArrayLiteral:
127
- Enabled: false
128
- StyleGuide: https://relaxed.ruby.style/#styletrailingcommainarrayliteral
129
-
130
- Style/TrailingCommaInHashLiteral:
131
- Enabled: false
132
- StyleGuide: https://relaxed.ruby.style/#styletrailingcommainhashliteral
133
-
134
- Style/SymbolArray:
135
- Enabled: false
136
- StyleGuide: http://relaxed.ruby.style/#stylesymbolarray
137
-
138
- Style/WhileUntilModifier:
139
- Enabled: false
140
- StyleGuide: https://relaxed.ruby.style/#stylewhileuntilmodifier
141
-
142
- Style/WordArray:
143
- Enabled: false
144
- StyleGuide: https://relaxed.ruby.style/#stylewordarray
145
-
146
- Lint/AmbiguousRegexpLiteral:
147
- Enabled: false
148
- StyleGuide: https://relaxed.ruby.style/#lintambiguousregexpliteral
149
-
150
- Lint/AssignmentInCondition:
151
- Enabled: false
152
- StyleGuide: https://relaxed.ruby.style/#lintassignmentincondition
153
-
154
- Layout/LineLength:
155
- Enabled: false
156
-
157
- Style/StringLiteralsInInterpolation:
158
- Enabled: false
159
-
160
- Metrics:
161
- Enabled: false
@@ -1,48 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "parallel"
4
-
5
- module Mihari
6
- module Analyzers
7
- class FreeText < Base
8
- attr_reader :query, :title, :description, :tags
9
-
10
- ANALYZERS = [
11
- Mihari::Analyzers::BinaryEdge,
12
- Mihari::Analyzers::Censys
13
- ].freeze
14
-
15
- def initialize(query, title: nil, description: nil, tags: [])
16
- super()
17
-
18
- @query = query
19
-
20
- @title = title || "Free text cross search"
21
- @description = description || "query = #{query}"
22
- @tags = tags
23
- end
24
-
25
- def artifacts
26
- Parallel.map(analyzers) do |analyzer|
27
- run_analyzer analyzer
28
- end.flatten
29
- end
30
-
31
- private
32
-
33
- def analyzers
34
- ANALYZERS.map do |klass|
35
- klass.new(query)
36
- end
37
- end
38
-
39
- def run_analyzer(analyzer)
40
- analyzer.artifacts
41
- rescue ArgumentError, InvalidInputError => _e
42
- nil
43
- rescue ::BinaryEdge::Error, ::Censys::Error => _e
44
- nil
45
- end
46
- end
47
- end
48
- end
@@ -1,100 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "parallel"
4
-
5
- module Mihari
6
- module Analyzers
7
- class HTTPHash < Base
8
- attr_reader :md5, :sha256, :mmh3, :html, :title, :description, :tags
9
-
10
- def initialize(_query, md5: nil, sha256: nil, mmh3: nil, html: nil, title: nil, description: nil, tags: [])
11
- super()
12
-
13
- @md5 = md5
14
- @sha256 = sha256
15
- @mmh3 = mmh3
16
-
17
- @html = html
18
- load_from_html
19
-
20
- @title = title || "HTTP hash cross search"
21
- @description = description || "query = #{query}"
22
- @tags = tags
23
- end
24
-
25
- def artifacts
26
- Parallel.map(analyzers) do |analyzer|
27
- run_analyzer analyzer
28
- end.flatten
29
- end
30
-
31
- private
32
-
33
- def valid_query?
34
- [md5, sha256, mmh3].compact.any?
35
- end
36
-
37
- def load_from_html
38
- return unless html
39
-
40
- html_file = HTML.new(html)
41
- return unless html_file.exists?
42
-
43
- @md5 = html_file.md5
44
- @sha256 = html_file.sha256
45
- @mmh3 = html_file.mmh3
46
- end
47
-
48
- def query
49
- [
50
- md5 ? "md5:#{md5}" : nil,
51
- sha256 ? "sha256:#{sha256}" : nil,
52
- mmh3 ? "mmh3:#{mmh3}" : nil
53
- ].compact.join(",")
54
- end
55
-
56
- def binary_edge
57
- return nil unless sha256
58
-
59
- BinaryEdge.new sha256
60
- end
61
-
62
- def censys
63
- return nil unless sha256
64
-
65
- Censys.new sha256
66
- end
67
-
68
- def onyphe
69
- return nil unless md5
70
-
71
- Onyphe.new "app.http.bodymd5:#{md5}"
72
- end
73
-
74
- def shodan
75
- return nil unless mmh3
76
-
77
- Shodan.new "http.html_hash:#{mmh3}"
78
- end
79
-
80
- def analyzers
81
- raise InvalidInputError, "No hashes are given." unless valid_query?
82
-
83
- [
84
- binary_edge,
85
- censys,
86
- onyphe,
87
- shodan
88
- ].compact
89
- end
90
-
91
- def run_analyzer(analyzer)
92
- analyzer.artifacts
93
- rescue ArgumentError, InvalidInputError => _e
94
- nil
95
- rescue ::BinaryEdge::Error, ::Censys::ResponseError, ::Onyphe::Error, ::Shodan::Error => _e
96
- nil
97
- end
98
- end
99
- end
100
- end
@@ -1,59 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "parallel"
4
-
5
- module Mihari
6
- module Analyzers
7
- class PassiveDNS < Base
8
- attr_reader :query, :type, :title, :description, :tags
9
-
10
- ANALYZERS = [
11
- Mihari::Analyzers::CIRCL,
12
- Mihari::Analyzers::OTX,
13
- Mihari::Analyzers::PassiveTotal,
14
- Mihari::Analyzers::Pulsedive,
15
- Mihari::Analyzers::SecurityTrails,
16
- Mihari::Analyzers::VirusTotal
17
- ].freeze
18
-
19
- def initialize(query, title: nil, description: nil, tags: [])
20
- super()
21
-
22
- @query = query
23
- @type = TypeChecker.type(query)
24
-
25
- @title = title || "PassiveDNS cross search"
26
- @description = description || "query = #{query}"
27
- @tags = tags
28
- end
29
-
30
- def artifacts
31
- Parallel.map(analyzers) do |analyzer|
32
- run_analyzer analyzer
33
- end.flatten
34
- end
35
-
36
- private
37
-
38
- def valid_type?
39
- %w[ip domain].include? type
40
- end
41
-
42
- def analyzers
43
- raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
44
-
45
- ANALYZERS.map do |klass|
46
- klass.new(query)
47
- end
48
- end
49
-
50
- def run_analyzer(analyzer)
51
- analyzer.artifacts
52
- rescue ArgumentError, InvalidInputError => _e
53
- nil
54
- rescue Faraday::Error, ::PassiveCIRCL::Error, ::PassiveTotal::Error, ::Pulsedive::ResponseError, ::SecurityTrails::Error, ::VirusTotal::Error => _e
55
- nil
56
- end
57
- end
58
- end
59
- end
@@ -1,55 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "parallel"
4
-
5
- module Mihari
6
- module Analyzers
7
- class PassiveSSL < Base
8
- attr_reader :query, :type, :title, :description, :tags
9
-
10
- ANALYZERS = [
11
- Mihari::Analyzers::CIRCL,
12
- Mihari::Analyzers::PassiveTotal
13
- ].freeze
14
-
15
- def initialize(query, title: nil, description: nil, tags: [])
16
- super()
17
-
18
- @query = query
19
- @type = TypeChecker.detailed_type(query)
20
-
21
- @title = title || "PassiveSSL cross search"
22
- @description = description || "query = #{query}"
23
- @tags = tags
24
- end
25
-
26
- def artifacts
27
- Parallel.map(analyzers) do |analyzer|
28
- run_analyzer analyzer
29
- end.flatten
30
- end
31
-
32
- private
33
-
34
- def valid_type?
35
- %w[sha1].include? type
36
- end
37
-
38
- def analyzers
39
- raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
40
-
41
- ANALYZERS.map do |klass|
42
- klass.new(query)
43
- end
44
- end
45
-
46
- def run_analyzer(analyzer)
47
- analyzer.artifacts
48
- rescue ArgumentError, InvalidInputError => _e
49
- nil
50
- rescue ::PassiveCIRCL::Error, ::PassiveTotal::Error => _e
51
- nil
52
- end
53
- end
54
- end
55
- end
@@ -1,55 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "parallel"
4
-
5
- module Mihari
6
- module Analyzers
7
- class ReveseWhois < Base
8
- attr_reader :query, :type, :title, :description, :tags
9
-
10
- ANALYZERS = [
11
- Mihari::Analyzers::PassiveTotal,
12
- Mihari::Analyzers::SecurityTrails
13
- ].freeze
14
-
15
- def initialize(query, title: nil, description: nil, tags: [])
16
- super()
17
-
18
- @query = query
19
- @type = TypeChecker.type(query)
20
-
21
- @title = title || "ReveseWhois cross search"
22
- @description = description || "query = #{query}"
23
- @tags = tags
24
- end
25
-
26
- def artifacts
27
- Parallel.map(analyzers) do |analyzer|
28
- run_analyzer analyzer
29
- end.flatten
30
- end
31
-
32
- private
33
-
34
- def valid_type?
35
- %w[mail].include? type
36
- end
37
-
38
- def analyzers
39
- raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
40
-
41
- ANALYZERS.map do |klass|
42
- klass.new(query)
43
- end
44
- end
45
-
46
- def run_analyzer(analyzer)
47
- analyzer.artifacts
48
- rescue ArgumentError, InvalidInputError => _e
49
- nil
50
- rescue ::PassiveTotal::Error, ::SecurityTrails::Error => _e
51
- nil
52
- end
53
- end
54
- end
55
- end