mihari 5.6.1 → 5.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (153) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +5 -1
  3. data/README.md +1 -0
  4. data/config.ru +1 -1
  5. data/docs/analyzers/fofa.md +31 -0
  6. data/docs/analyzers/index.md +1 -0
  7. data/frontend/package-lock.json +183 -186
  8. data/frontend/package.json +10 -10
  9. data/frontend/src/components/alert/Form.vue +1 -14
  10. data/frontend/src/components/artifact/AS.vue +2 -8
  11. data/frontend/src/components/artifact/DnsRecords.vue +2 -8
  12. data/frontend/src/components/artifact/ReverseDnsNames.vue +2 -10
  13. data/frontend/src/components/artifact/WhoisRecord.vue +1 -1
  14. data/lib/mihari/{base.rb → actor.rb} +27 -3
  15. data/lib/mihari/analyzers/base.rb +16 -20
  16. data/lib/mihari/analyzers/binaryedge.rb +4 -1
  17. data/lib/mihari/analyzers/censys.rb +5 -3
  18. data/lib/mihari/analyzers/circl.rb +4 -1
  19. data/lib/mihari/analyzers/crtsh.rb +4 -1
  20. data/lib/mihari/analyzers/dnstwister.rb +4 -1
  21. data/lib/mihari/analyzers/feed.rb +3 -0
  22. data/lib/mihari/analyzers/fofa.rb +65 -0
  23. data/lib/mihari/analyzers/greynoise.rb +4 -1
  24. data/lib/mihari/analyzers/hunterhow.rb +7 -2
  25. data/lib/mihari/analyzers/onyphe.rb +4 -1
  26. data/lib/mihari/analyzers/otx.rb +4 -1
  27. data/lib/mihari/analyzers/passivetotal.rb +5 -2
  28. data/lib/mihari/analyzers/pulsedive.rb +4 -1
  29. data/lib/mihari/analyzers/securitytrails.rb +5 -2
  30. data/lib/mihari/analyzers/shodan.rb +4 -1
  31. data/lib/mihari/analyzers/urlscan.rb +5 -2
  32. data/lib/mihari/analyzers/virustotal.rb +9 -6
  33. data/lib/mihari/analyzers/virustotal_intelligence.rb +4 -1
  34. data/lib/mihari/analyzers/zoomeye.rb +8 -5
  35. data/lib/mihari/cli/alert.rb +3 -0
  36. data/lib/mihari/cli/base.rb +3 -0
  37. data/lib/mihari/cli/database.rb +3 -0
  38. data/lib/mihari/cli/main.rb +3 -0
  39. data/lib/mihari/cli/rule.rb +3 -0
  40. data/lib/mihari/clients/base.rb +3 -0
  41. data/lib/mihari/clients/binaryedge.rb +5 -2
  42. data/lib/mihari/clients/censys.rb +7 -4
  43. data/lib/mihari/clients/circl.rb +3 -0
  44. data/lib/mihari/clients/crtsh.rb +5 -2
  45. data/lib/mihari/clients/dnstwister.rb +3 -0
  46. data/lib/mihari/clients/fofa.rb +83 -0
  47. data/lib/mihari/clients/greynoise.rb +5 -2
  48. data/lib/mihari/clients/hunterhow.rb +5 -2
  49. data/lib/mihari/clients/misp.rb +3 -0
  50. data/lib/mihari/clients/onyphe.rb +5 -2
  51. data/lib/mihari/clients/otx.rb +3 -0
  52. data/lib/mihari/clients/passivetotal.rb +7 -4
  53. data/lib/mihari/clients/publsedive.rb +4 -1
  54. data/lib/mihari/clients/securitytrails.rb +6 -3
  55. data/lib/mihari/clients/shodan.rb +5 -2
  56. data/lib/mihari/clients/the_hive.rb +3 -0
  57. data/lib/mihari/clients/urlscan.rb +7 -4
  58. data/lib/mihari/clients/virustotal.rb +5 -2
  59. data/lib/mihari/clients/zoomeye.rb +3 -0
  60. data/lib/mihari/commands/alert.rb +5 -14
  61. data/lib/mihari/commands/database.rb +3 -0
  62. data/lib/mihari/commands/rule.rb +11 -11
  63. data/lib/mihari/commands/search.rb +9 -6
  64. data/lib/mihari/commands/version.rb +3 -0
  65. data/lib/mihari/commands/web.rb +4 -1
  66. data/lib/mihari/config.rb +139 -150
  67. data/lib/mihari/constants.rb +1 -1
  68. data/lib/mihari/database.rb +6 -0
  69. data/lib/mihari/emitters/base.rb +16 -25
  70. data/lib/mihari/emitters/database.rb +10 -9
  71. data/lib/mihari/emitters/misp.rb +20 -41
  72. data/lib/mihari/emitters/slack.rb +16 -13
  73. data/lib/mihari/emitters/the_hive.rb +18 -46
  74. data/lib/mihari/emitters/webhook.rb +34 -23
  75. data/lib/mihari/enrichers/base.rb +16 -15
  76. data/lib/mihari/enrichers/google_public_dns.rb +6 -5
  77. data/lib/mihari/enrichers/ipinfo.rb +10 -8
  78. data/lib/mihari/enrichers/shodan.rb +4 -6
  79. data/lib/mihari/enrichers/whois.rb +13 -10
  80. data/lib/mihari/errors.rb +6 -0
  81. data/lib/mihari/feed/parser.rb +3 -0
  82. data/lib/mihari/feed/reader.rb +3 -0
  83. data/lib/mihari/http.rb +6 -0
  84. data/lib/mihari/mixins/autonomous_system.rb +3 -0
  85. data/lib/mihari/mixins/configurable.rb +3 -0
  86. data/lib/mihari/mixins/error_notification.rb +3 -0
  87. data/lib/mihari/mixins/falsepositive.rb +3 -0
  88. data/lib/mihari/mixins/refang.rb +3 -0
  89. data/lib/mihari/mixins/retriable.rb +6 -2
  90. data/lib/mihari/models/alert.rb +78 -73
  91. data/lib/mihari/models/artifact.rb +186 -178
  92. data/lib/mihari/models/autonomous_system.rb +25 -20
  93. data/lib/mihari/models/cpe.rb +24 -19
  94. data/lib/mihari/models/dns.rb +27 -22
  95. data/lib/mihari/models/geolocation.rb +25 -20
  96. data/lib/mihari/models/port.rb +24 -19
  97. data/lib/mihari/models/reverse_dns.rb +24 -19
  98. data/lib/mihari/models/rule.rb +71 -66
  99. data/lib/mihari/models/tag.rb +8 -3
  100. data/lib/mihari/models/tagging.rb +8 -3
  101. data/lib/mihari/models/whois.rb +20 -17
  102. data/lib/mihari/rule.rb +357 -0
  103. data/lib/mihari/schemas/alert.rb +3 -0
  104. data/lib/mihari/schemas/analyzer.rb +105 -87
  105. data/lib/mihari/schemas/emitter.rb +12 -5
  106. data/lib/mihari/schemas/enricher.rb +11 -4
  107. data/lib/mihari/schemas/macros.rb +4 -0
  108. data/lib/mihari/schemas/mixins.rb +20 -0
  109. data/lib/mihari/schemas/rule.rb +6 -10
  110. data/lib/mihari/service.rb +16 -0
  111. data/lib/mihari/services/alert_builder.rb +8 -5
  112. data/lib/mihari/services/alert_proxy.rb +16 -7
  113. data/lib/mihari/services/alert_runner.rb +10 -14
  114. data/lib/mihari/services/rule_builder.rb +10 -7
  115. data/lib/mihari/services/rule_runner.rb +11 -13
  116. data/lib/mihari/structs/binaryedge.rb +14 -29
  117. data/lib/mihari/structs/censys.rb +54 -133
  118. data/lib/mihari/structs/config.rb +20 -31
  119. data/lib/mihari/structs/filters.rb +38 -0
  120. data/lib/mihari/structs/fofa.rb +44 -0
  121. data/lib/mihari/structs/google_public_dns.rb +10 -28
  122. data/lib/mihari/structs/greynoise.rb +38 -89
  123. data/lib/mihari/structs/hunterhow.rb +27 -25
  124. data/lib/mihari/structs/ipinfo.rb +14 -35
  125. data/lib/mihari/structs/onyphe.rb +36 -81
  126. data/lib/mihari/structs/shodan.rb +53 -118
  127. data/lib/mihari/structs/urlscan.rb +27 -66
  128. data/lib/mihari/structs/virustotal_intelligence.rb +23 -59
  129. data/lib/mihari/type_checker.rb +4 -0
  130. data/lib/mihari/types.rb +3 -0
  131. data/lib/mihari/version.rb +1 -1
  132. data/lib/mihari/web/api.rb +15 -10
  133. data/lib/mihari/web/app.rb +59 -54
  134. data/lib/mihari/web/endpoints/alerts.rb +94 -89
  135. data/lib/mihari/web/endpoints/artifacts.rb +115 -110
  136. data/lib/mihari/web/endpoints/configs.rb +18 -13
  137. data/lib/mihari/web/endpoints/ip_addresses.rb +21 -16
  138. data/lib/mihari/web/endpoints/rules.rb +202 -204
  139. data/lib/mihari/web/endpoints/tags.rb +41 -36
  140. data/lib/mihari/web/middleware/connection_adapter.rb +16 -9
  141. data/lib/mihari/web/middleware/error_notification_adapter.rb +17 -10
  142. data/lib/mihari/web/public/assets/{index-9cc489e6.js → index-821134e2.js} +54 -54
  143. data/lib/mihari/web/public/assets/mode-yaml-24faa242.js +8 -0
  144. data/lib/mihari/web/public/index.html +1 -1
  145. data/lib/mihari.rb +30 -13
  146. data/mihari.gemspec +9 -3
  147. data/mkdocs.yml +3 -2
  148. data/requirements.txt +1 -1
  149. metadata +44 -26
  150. data/lib/mihari/analyzers/rule.rb +0 -232
  151. data/lib/mihari/services/rule_proxy.rb +0 -182
  152. data/lib/mihari/templates/rule.yml.erb +0 -5
  153. data/lib/mihari/web/public/assets/mode-yaml-a21faa53.js +0 -8
@@ -1,182 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "json"
4
-
5
- module Mihari
6
- module Services
7
- #
8
- # proxy (or converter) class for rule
9
- # proxying rule schema data into analyzer & model
10
- #
11
- class RuleProxy
12
- include Mixins::FalsePositive
13
-
14
- # @return [Hash]
15
- attr_reader :data
16
-
17
- # @return [Array, nil]
18
- attr_reader :errors
19
-
20
- #
21
- # Initialize
22
- #
23
- # @param [Hash] data
24
- #
25
- def initialize(data)
26
- @data = data.deep_symbolize_keys
27
- @errors = nil
28
-
29
- validate!
30
- end
31
-
32
- #
33
- # @return [Boolean]
34
- #
35
- def errors?
36
- return false if @errors.nil?
37
-
38
- !@errors.empty?
39
- end
40
-
41
- def validate!
42
- contract = Schemas::RuleContract.new
43
- result = contract.call(data)
44
-
45
- @data = result.to_h
46
- @errors = result.errors
47
-
48
- raise ValidationError.new("Validation failed", errors) if errors?
49
- end
50
-
51
- def [](key)
52
- data key.to_sym
53
- end
54
-
55
- #
56
- # @return [String]
57
- #
58
- def id
59
- @id ||= data[:id]
60
- end
61
-
62
- #
63
- # @return [String]
64
- #
65
- def title
66
- @title ||= data[:title]
67
- end
68
-
69
- #
70
- # @return [String]
71
- #
72
- def description
73
- @description ||= data[:description]
74
- end
75
-
76
- #
77
- # @return [String]
78
- #
79
- def yaml
80
- @yaml ||= data.deep_stringify_keys.to_yaml
81
- end
82
-
83
- #
84
- # @return [Array<Hash>]
85
- #
86
- def queries
87
- @queries ||= data[:queries]
88
- end
89
-
90
- #
91
- # @return [Array<String>]
92
- #
93
- def data_types
94
- @data_types ||= data[:data_types]
95
- end
96
-
97
- #
98
- # @return [Array<String>]
99
- #
100
- def tags
101
- @tags ||= data[:tags]
102
- end
103
-
104
- #
105
- # @return [Array<String, RegExp>]
106
- #
107
- def falsepositives
108
- @falsepositives ||= data[:falsepositives].map { |fp| normalize_falsepositive fp }
109
- end
110
-
111
- #
112
- # @return [Array<Hash>]
113
- #
114
- def emitters
115
- @emitters ||= data[:emitters]
116
- end
117
-
118
- #
119
- # @return [Array<Hash>]
120
- #
121
- def enrichers
122
- @enrichers ||= data[:enrichers]
123
- end
124
-
125
- #
126
- # @return [Integer, nil]
127
- #
128
- def artifact_lifetime
129
- @artifact_lifetime ||= data[:artifact_lifetime] || data[:artifact_ttl]
130
- end
131
-
132
- #
133
- # @return [Mihari::Rule]
134
- #
135
- def model
136
- rule = Mihari::Rule.find(id)
137
-
138
- rule.title = title
139
- rule.description = description
140
- rule.data = data
141
-
142
- rule
143
- rescue ActiveRecord::RecordNotFound
144
- Mihari::Rule.new(
145
- id: id,
146
- title: title,
147
- description: description,
148
- data: data
149
- )
150
- end
151
-
152
- #
153
- # @return [Mihari::Analyzers::Rule]
154
- #
155
- def analyzer
156
- Mihari::Analyzers::Rule.new self
157
- end
158
-
159
- class << self
160
- #
161
- # Load rule from YAML string
162
- #
163
- # @param [String] yaml
164
- #
165
- # @return [Mihari::Services::Rule]
166
- #
167
- def from_yaml(yaml)
168
- new YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
169
- end
170
-
171
- #
172
- # @param [Mihari::Rule] model
173
- #
174
- # @return [Mihari::Services::Rule]
175
- #
176
- def from_model(model)
177
- new model.data
178
- end
179
- end
180
- end
181
- end
182
- end
@@ -1,5 +0,0 @@
1
- id: <%= SecureRandom.uuid %>
2
- title: Title goes here
3
- description: Description goes here
4
- created_on: <%= Date.today %>
5
- queries: []
@@ -1,8 +0,0 @@
1
- ace.define("ace/mode/yaml_highlight_rules",["require","exports","module","ace/lib/oop","ace/mode/text_highlight_rules"],function(e,t,n){"use strict";var r=e("../lib/oop"),i=e("./text_highlight_rules").TextHighlightRules,s=function(){this.$rules={start:[{token:"comment",regex:"#.*$"},{token:"list.markup",regex:/^(?:-{3}|\.{3})\s*(?=#|$)/},{token:"list.markup",regex:/^\s*[\-?](?:$|\s)/},{token:"constant",regex:"!![\\w//]+"},{token:"constant.language",regex:"[&\\*][a-zA-Z0-9-_]+"},{token:["meta.tag","keyword"],regex:/^(\s*\w[^\s:]*?)(:(?=\s|$))/},{token:["meta.tag","keyword"],regex:/(\w[^\s:]*?)(\s*:(?=\s|$))/},{token:"keyword.operator",regex:"<<\\w*:\\w*"},{token:"keyword.operator",regex:"-\\s*(?=[{])"},{token:"string",regex:'["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'},{token:"string",regex:/[|>][-+\d]*(?:$|\s+(?:$|#))/,onMatch:function(e,t,n,r){r=r.replace(/ #.*/,"");var i=/^ *((:\s*)?-(\s*[^|>])?)?/.exec(r)[0].replace(/\S\s*$/,"").length,s=parseInt(/\d+[\s+-]*$/.exec(r));return s?(i+=s-1,this.next="mlString"):this.next="mlStringPre",n.length?(n[0]=this.next,n[1]=i):(n.push(this.next),n.push(i)),this.token},next:"mlString"},{token:"string",regex:"['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"},{token:"constant.numeric",regex:/(\b|[+\-\.])[\d_]+(?:(?:\.[\d_]*)?(?:[eE][+\-]?[\d_]+)?)(?=[^\d-\w]|$)$/},{token:"constant.numeric",regex:/[+\-]?\.inf\b|NaN\b|0x[\dA-Fa-f_]+|0b[10_]+/},{token:"constant.language.boolean",regex:"\\b(?:true|false|TRUE|FALSE|True|False|yes|no)\\b"},{token:"paren.lparen",regex:"[[({]"},{token:"paren.rparen",regex:"[\\])}]"},{token:"text",regex:/[^\s,:\[\]\{\}]+/}],mlStringPre:[{token:"indent",regex:/^ *$/},{token:"indent",regex:/^ */,onMatch:function(e,t,n){var r=n[1];return r>=e.length?(this.next="start",n.shift(),n.shift()):(n[1]=e.length-1,this.next=n[0]="mlString"),this.token},next:"mlString"},{defaultToken:"string"}],mlString:[{token:"indent",regex:/^ *$/},{token:"indent",regex:/^ */,onMatch:function(e,t,n){var r=n[1];return r>=e.length?(this.next="start",n.splice(0)):this.next="mlString",this.token},next:"mlString"},{token:"string",regex:".+"}]},this.normalizeRules()};r.inherits(s,i),t.YamlHighlightRules=s}),ace.define("ace/mode/matching_brace_outdent",["require","exports","module","ace/range"],function(e,t,n){"use strict";var r=e("../range").Range,i=function(){};(function(){this.checkOutdent=function(e,t){return/^\s+$/.test(e)?/^\s*\}/.test(t):!1},this.autoOutdent=function(e,t){var n=e.getLine(t),i=n.match(/^(\s*\})/);if(!i)return 0;var s=i[1].length,o=e.findMatchingBracket({row:t,column:s});if(!o||o.row==t)return 0;var u=this.$getIndent(e.getLine(o.row));e.replace(new r(t,0,t,s-1),u)},this.$getIndent=function(e){return e.match(/^\s*/)[0]}}).call(i.prototype),t.MatchingBraceOutdent=i}),ace.define("ace/mode/folding/coffee",["require","exports","module","ace/lib/oop","ace/mode/folding/fold_mode","ace/range"],function(e,t,n){"use strict";var r=e("../../lib/oop"),i=e("./fold_mode").FoldMode,s=e("../../range").Range,o=t.FoldMode=function(){};r.inherits(o,i),function(){this.getFoldWidgetRange=function(e,t,n){var r=this.indentationBlock(e,n);if(r)return r;var i=/\S/,o=e.getLine(n),u=o.search(i);if(u==-1||o[u]!="#")return;var a=o.length,f=e.getLength(),l=n,c=n;while(++n<f){o=e.getLine(n);var h=o.search(i);if(h==-1)continue;if(o[h]!="#")break;c=n}if(c>l){var p=e.getLine(c).length;return new s(l,a,c,p)}},this.getFoldWidget=function(e,t,n){var r=e.getLine(n),i=r.search(/\S/),s=e.getLine(n+1),o=e.getLine(n-1),u=o.search(/\S/),a=s.search(/\S/);if(i==-1)return e.foldWidgets[n-1]=u!=-1&&u<a?"start":"","";if(u==-1){if(i==a&&r[i]=="#"&&s[i]=="#")return e.foldWidgets[n-1]="",e.foldWidgets[n+1]="","start"}else if(u==i&&r[i]=="#"&&o[i]=="#"&&e.getLine(n-2).search(/\S/)==-1)return e.foldWidgets[n-1]="start",e.foldWidgets[n+1]="","";return u!=-1&&u<i?e.foldWidgets[n-1]="start":e.foldWidgets[n-1]="",i<a?"start":""}}.call(o.prototype)}),ace.define("ace/mode/yaml",["require","exports","module","ace/lib/oop","ace/mode/text","ace/mode/yaml_highlight_rules","ace/mode/matching_brace_outdent","ace/mode/folding/coffee","ace/worker/worker_client"],function(e,t,n){"use strict";var r=e("../lib/oop"),i=e("./text").Mode,s=e("./yaml_highlight_rules").YamlHighlightRules,o=e("./matching_brace_outdent").MatchingBraceOutdent,u=e("./folding/coffee").FoldMode,a=e("../worker/worker_client").WorkerClient,f=function(){this.HighlightRules=s,this.$outdent=new o,this.foldingRules=new u,this.$behaviour=this.$defaultBehaviour};r.inherits(f,i),function(){this.lineCommentStart=["#"],this.getNextLineIndent=function(e,t,n){var r=this.$getIndent(t);if(e=="start"){var i=t.match(/^.*[\{\(\[]\s*$/);i&&(r+=n)}return r},this.checkOutdent=function(e,t,n){return this.$outdent.checkOutdent(t,n)},this.autoOutdent=function(e,t,n){this.$outdent.autoOutdent(t,n)},this.createWorker=function(e){var t=new a(["ace"],"ace/mode/yaml_worker","YamlWorker");return t.attachToDocument(e.getDocument()),t.on("annotate",function(t){e.setAnnotations(t.data)}),t.on("terminate",function(){e.clearAnnotations()}),t},this.$id="ace/mode/yaml"}.call(f.prototype),t.Mode=f}); (function() {
2
- ace.require(["ace/mode/yaml"], function(m) {
3
- if (typeof module == "object" && typeof exports == "object" && module) {
4
- module.exports = m;
5
- }
6
- });
7
- })();
8
-