mihari 0.17.5 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +3 -0
  3. data/.rubocop.yml +155 -0
  4. data/.travis.yml +1 -0
  5. data/Gemfile +2 -0
  6. data/README.md +30 -72
  7. data/config/pre_commit.yml +3 -0
  8. data/lib/mihari.rb +12 -8
  9. data/lib/mihari/alert_viewer.rb +6 -28
  10. data/lib/mihari/analyzers/base.rb +7 -19
  11. data/lib/mihari/analyzers/basic.rb +3 -1
  12. data/lib/mihari/analyzers/binaryedge.rb +2 -2
  13. data/lib/mihari/analyzers/censys.rb +2 -2
  14. data/lib/mihari/analyzers/circl.rb +2 -2
  15. data/lib/mihari/analyzers/onyphe.rb +3 -3
  16. data/lib/mihari/analyzers/passivetotal.rb +2 -2
  17. data/lib/mihari/analyzers/pulsedive.rb +2 -2
  18. data/lib/mihari/analyzers/securitytrails.rb +2 -2
  19. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +2 -2
  20. data/lib/mihari/analyzers/shodan.rb +2 -2
  21. data/lib/mihari/analyzers/virustotal.rb +2 -2
  22. data/lib/mihari/analyzers/zoomeye.rb +2 -2
  23. data/lib/mihari/cli.rb +2 -2
  24. data/lib/mihari/config.rb +68 -2
  25. data/lib/mihari/configurable.rb +1 -1
  26. data/lib/mihari/database.rb +45 -0
  27. data/lib/mihari/emitters/base.rb +1 -1
  28. data/lib/mihari/emitters/misp.rb +8 -1
  29. data/lib/mihari/emitters/slack.rb +2 -2
  30. data/lib/mihari/emitters/sqlite.rb +29 -0
  31. data/lib/mihari/emitters/stdout.rb +2 -1
  32. data/lib/mihari/emitters/the_hive.rb +28 -14
  33. data/lib/mihari/models/alert.rb +11 -0
  34. data/lib/mihari/models/artifact.rb +27 -0
  35. data/lib/mihari/models/tag.rb +10 -0
  36. data/lib/mihari/models/tagging.rb +10 -0
  37. data/lib/mihari/notifiers/slack.rb +4 -4
  38. data/lib/mihari/serializers/alert.rb +12 -0
  39. data/lib/mihari/serializers/artifact.rb +9 -0
  40. data/lib/mihari/serializers/tag.rb +9 -0
  41. data/lib/mihari/status.rb +1 -1
  42. data/lib/mihari/type_checker.rb +1 -1
  43. data/lib/mihari/version.rb +1 -1
  44. data/mihari.gemspec +11 -5
  45. metadata +120 -31
  46. data/lib/mihari/artifact.rb +0 -36
  47. data/lib/mihari/cache.rb +0 -35
  48. data/lib/mihari/the_hive.rb +0 -42
  49. data/lib/mihari/the_hive/alert.rb +0 -25
  50. data/lib/mihari/the_hive/artifact.rb +0 -33
  51. data/lib/mihari/the_hive/base.rb +0 -14
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ module Mihari
6
+ class Alert < ActiveRecord::Base
7
+ has_many :taggings, dependent: :destroy
8
+ has_many :artifacts, dependent: :destroy
9
+ has_many :tags, through: :taggings
10
+ end
11
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ class ArtifactValidator < ActiveModel::Validator
6
+ def validate(record)
7
+ return if record.data_type
8
+
9
+ record.errors[:data] << "#{record.data} is not supported"
10
+ end
11
+ end
12
+
13
+ module Mihari
14
+ class Artifact < ActiveRecord::Base
15
+ include ActiveModel::Validations
16
+ validates_with ArtifactValidator
17
+
18
+ def initialize(attributes)
19
+ super
20
+ self.data_type = TypeChecker.type(data)
21
+ end
22
+
23
+ def unique?
24
+ self.class.find_by(data: data).nil?
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ module Mihari
6
+ class Tag < ActiveRecord::Base
7
+ has_many :taggings, dependent: :destroy
8
+ has_many :tags, through: :taggings
9
+ end
10
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ module Mihari
6
+ class Tagging < ActiveRecord::Base
7
+ belongs_to :alert
8
+ belongs_to :tag
9
+ end
10
+ end
@@ -8,15 +8,15 @@ module Mihari
8
8
  DEFAULT_USERNAME = "mihari"
9
9
 
10
10
  def slack_channel
11
- ENV.fetch SLACK_CHANNEL_KEY, "#general"
11
+ Mihari.config.slack_channel || "#general"
12
12
  end
13
13
 
14
14
  def slack_webhook_url
15
- ENV.fetch SLACK_WEBHOOK_URL_KEY
15
+ Mihari.config.slack_webhook_url
16
16
  end
17
17
 
18
18
  def slack_webhook_url?
19
- ENV.key? SLACK_WEBHOOK_URL_KEY
19
+ !Mihari.config.slack_webhook_url.nil?
20
20
  end
21
21
 
22
22
  def valid?
@@ -25,7 +25,7 @@ module Mihari
25
25
 
26
26
  def notify(text:, attachments: [], mrkdwn: true)
27
27
  notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
28
- notifier.post(text: text, attachments: attachments, mrkdwn: true)
28
+ notifier.post(text: text, attachments: attachments, mrkdwn: mrkdwn)
29
29
  end
30
30
  end
31
31
  end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_model_serializers"
4
+
5
+ module Mihari
6
+ class AlertSerializer < ActiveModel::Serializer
7
+ attributes :title, :description, :source, :created_at
8
+
9
+ has_many :artifacts
10
+ has_many :tags, through: :taggings
11
+ end
12
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_model_serializers"
4
+
5
+ module Mihari
6
+ class ArtifactSerializer < ActiveModel::Serializer
7
+ attributes :data, :data_type
8
+ end
9
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_model_serializers"
4
+
5
+ module Mihari
6
+ class TagSerializer < ActiveModel::Serializer
7
+ attributes :name
8
+ end
9
+ end
@@ -4,7 +4,7 @@ module Mihari
4
4
  class Status
5
5
  def check
6
6
  statuses.map do |key, value|
7
- [key, convert(value)]
7
+ [key, convert(**value)]
8
8
  end.to_h
9
9
  end
10
10
 
@@ -44,7 +44,7 @@ module Mihari
44
44
 
45
45
  # @return [true, false]
46
46
  def mail?
47
- EmailAddress.valid? data
47
+ EmailAddress.valid? data, host_validation: :syntax
48
48
  end
49
49
 
50
50
  # @return [String, nil]
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- VERSION = "0.17.5"
4
+ VERSION = "1.0.0"
5
5
  end
@@ -26,27 +26,32 @@ Gem::Specification.new do |spec|
26
26
 
27
27
  spec.add_development_dependency "bundler", "~> 2.1"
28
28
  spec.add_development_dependency "coveralls", "~> 0.8"
29
- spec.add_development_dependency "fakefs", "~> 1.0"
29
+ spec.add_development_dependency "execjs", "~> 2.0"
30
+ spec.add_development_dependency "fakefs", "~> 1.2"
31
+ spec.add_development_dependency "pre-commit", "~> 0.39"
30
32
  spec.add_development_dependency "rake", "~> 13.0"
31
33
  spec.add_development_dependency "rspec", "~> 3.9"
34
+ spec.add_development_dependency "rubocop", "~> 0.82"
35
+ spec.add_development_dependency "rubocop-performance", "~> 1.5"
32
36
  spec.add_development_dependency "timecop", "~> 0.9"
33
- spec.add_development_dependency "vcr", "~> 5.0"
37
+ spec.add_development_dependency "vcr", "~> 5.1"
34
38
  spec.add_development_dependency "webmock", "~> 3.8"
35
39
 
40
+ spec.add_dependency "active_model_serializers", "~> 0.10"
41
+ spec.add_dependency "activerecord", "~> 6.0"
36
42
  spec.add_dependency "addressable", "~> 2.7"
37
43
  spec.add_dependency "binaryedge", "~> 0.1"
38
44
  spec.add_dependency "censu", "~> 0.2"
39
- spec.add_dependency "crtsh-rb", "~> 0.2"
45
+ spec.add_dependency "crtsh-rb", "~> 0.3"
40
46
  spec.add_dependency "dnpedia", "~> 0.1"
41
47
  spec.add_dependency "dnstwister", "~> 0.1"
42
48
  spec.add_dependency "email_address", "~> 0.1"
43
49
  spec.add_dependency "hachi", "~> 0.3"
44
- spec.add_dependency "lightly", "~> 0.3"
45
50
  spec.add_dependency "mem", "~> 0.1"
46
51
  spec.add_dependency "misp", "~> 0.1"
47
52
  spec.add_dependency "murmurhash3", "~> 0.1"
48
53
  spec.add_dependency "net-ping", "~> 2.0"
49
- spec.add_dependency "onyphe", "~> 1.1"
54
+ spec.add_dependency "onyphe", "~> 2.0"
50
55
  spec.add_dependency "parallel", "~> 1.19"
51
56
  spec.add_dependency "passive_circl", "~> 0.1"
52
57
  spec.add_dependency "passivetotalx", "~> 0.1"
@@ -55,6 +60,7 @@ Gem::Specification.new do |spec|
55
60
  spec.add_dependency "securitytrails", "~> 1.0"
56
61
  spec.add_dependency "shodanx", "~> 0.2"
57
62
  spec.add_dependency "slack-notifier", "~> 2.3"
63
+ spec.add_dependency "sqlite3", "~> 1.4"
58
64
  spec.add_dependency "thor", "~> 1.0"
59
65
  spec.add_dependency "urlscan", "~> 0.5"
60
66
  spec.add_dependency "virustotalx", "~> 1.1"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mihari
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.17.5
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-01-19 00:00:00.000000000 Z
11
+ date: 2020-05-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -38,20 +38,48 @@ dependencies:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0.8'
41
+ - !ruby/object:Gem::Dependency
42
+ name: execjs
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '2.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '2.0'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: fakefs
43
57
  requirement: !ruby/object:Gem::Requirement
44
58
  requirements:
45
59
  - - "~>"
46
60
  - !ruby/object:Gem::Version
47
- version: '1.0'
61
+ version: '1.2'
48
62
  type: :development
49
63
  prerelease: false
50
64
  version_requirements: !ruby/object:Gem::Requirement
51
65
  requirements:
52
66
  - - "~>"
53
67
  - !ruby/object:Gem::Version
54
- version: '1.0'
68
+ version: '1.2'
69
+ - !ruby/object:Gem::Dependency
70
+ name: pre-commit
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '0.39'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '0.39'
55
83
  - !ruby/object:Gem::Dependency
56
84
  name: rake
57
85
  requirement: !ruby/object:Gem::Requirement
@@ -80,6 +108,34 @@ dependencies:
80
108
  - - "~>"
81
109
  - !ruby/object:Gem::Version
82
110
  version: '3.9'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rubocop
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.82'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.82'
125
+ - !ruby/object:Gem::Dependency
126
+ name: rubocop-performance
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '1.5'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '1.5'
83
139
  - !ruby/object:Gem::Dependency
84
140
  name: timecop
85
141
  requirement: !ruby/object:Gem::Requirement
@@ -100,14 +156,14 @@ dependencies:
100
156
  requirements:
101
157
  - - "~>"
102
158
  - !ruby/object:Gem::Version
103
- version: '5.0'
159
+ version: '5.1'
104
160
  type: :development
105
161
  prerelease: false
106
162
  version_requirements: !ruby/object:Gem::Requirement
107
163
  requirements:
108
164
  - - "~>"
109
165
  - !ruby/object:Gem::Version
110
- version: '5.0'
166
+ version: '5.1'
111
167
  - !ruby/object:Gem::Dependency
112
168
  name: webmock
113
169
  requirement: !ruby/object:Gem::Requirement
@@ -122,6 +178,34 @@ dependencies:
122
178
  - - "~>"
123
179
  - !ruby/object:Gem::Version
124
180
  version: '3.8'
181
+ - !ruby/object:Gem::Dependency
182
+ name: active_model_serializers
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '0.10'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '0.10'
195
+ - !ruby/object:Gem::Dependency
196
+ name: activerecord
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '6.0'
202
+ type: :runtime
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '6.0'
125
209
  - !ruby/object:Gem::Dependency
126
210
  name: addressable
127
211
  requirement: !ruby/object:Gem::Requirement
@@ -170,14 +254,14 @@ dependencies:
170
254
  requirements:
171
255
  - - "~>"
172
256
  - !ruby/object:Gem::Version
173
- version: '0.2'
257
+ version: '0.3'
174
258
  type: :runtime
175
259
  prerelease: false
176
260
  version_requirements: !ruby/object:Gem::Requirement
177
261
  requirements:
178
262
  - - "~>"
179
263
  - !ruby/object:Gem::Version
180
- version: '0.2'
264
+ version: '0.3'
181
265
  - !ruby/object:Gem::Dependency
182
266
  name: dnpedia
183
267
  requirement: !ruby/object:Gem::Requirement
@@ -234,20 +318,6 @@ dependencies:
234
318
  - - "~>"
235
319
  - !ruby/object:Gem::Version
236
320
  version: '0.3'
237
- - !ruby/object:Gem::Dependency
238
- name: lightly
239
- requirement: !ruby/object:Gem::Requirement
240
- requirements:
241
- - - "~>"
242
- - !ruby/object:Gem::Version
243
- version: '0.3'
244
- type: :runtime
245
- prerelease: false
246
- version_requirements: !ruby/object:Gem::Requirement
247
- requirements:
248
- - - "~>"
249
- - !ruby/object:Gem::Version
250
- version: '0.3'
251
321
  - !ruby/object:Gem::Dependency
252
322
  name: mem
253
323
  requirement: !ruby/object:Gem::Requirement
@@ -310,14 +380,14 @@ dependencies:
310
380
  requirements:
311
381
  - - "~>"
312
382
  - !ruby/object:Gem::Version
313
- version: '1.1'
383
+ version: '2.0'
314
384
  type: :runtime
315
385
  prerelease: false
316
386
  version_requirements: !ruby/object:Gem::Requirement
317
387
  requirements:
318
388
  - - "~>"
319
389
  - !ruby/object:Gem::Version
320
- version: '1.1'
390
+ version: '2.0'
321
391
  - !ruby/object:Gem::Dependency
322
392
  name: parallel
323
393
  requirement: !ruby/object:Gem::Requirement
@@ -430,6 +500,20 @@ dependencies:
430
500
  - - "~>"
431
501
  - !ruby/object:Gem::Version
432
502
  version: '2.3'
503
+ - !ruby/object:Gem::Dependency
504
+ name: sqlite3
505
+ requirement: !ruby/object:Gem::Requirement
506
+ requirements:
507
+ - - "~>"
508
+ - !ruby/object:Gem::Version
509
+ version: '1.4'
510
+ type: :runtime
511
+ prerelease: false
512
+ version_requirements: !ruby/object:Gem::Requirement
513
+ requirements:
514
+ - - "~>"
515
+ - !ruby/object:Gem::Version
516
+ version: '1.4'
433
517
  - !ruby/object:Gem::Dependency
434
518
  name: thor
435
519
  requirement: !ruby/object:Gem::Requirement
@@ -496,6 +580,7 @@ extra_rdoc_files: []
496
580
  files:
497
581
  - ".gitignore"
498
582
  - ".rspec"
583
+ - ".rubocop.yml"
499
584
  - ".travis.yml"
500
585
  - Gemfile
501
586
  - LICENSE
@@ -503,6 +588,7 @@ files:
503
588
  - Rakefile
504
589
  - bin/console
505
590
  - bin/setup
591
+ - config/pre_commit.yml
506
592
  - docker/Dockerfile
507
593
  - examples/ipinfo_hosted_domains.rb
508
594
  - exe/mihari
@@ -531,27 +617,30 @@ files:
531
617
  - lib/mihari/analyzers/urlscan.rb
532
618
  - lib/mihari/analyzers/virustotal.rb
533
619
  - lib/mihari/analyzers/zoomeye.rb
534
- - lib/mihari/artifact.rb
535
- - lib/mihari/cache.rb
536
620
  - lib/mihari/cli.rb
537
621
  - lib/mihari/config.rb
538
622
  - lib/mihari/configurable.rb
623
+ - lib/mihari/database.rb
539
624
  - lib/mihari/emitters/base.rb
540
625
  - lib/mihari/emitters/misp.rb
541
626
  - lib/mihari/emitters/slack.rb
627
+ - lib/mihari/emitters/sqlite.rb
542
628
  - lib/mihari/emitters/stdout.rb
543
629
  - lib/mihari/emitters/the_hive.rb
544
630
  - lib/mihari/errors.rb
545
631
  - lib/mihari/html.rb
632
+ - lib/mihari/models/alert.rb
633
+ - lib/mihari/models/artifact.rb
634
+ - lib/mihari/models/tag.rb
635
+ - lib/mihari/models/tagging.rb
546
636
  - lib/mihari/notifiers/base.rb
547
637
  - lib/mihari/notifiers/exception_notifier.rb
548
638
  - lib/mihari/notifiers/slack.rb
549
639
  - lib/mihari/retriable.rb
640
+ - lib/mihari/serializers/alert.rb
641
+ - lib/mihari/serializers/artifact.rb
642
+ - lib/mihari/serializers/tag.rb
550
643
  - lib/mihari/status.rb
551
- - lib/mihari/the_hive.rb
552
- - lib/mihari/the_hive/alert.rb
553
- - lib/mihari/the_hive/artifact.rb
554
- - lib/mihari/the_hive/base.rb
555
644
  - lib/mihari/type_checker.rb
556
645
  - lib/mihari/version.rb
557
646
  - mihari.gemspec
@@ -578,7 +667,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
578
667
  - !ruby/object:Gem::Version
579
668
  version: '0'
580
669
  requirements: []
581
- rubygems_version: 3.0.3
670
+ rubygems_version: 3.1.2
582
671
  signing_key:
583
672
  specification_version: 4
584
673
  summary: A framework for continuous malicious hosts monitoring.