mihari 0.17.5 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +3 -0
  3. data/.rubocop.yml +155 -0
  4. data/.travis.yml +7 -1
  5. data/Gemfile +2 -0
  6. data/README.md +45 -73
  7. data/config/pre_commit.yml +3 -0
  8. data/docker/Dockerfile +1 -1
  9. data/lib/mihari.rb +13 -8
  10. data/lib/mihari/alert_viewer.rb +16 -34
  11. data/lib/mihari/analyzers/base.rb +7 -19
  12. data/lib/mihari/analyzers/basic.rb +3 -1
  13. data/lib/mihari/analyzers/binaryedge.rb +2 -2
  14. data/lib/mihari/analyzers/censys.rb +2 -2
  15. data/lib/mihari/analyzers/circl.rb +2 -2
  16. data/lib/mihari/analyzers/onyphe.rb +3 -3
  17. data/lib/mihari/analyzers/otx.rb +74 -0
  18. data/lib/mihari/analyzers/passive_dns.rb +2 -1
  19. data/lib/mihari/analyzers/passivetotal.rb +2 -2
  20. data/lib/mihari/analyzers/pulsedive.rb +2 -2
  21. data/lib/mihari/analyzers/securitytrails.rb +2 -2
  22. data/lib/mihari/analyzers/securitytrails_domain_feed.rb +2 -2
  23. data/lib/mihari/analyzers/shodan.rb +2 -2
  24. data/lib/mihari/analyzers/virustotal.rb +2 -2
  25. data/lib/mihari/analyzers/zoomeye.rb +2 -2
  26. data/lib/mihari/cli.rb +23 -4
  27. data/lib/mihari/config.rb +70 -2
  28. data/lib/mihari/configurable.rb +1 -1
  29. data/lib/mihari/database.rb +68 -0
  30. data/lib/mihari/emitters/base.rb +1 -1
  31. data/lib/mihari/emitters/database.rb +29 -0
  32. data/lib/mihari/emitters/misp.rb +8 -1
  33. data/lib/mihari/emitters/slack.rb +4 -2
  34. data/lib/mihari/emitters/stdout.rb +2 -1
  35. data/lib/mihari/emitters/the_hive.rb +28 -14
  36. data/lib/mihari/models/alert.rb +11 -0
  37. data/lib/mihari/models/artifact.rb +27 -0
  38. data/lib/mihari/models/tag.rb +10 -0
  39. data/lib/mihari/models/tagging.rb +10 -0
  40. data/lib/mihari/notifiers/slack.rb +7 -4
  41. data/lib/mihari/serializers/alert.rb +12 -0
  42. data/lib/mihari/serializers/artifact.rb +9 -0
  43. data/lib/mihari/serializers/tag.rb +9 -0
  44. data/lib/mihari/slack_monkeypatch.rb +16 -0
  45. data/lib/mihari/status.rb +1 -1
  46. data/lib/mihari/type_checker.rb +1 -1
  47. data/lib/mihari/version.rb +1 -1
  48. data/mihari.gemspec +13 -5
  49. metadata +149 -30
  50. data/lib/mihari/artifact.rb +0 -36
  51. data/lib/mihari/cache.rb +0 -35
  52. data/lib/mihari/the_hive.rb +0 -42
  53. data/lib/mihari/the_hive/alert.rb +0 -25
  54. data/lib/mihari/the_hive/artifact.rb +0 -33
  55. data/lib/mihari/the_hive/base.rb +0 -14
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_model_serializers"
4
+
5
+ module Mihari
6
+ class ArtifactSerializer < ActiveModel::Serializer
7
+ attributes :data, :data_type
8
+ end
9
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_model_serializers"
4
+
5
+ module Mihari
6
+ class TagSerializer < ActiveModel::Serializer
7
+ attributes :name
8
+ end
9
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Slack
4
+ class Notifier
5
+ module Util
6
+ class LinkFormatter
7
+ class << self
8
+ def format(string, opts = {})
9
+ # Resolve warning in Ruby 2.7
10
+ LinkFormatter.new(string, **opts).formatted
11
+ end
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -4,7 +4,7 @@ module Mihari
4
4
  class Status
5
5
  def check
6
6
  statuses.map do |key, value|
7
- [key, convert(value)]
7
+ [key, convert(**value)]
8
8
  end.to_h
9
9
  end
10
10
 
@@ -44,7 +44,7 @@ module Mihari
44
44
 
45
45
  # @return [true, false]
46
46
  def mail?
47
- EmailAddress.valid? data
47
+ EmailAddress.valid? data, host_validation: :syntax
48
48
  end
49
49
 
50
50
  # @return [String, nil]
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Mihari
4
- VERSION = "0.17.5"
4
+ VERSION = "1.2.0"
5
5
  end
@@ -26,35 +26,43 @@ Gem::Specification.new do |spec|
26
26
 
27
27
  spec.add_development_dependency "bundler", "~> 2.1"
28
28
  spec.add_development_dependency "coveralls", "~> 0.8"
29
- spec.add_development_dependency "fakefs", "~> 1.0"
29
+ spec.add_development_dependency "execjs", "~> 2.7"
30
+ spec.add_development_dependency "fakefs", "~> 1.2"
31
+ spec.add_development_dependency "pre-commit", "~> 0.39"
30
32
  spec.add_development_dependency "rake", "~> 13.0"
31
33
  spec.add_development_dependency "rspec", "~> 3.9"
34
+ spec.add_development_dependency "rubocop", "~> 0.88"
35
+ spec.add_development_dependency "rubocop-performance", "~> 1.7"
32
36
  spec.add_development_dependency "timecop", "~> 0.9"
33
- spec.add_development_dependency "vcr", "~> 5.0"
37
+ spec.add_development_dependency "vcr", "~> 6.0"
34
38
  spec.add_development_dependency "webmock", "~> 3.8"
35
39
 
40
+ spec.add_dependency "active_model_serializers", "~> 0.10"
41
+ spec.add_dependency "activerecord", "~> 6.0"
36
42
  spec.add_dependency "addressable", "~> 2.7"
37
43
  spec.add_dependency "binaryedge", "~> 0.1"
38
44
  spec.add_dependency "censu", "~> 0.2"
39
- spec.add_dependency "crtsh-rb", "~> 0.2"
45
+ spec.add_dependency "crtsh-rb", "~> 0.3"
40
46
  spec.add_dependency "dnpedia", "~> 0.1"
41
47
  spec.add_dependency "dnstwister", "~> 0.1"
42
48
  spec.add_dependency "email_address", "~> 0.1"
43
49
  spec.add_dependency "hachi", "~> 0.3"
44
- spec.add_dependency "lightly", "~> 0.3"
45
50
  spec.add_dependency "mem", "~> 0.1"
46
51
  spec.add_dependency "misp", "~> 0.1"
47
52
  spec.add_dependency "murmurhash3", "~> 0.1"
48
53
  spec.add_dependency "net-ping", "~> 2.0"
49
- spec.add_dependency "onyphe", "~> 1.1"
54
+ spec.add_dependency "onyphe", "~> 2.0"
55
+ spec.add_dependency "otx_ruby", "~> 0.9"
50
56
  spec.add_dependency "parallel", "~> 1.19"
51
57
  spec.add_dependency "passive_circl", "~> 0.1"
52
58
  spec.add_dependency "passivetotalx", "~> 0.1"
59
+ spec.add_dependency "pg", "~> 1.2"
53
60
  spec.add_dependency "public_suffix", "~> 4.0"
54
61
  spec.add_dependency "pulsedive", "~> 0.1"
55
62
  spec.add_dependency "securitytrails", "~> 1.0"
56
63
  spec.add_dependency "shodanx", "~> 0.2"
57
64
  spec.add_dependency "slack-notifier", "~> 2.3"
65
+ spec.add_dependency "sqlite3", "~> 1.4"
58
66
  spec.add_dependency "thor", "~> 1.0"
59
67
  spec.add_dependency "urlscan", "~> 0.5"
60
68
  spec.add_dependency "virustotalx", "~> 1.1"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mihari
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.17.5
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-01-19 00:00:00.000000000 Z
11
+ date: 2020-08-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -38,20 +38,48 @@ dependencies:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0.8'
41
+ - !ruby/object:Gem::Dependency
42
+ name: execjs
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '2.7'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '2.7'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: fakefs
43
57
  requirement: !ruby/object:Gem::Requirement
44
58
  requirements:
45
59
  - - "~>"
46
60
  - !ruby/object:Gem::Version
47
- version: '1.0'
61
+ version: '1.2'
48
62
  type: :development
49
63
  prerelease: false
50
64
  version_requirements: !ruby/object:Gem::Requirement
51
65
  requirements:
52
66
  - - "~>"
53
67
  - !ruby/object:Gem::Version
54
- version: '1.0'
68
+ version: '1.2'
69
+ - !ruby/object:Gem::Dependency
70
+ name: pre-commit
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '0.39'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '0.39'
55
83
  - !ruby/object:Gem::Dependency
56
84
  name: rake
57
85
  requirement: !ruby/object:Gem::Requirement
@@ -80,6 +108,34 @@ dependencies:
80
108
  - - "~>"
81
109
  - !ruby/object:Gem::Version
82
110
  version: '3.9'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rubocop
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.88'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.88'
125
+ - !ruby/object:Gem::Dependency
126
+ name: rubocop-performance
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '1.7'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '1.7'
83
139
  - !ruby/object:Gem::Dependency
84
140
  name: timecop
85
141
  requirement: !ruby/object:Gem::Requirement
@@ -100,14 +156,14 @@ dependencies:
100
156
  requirements:
101
157
  - - "~>"
102
158
  - !ruby/object:Gem::Version
103
- version: '5.0'
159
+ version: '6.0'
104
160
  type: :development
105
161
  prerelease: false
106
162
  version_requirements: !ruby/object:Gem::Requirement
107
163
  requirements:
108
164
  - - "~>"
109
165
  - !ruby/object:Gem::Version
110
- version: '5.0'
166
+ version: '6.0'
111
167
  - !ruby/object:Gem::Dependency
112
168
  name: webmock
113
169
  requirement: !ruby/object:Gem::Requirement
@@ -122,6 +178,34 @@ dependencies:
122
178
  - - "~>"
123
179
  - !ruby/object:Gem::Version
124
180
  version: '3.8'
181
+ - !ruby/object:Gem::Dependency
182
+ name: active_model_serializers
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '0.10'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '0.10'
195
+ - !ruby/object:Gem::Dependency
196
+ name: activerecord
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '6.0'
202
+ type: :runtime
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '6.0'
125
209
  - !ruby/object:Gem::Dependency
126
210
  name: addressable
127
211
  requirement: !ruby/object:Gem::Requirement
@@ -170,14 +254,14 @@ dependencies:
170
254
  requirements:
171
255
  - - "~>"
172
256
  - !ruby/object:Gem::Version
173
- version: '0.2'
257
+ version: '0.3'
174
258
  type: :runtime
175
259
  prerelease: false
176
260
  version_requirements: !ruby/object:Gem::Requirement
177
261
  requirements:
178
262
  - - "~>"
179
263
  - !ruby/object:Gem::Version
180
- version: '0.2'
264
+ version: '0.3'
181
265
  - !ruby/object:Gem::Dependency
182
266
  name: dnpedia
183
267
  requirement: !ruby/object:Gem::Requirement
@@ -235,21 +319,21 @@ dependencies:
235
319
  - !ruby/object:Gem::Version
236
320
  version: '0.3'
237
321
  - !ruby/object:Gem::Dependency
238
- name: lightly
322
+ name: mem
239
323
  requirement: !ruby/object:Gem::Requirement
240
324
  requirements:
241
325
  - - "~>"
242
326
  - !ruby/object:Gem::Version
243
- version: '0.3'
327
+ version: '0.1'
244
328
  type: :runtime
245
329
  prerelease: false
246
330
  version_requirements: !ruby/object:Gem::Requirement
247
331
  requirements:
248
332
  - - "~>"
249
333
  - !ruby/object:Gem::Version
250
- version: '0.3'
334
+ version: '0.1'
251
335
  - !ruby/object:Gem::Dependency
252
- name: mem
336
+ name: misp
253
337
  requirement: !ruby/object:Gem::Requirement
254
338
  requirements:
255
339
  - - "~>"
@@ -263,7 +347,7 @@ dependencies:
263
347
  - !ruby/object:Gem::Version
264
348
  version: '0.1'
265
349
  - !ruby/object:Gem::Dependency
266
- name: misp
350
+ name: murmurhash3
267
351
  requirement: !ruby/object:Gem::Requirement
268
352
  requirements:
269
353
  - - "~>"
@@ -277,21 +361,21 @@ dependencies:
277
361
  - !ruby/object:Gem::Version
278
362
  version: '0.1'
279
363
  - !ruby/object:Gem::Dependency
280
- name: murmurhash3
364
+ name: net-ping
281
365
  requirement: !ruby/object:Gem::Requirement
282
366
  requirements:
283
367
  - - "~>"
284
368
  - !ruby/object:Gem::Version
285
- version: '0.1'
369
+ version: '2.0'
286
370
  type: :runtime
287
371
  prerelease: false
288
372
  version_requirements: !ruby/object:Gem::Requirement
289
373
  requirements:
290
374
  - - "~>"
291
375
  - !ruby/object:Gem::Version
292
- version: '0.1'
376
+ version: '2.0'
293
377
  - !ruby/object:Gem::Dependency
294
- name: net-ping
378
+ name: onyphe
295
379
  requirement: !ruby/object:Gem::Requirement
296
380
  requirements:
297
381
  - - "~>"
@@ -305,19 +389,19 @@ dependencies:
305
389
  - !ruby/object:Gem::Version
306
390
  version: '2.0'
307
391
  - !ruby/object:Gem::Dependency
308
- name: onyphe
392
+ name: otx_ruby
309
393
  requirement: !ruby/object:Gem::Requirement
310
394
  requirements:
311
395
  - - "~>"
312
396
  - !ruby/object:Gem::Version
313
- version: '1.1'
397
+ version: '0.9'
314
398
  type: :runtime
315
399
  prerelease: false
316
400
  version_requirements: !ruby/object:Gem::Requirement
317
401
  requirements:
318
402
  - - "~>"
319
403
  - !ruby/object:Gem::Version
320
- version: '1.1'
404
+ version: '0.9'
321
405
  - !ruby/object:Gem::Dependency
322
406
  name: parallel
323
407
  requirement: !ruby/object:Gem::Requirement
@@ -360,6 +444,20 @@ dependencies:
360
444
  - - "~>"
361
445
  - !ruby/object:Gem::Version
362
446
  version: '0.1'
447
+ - !ruby/object:Gem::Dependency
448
+ name: pg
449
+ requirement: !ruby/object:Gem::Requirement
450
+ requirements:
451
+ - - "~>"
452
+ - !ruby/object:Gem::Version
453
+ version: '1.2'
454
+ type: :runtime
455
+ prerelease: false
456
+ version_requirements: !ruby/object:Gem::Requirement
457
+ requirements:
458
+ - - "~>"
459
+ - !ruby/object:Gem::Version
460
+ version: '1.2'
363
461
  - !ruby/object:Gem::Dependency
364
462
  name: public_suffix
365
463
  requirement: !ruby/object:Gem::Requirement
@@ -430,6 +528,20 @@ dependencies:
430
528
  - - "~>"
431
529
  - !ruby/object:Gem::Version
432
530
  version: '2.3'
531
+ - !ruby/object:Gem::Dependency
532
+ name: sqlite3
533
+ requirement: !ruby/object:Gem::Requirement
534
+ requirements:
535
+ - - "~>"
536
+ - !ruby/object:Gem::Version
537
+ version: '1.4'
538
+ type: :runtime
539
+ prerelease: false
540
+ version_requirements: !ruby/object:Gem::Requirement
541
+ requirements:
542
+ - - "~>"
543
+ - !ruby/object:Gem::Version
544
+ version: '1.4'
433
545
  - !ruby/object:Gem::Dependency
434
546
  name: thor
435
547
  requirement: !ruby/object:Gem::Requirement
@@ -496,6 +608,7 @@ extra_rdoc_files: []
496
608
  files:
497
609
  - ".gitignore"
498
610
  - ".rspec"
611
+ - ".rubocop.yml"
499
612
  - ".travis.yml"
500
613
  - Gemfile
501
614
  - LICENSE
@@ -503,6 +616,7 @@ files:
503
616
  - Rakefile
504
617
  - bin/console
505
618
  - bin/setup
619
+ - config/pre_commit.yml
506
620
  - docker/Dockerfile
507
621
  - examples/ipinfo_hosted_domains.rb
508
622
  - exe/mihari
@@ -519,6 +633,7 @@ files:
519
633
  - lib/mihari/analyzers/free_text.rb
520
634
  - lib/mihari/analyzers/http_hash.rb
521
635
  - lib/mihari/analyzers/onyphe.rb
636
+ - lib/mihari/analyzers/otx.rb
522
637
  - lib/mihari/analyzers/passive_dns.rb
523
638
  - lib/mihari/analyzers/passive_ssl.rb
524
639
  - lib/mihari/analyzers/passivetotal.rb
@@ -531,27 +646,31 @@ files:
531
646
  - lib/mihari/analyzers/urlscan.rb
532
647
  - lib/mihari/analyzers/virustotal.rb
533
648
  - lib/mihari/analyzers/zoomeye.rb
534
- - lib/mihari/artifact.rb
535
- - lib/mihari/cache.rb
536
649
  - lib/mihari/cli.rb
537
650
  - lib/mihari/config.rb
538
651
  - lib/mihari/configurable.rb
652
+ - lib/mihari/database.rb
539
653
  - lib/mihari/emitters/base.rb
654
+ - lib/mihari/emitters/database.rb
540
655
  - lib/mihari/emitters/misp.rb
541
656
  - lib/mihari/emitters/slack.rb
542
657
  - lib/mihari/emitters/stdout.rb
543
658
  - lib/mihari/emitters/the_hive.rb
544
659
  - lib/mihari/errors.rb
545
660
  - lib/mihari/html.rb
661
+ - lib/mihari/models/alert.rb
662
+ - lib/mihari/models/artifact.rb
663
+ - lib/mihari/models/tag.rb
664
+ - lib/mihari/models/tagging.rb
546
665
  - lib/mihari/notifiers/base.rb
547
666
  - lib/mihari/notifiers/exception_notifier.rb
548
667
  - lib/mihari/notifiers/slack.rb
549
668
  - lib/mihari/retriable.rb
669
+ - lib/mihari/serializers/alert.rb
670
+ - lib/mihari/serializers/artifact.rb
671
+ - lib/mihari/serializers/tag.rb
672
+ - lib/mihari/slack_monkeypatch.rb
550
673
  - lib/mihari/status.rb
551
- - lib/mihari/the_hive.rb
552
- - lib/mihari/the_hive/alert.rb
553
- - lib/mihari/the_hive/artifact.rb
554
- - lib/mihari/the_hive/base.rb
555
674
  - lib/mihari/type_checker.rb
556
675
  - lib/mihari/version.rb
557
676
  - mihari.gemspec
@@ -563,7 +682,7 @@ homepage: https://github.com/ninoseki/mihari
563
682
  licenses:
564
683
  - MIT
565
684
  metadata: {}
566
- post_install_message:
685
+ post_install_message:
567
686
  rdoc_options: []
568
687
  require_paths:
569
688
  - lib
@@ -578,8 +697,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
578
697
  - !ruby/object:Gem::Version
579
698
  version: '0'
580
699
  requirements: []
581
- rubygems_version: 3.0.3
582
- signing_key:
700
+ rubygems_version: 3.1.2
701
+ signing_key:
583
702
  specification_version: 4
584
703
  summary: A framework for continuous malicious hosts monitoring.
585
704
  test_files: []