mihari 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6117c49bfadf5c4d263727d684ac3f54f5296860078a8b65e4bdf9274574eaf7
-  data.tar.gz: f06c30c6abc0d61eda4beafa42be1f8034bea0478142fc0e15b36a4f3cde20ad
+  metadata.gz: ced23a972f1bf15c0d379cebbb11c303b3819d09ec56cf7f3f8936f6376cdd43
+  data.tar.gz: 3bef1fa391842b63d005cd75041def63dbba8ec473de4b8f56adfdf1e7119ab6
 SHA512:
-  metadata.gz: e49d5771be75ef6277c3169abccd5cd67e349493c67fee2e1fbcd3e9b08d4c3bbc7855c78d7466017ef8f17b7356205acc1a50c43be83abfd5c3194d02115edc
-  data.tar.gz: f472ece83577e3c3e14297cb6b5b5349f8d9f906f8ba0d27ffbf4d8bf132dd0b3726877fa2a6d7f9770bce9a483d75c8154d0e7b058b3d9bb2ddfb7aa9390196
+  metadata.gz: 7e3a6503831ca7dbdd54fb28861d51e9fe7b8e622db4dd1843a69780ae14be8bf28f81ccf7fb93ed09af33ea2d4af89a82b85de14b420e55ada42c5bb564a7e4
+  data.tar.gz: 07fbc1dd07f0cf38f071482e6c28062719e417c3b1e544373a61e4cf692d32d7b14c2512382350d43f7612d276996e5d8cdbe61e614edd3358de5b28aa14987f

data/README.md

@@ -5,7 +5,7 @@
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 
-mihari(`見張り`) is a sidekick tool for [TheHive](https://github.com/TheHive-Project/TheHive) to monitor malicious hosts (C2 / landing page / phishing, etc.) continuously.
+mihari(`見張り`) is a sidekick tool for [TheHive](https://github.com/TheHive-Project/TheHive) for monitoring malicious hosts (C2 / landing page / phishing, etc.) continuously.
 
 ## How it works
 
@@ -14,6 +14,8 @@ mihari(`見張り`) is a sidekick tool for [TheHive](https://github.com/TheHive-
     - mihari creates an alert with the artifacts on the TheHive instance.
     - mihari sends a notification to Slack. (Optional)
 
+![img](./screenshots/eyecatch.png)
+
 Check this blog post for more detail: [Continuous C2 hunting with Censys, Shodan, Onyphe and TheHive](https://hackmd.io/s/SkUaSrqoE)
 
 ### Screenshots
@@ -137,6 +139,10 @@ example.run
 
 See `/examples` for more.
 
+## Caching
+
+mihari caches execution results in `/tmp/mihari` and the default cache duration is 7 days. If you want to clear the cache, please clear `/tmp/mihari`.
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/mihari.rb

@@ -17,8 +17,9 @@ require "mihari/version"
 
 require "mihari/errors"
 
-require "mihari/type_checker"
 require "mihari/artifact"
+require "mihari/cache"
+require "mihari/type_checker"
 
 require "mihari/the_hive/base"
 require "mihari/the_hive/alert"

data/lib/mihari/analyzers/base.rb

@@ -3,10 +3,9 @@
 module Mihari
   module Analyzers
     class Base
-      attr_reader :the_hive
-
       def initialize
         @the_hive = TheHive.new
+        @cache = Cache.new
       end
 
       # @return [Array<String>, Array<Mihari::Artifact>]
@@ -36,6 +35,8 @@ module Mihari
 
           run_emitter emitter
         end
+
+        save_as_cache unique_artifacts.map(&:data)
       end
 
       def run_emitter(emitter)
@@ -53,11 +54,21 @@ module Mihari
         end.select(&:valid?)
       end
 
+      def uncached_artifacts
+        @uncached_artifacts ||= normalized_artifacts.reject do |artifact|
+          @cache.cached? artifact.data
+        end
+      end
+
       # @return [Array<Mihari::Artifact>]
       def unique_artifacts
-        return normalized_artifacts unless the_hive.valid?
+        return uncached_artifacts unless @the_hive.valid?
+
+        @unique_artifacts ||= @the_hive.artifact.find_non_existing_artifacts(uncached_artifacts)
+      end
 
-        the_hive.artifact.find_non_existing_artifacts(normalized_artifacts)
+      def save_as_cache(data)
+        @cache.save data
       end
     end
   end

data/lib/mihari/cache.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "lightly"
+
+module Mihari
+  class Cache
+    DEFAULT_CACHE_DIR = "/tmp/mihari"
+
+    def initialize
+      @data = Lightly.new(life: "7d", dir: DEFAULT_CACHE_DIR)
+    end
+
+    def cached?(key)
+      return false unless @data.enabled?
+
+      begin
+        @data.cached? key
+      rescue Errno::ENOENT => _e
+        false
+      end
+    end
+
+    def save(*keys)
+      return unless @data.enabled?
+
+      begin
+        keys.flatten.each do |key|
+          @data.save key, true
+        end
+      rescue Errno::ENOENT => _e
+        nil
+      end
+    end
+  end
+end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "0.8.0"
+  VERSION = "0.8.1"
 end

data/mihari.gemspec

@@ -26,8 +26,10 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "coveralls", "~> 0.8"
+  spec.add_development_dependency "fakefs", "~> 0.20"
   spec.add_development_dependency "rake", "~> 12.3"
   spec.add_development_dependency "rspec", "~> 3.8"
+  spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 5.0"
   spec.add_development_dependency "webmock", "~> 3.7"
 
@@ -36,14 +38,15 @@ Gem::Specification.new do |spec|
   spec.add_dependency "crtsh-rb", "~> 0.1"
   spec.add_dependency "email_address", "~> 0.1"
   spec.add_dependency "hachi", "~> 0.2"
+  spec.add_dependency "lightly", "~> 0.3"
   spec.add_dependency "mem", "~> 0.1"
   spec.add_dependency "net-ping", "~> 2.0"
   spec.add_dependency "onyphe", "~> 0.2"
-  spec.add_dependency "public_suffix", "~> 3.1"
+  spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "securitytrails", "~> 0.2"
-  spec.add_dependency "shodanx", "~> 0.1"
+  spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.3"
   spec.add_dependency "thor", "~> 0.20"
-  spec.add_dependency "urlscan", "~> 0.3"
+  spec.add_dependency "urlscan", "~> 0.4"
   spec.add_dependency "virustotalx", "~> 0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-09-13 00:00:00.000000000 Z
+date: 2019-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.20'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.20'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: lightly
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: mem
   requirement: !ruby/object:Gem::Requirement
@@ -212,14 +254,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: securitytrails
   requirement: !ruby/object:Gem::Requirement
@@ -240,14 +282,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
@@ -282,14 +324,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: virustotalx
   requirement: !ruby/object:Gem::Requirement
@@ -337,6 +379,7 @@ files:
 - lib/mihari/analyzers/urlscan.rb
 - lib/mihari/analyzers/virustotal.rb
 - lib/mihari/artifact.rb
+- lib/mihari/cache.rb
 - lib/mihari/cli.rb
 - lib/mihari/emitters/base.rb
 - lib/mihari/emitters/slack.rb
@@ -355,6 +398,7 @@ files:
 - lib/mihari/version.rb
 - mihari.gemspec
 - screenshots/alert.png
+- screenshots/eyecatch.png
 - screenshots/slack.png
 homepage: https://github.com/ninoseki/mihari
 licenses: