mihari 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '00748afea4184f8ea3595e822be84acf14a97f78d7d91390013c7bc49d3792bc'
-  data.tar.gz: fb9f40550592648a4c4c9095aadf714efbea3f35de82f26f1fd2648c9c054409
+  metadata.gz: afc816e7fe81735a8e8b007fe32884b1188f09b9fa905fe1603584b4a6d6fbfb
+  data.tar.gz: c7f689b44a4964e982744d67861c6f93f567e4c01ae0d6c43c6419dba063aa73
 SHA512:
-  metadata.gz: bd032efa642aa96133776a8e1483f7e98485dd5c86395b2eec41c9c8806ebf300131c2a2cafa91c9a0027b08136679f4d6926016ad5d0504514bb657f2440de1
-  data.tar.gz: d77bebb5f059bc55e0a467f3547b5cd3148551d3101037cad0667a1568c8e2cec1f69ea8bee7bf86f71805909ed695073450a19296d39ae0c1c679762a8232f6
+  metadata.gz: 3dfe252ff2f2572328a2101fd50a97cb4ff236430d60d5985f861b885f78daef780f93f67975e78be5debe6ee6fab88777bacc2f7e338681becf2251e755513f
+  data.tar.gz: 8ec54176b1f19bcb1b7542e376d7a8a87aef0fb076270408bdea60a08bbb6efe787b9824436c39129a61d0e99a953154d31277a9ebb67dd644fcbcaaf91650be

data/README.md

@@ -1,10 +1,11 @@
 # mihari
 
+[![Gem Version](https://badge.fury.io/rb/mihari.svg)](https://badge.fury.io/rb/mihari)
 [![Build Status](https://travis-ci.org/ninoseki/mihari.svg?branch=master)](https://travis-ci.org/ninoseki/mihari)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 
-mihari(`見張り`) is a framework for continuous malicious hosts (C2 / landing page / phishing, etc.) monitoring backended with [TheHive](https://github.com/TheHive-Project/TheHive).
+mihari(`見張り`) is a sidekick tool for [TheHive](https://github.com/TheHive-Project/TheHive) to monitor malicious hosts (C2 / landing page / phishing, etc.) continuously.
 
 ## How it works
 
@@ -33,41 +34,18 @@ gem install mihari
 
 ## Basic usage
 
-mihari supports Censys, Shodan and Onyphe by default.
+mihari supports Censys, Shodan, Onyphe, urlscan and VirusTotal by default.
 
 ```bash
 $ mihari
 Commands:
-  mihari censys [QUERY]    # Censys IPv4 lookup by a given query
-  mihari help [COMMAND]    # Describe available commands or one specific command
-  mihari import_from_json  # Give a JSON input via STDIN
-  mihari onyphe [QUERY]    # Onyphe datascan lookup by a given query
-  mihari shodan [QUERY]    # Shodan host lookup by a given query
-  mihari urlscan [QUERY]   # urlscan lookup by a given query
-```
-
-### Censys
-
-```bash
-mihari censys "YOUR_QUERY"
-```
-
-### Shodan
-
-```bash
-mihari shodan "YOUR QUERY"
-```
-
-### Onyphe
-
-```bash
-mihari onyphe "YOUR QUERY"
-```
-
-### urlscan.io
-
-```bash
-mihari urlscan "YOUR QUERY"
+  mihari censys [QUERY]          # Censys IPv4 lookup by a given query
+  mihari help [COMMAND]          # Describe available commands or one specific command
+  mihari import_from_json        # Give a JSON input via STDIN
+  mihari onyphe [QUERY]          # Onyphe datascan lookup by a given query
+  mihari shodan [QUERY]          # Shodan host lookup by a given query
+  mihari urlscan [QUERY]         # urlscan lookup by a given query
+  mihari virustotal [IP|DOMAIN]  # VirusTotal resolutions lookup by a given ip or domain
 ```
 
 ### Import from JSON
@@ -108,6 +86,7 @@ All configuration is done via ENV variables.
 | CENSYS_SECRET        | Censys secret      | Optional                       |
 | SHODAN_API_KEY       | Shodan API key     | Optional                       |
 | ONYPHE_API_KEY       | Onyphe API key     | Optional                       |
+| VIRUSTOTAL_API_KEY   | VirusTotal API key | Optional                       |
 
 ## How to create a custom analyzer
 

data/lib/mihari/analyzers/base.rb

@@ -30,18 +30,16 @@ module Mihari
       end
 
       def run(reject_exists_ones: true)
-        unique_artifacts = normalized_artifacts.reject do |artifact|
-          reject_exists_ones & the_hive.valid? && the_hive.exists?(data: artifact.data, data_type: artifact.data_type)
-        end
+        artifacts = reject_exists_ones ? unique_artifacts : normalized_artifacts
 
-        Mihari.notifiers.each do |notifier_class|
-          notifier = notifier_class.new
-          next unless notifier.valid?
+        Mihari.emitters.each do |emitter_class|
+          emitter = emitter_class.new
+          next unless emitter.valid?
 
           begin
-            notifier.notify(title: title, description: description, artifacts: unique_artifacts, tags: tags)
+            emitter.emit(title: title, description: description, artifacts: artifacts, tags: tags)
           rescue StandardError => e
-            puts "Sending notification by #{notifier.class} is failed: #{e}"
+            puts "Sending notification by #{emitter.class} is failed: #{e}"
           end
         end
       end
@@ -54,6 +52,13 @@ module Mihari
           artifact.is_a?(Artifact) ? artifact : Artifact.new(artifact)
         end.select(&:valid?)
       end
+
+      # @return [Array<Mihari::Artifact>]
+      def unique_artifacts
+        normalized_artifacts.reject do |artifact|
+          the_hive.valid? && the_hive.exists?(data: artifact.data, data_type: artifact.data_type)
+        end
+      end
     end
   end
 end

data/lib/mihari/analyzers/urlscan.rb

@@ -28,7 +28,7 @@ module Mihari
         results = result.dig("results") || []
         results.map do |match|
           match.dig "task", "url"
-        end.compact
+        end.compact.uniq
       end
 
       private

data/lib/mihari/analyzers/virustotal.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "virustotal"
+
+module Mihari
+  module Analyzers
+    class VirusTotal < Base
+      attr_reader :api
+      attr_reader :indicator
+      attr_reader :type
+
+      attr_reader :title
+      attr_reader :description
+      attr_reader :tags
+
+      def initialize(indicator, tags: [])
+        super()
+
+        @api = ::VirusTotal::API.new
+        @indicator = indicator
+        @type = TypeChecker.type(indicator)
+
+        @title = "VirusTotal lookup"
+        @description = "indicator = #{indicator}"
+        @tags = tags
+      end
+
+      def artifacts
+        lookup || []
+      end
+
+      private
+
+      def valid_type?
+        %w(ip domain).include? type
+      end
+
+      def lookup
+        case type
+        when "domain"
+          domain_lookup
+        when "ip"
+          ip_lookup
+        else
+          raise ArgumentError, "#{indicator}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        end
+      rescue ::VirusTotal::Error => _e
+        nil
+      end
+
+      def domain_lookup
+        report = api.domain.report(indicator)
+        return nil unless report
+
+        resolutions = report.dig("resolutions") || []
+        resolutions.map do |resolution|
+          resolution.dig("ip_address")
+        end.compact.uniq
+      end
+
+      def ip_lookup
+        report = api.ip_address.report(indicator)
+        return nil unless report
+
+        resolutions = report.dig("resolutions") || []
+        resolutions.map do |resolution|
+          resolution.dig("hostname")
+        end.compact.uniq
+      end
+    end
+  end
+end

data/lib/mihari/cli.rb

@@ -37,6 +37,14 @@ module Mihari
       run_analyzer urlscan
     end
 
+    desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions lookup by a given ip or domain"
+    method_option :tags, type: :array, desc: "tags"
+    def virustotal(indiactor)
+      tags = options.dig("tags") || []
+      virustotal = Analyzers::VirusTotal.new(indiactor, tags: tags)
+      run_analyzer virustotal
+    end
+
     desc "import_from_json", "Give a JSON input via STDIN"
     def import_from_json(input = nil)
       json = input || STDIN.gets.chomp

data/lib/mihari/{notifiers → emitters}/base.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 module Mihari
-  module Notifiers
+  module Emitters
     class Base
       def self.inherited(child)
-        Mihari.notifiers << child
+        Mihari.emitters << child
       end
 
       # @return [true, false]
@@ -12,7 +12,7 @@ module Mihari
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
-      def notify(title:, description:, artifacts:)
+      def emit(title:, description:, artifacts:)
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
     end

data/lib/mihari/{notifiers → emitters}/slack.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require "slack/incoming/webhooks"
+require "slack-notifier"
 require "digest/sha2"
 require "mem"
 
 module Mihari
-  module Notifiers
+  module Emitters
     class Attachment
       include Mem
 
@@ -125,14 +125,14 @@ module Mihari
         end.flatten
       end
 
-      def notify(title:, description:, artifacts:, tags:)
+      def emit(title:, description:, artifacts:, tags:)
         return if artifacts.empty?
 
         attachments = to_attachments(artifacts)
         tags << ["N/A"] if tags.empty?
 
-        slack = ::Slack::Incoming::Webhooks.new(slack_webhook_url, channel: slack_channel)
-        slack.post("#{title} (desc.: #{description} / tags: #{tags.join(', ')})", attachments: attachments)
+        notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
+        notifier.post(text: "#{title} (desc.: #{description} / tags: #{tags.join(', ')})", attachments: attachments)
       end
     end
   end

data/lib/mihari/emitters/stdout.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Mihari
+  module Emitters
+    class StandardOutput < Base
+      def valid?
+        true
+      end
+
+      def emit(title:, description:, artifacts:, tags:)
+        h = {
+          title: title,
+          description: description,
+          artifacts: artifacts.map(&:data),
+          tags: tags
+        }
+        puts JSON.pretty_generate(h)
+      end
+    end
+  end
+end

data/lib/mihari/{notifiers → emitters}/the_hive.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Mihari
-  module Notifiers
+  module Emitters
     class TheHive < Base
       attr_reader :api
 
@@ -14,17 +14,15 @@ module Mihari
         api.valid?
       end
 
-      def notify(title:, description:, artifacts:, tags: [])
+      def emit(title:, description:, artifacts:, tags: [])
         return if artifacts.empty?
 
-        res = api.create_alert(
+        api.create_alert(
           title: title,
           description: description,
           artifacts: artifacts.map(&:to_h),
           tags: tags
         )
-        id = res.dig("id")
-        puts "A new alret is created. (id: #{id})"
       end
     end
   end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/mihari.rb

@@ -6,10 +6,10 @@ module Mihari
   class << self
     include Mem
 
-    def notifiers
+    def emitters
       []
     end
-    memoize :notifiers
+    memoize :emitters
   end
 end
 
@@ -28,9 +28,11 @@ require "mihari/analyzers/censys"
 require "mihari/analyzers/onyphe"
 require "mihari/analyzers/shodan"
 require "mihari/analyzers/urlscan"
+require "mihari/analyzers/virustotal"
 
-require "mihari/notifiers/base"
-require "mihari/notifiers/slack"
-require "mihari/notifiers/the_hive"
+require "mihari/emitters/base"
+require "mihari/emitters/slack"
+require "mihari/emitters/stdout"
+require "mihari/emitters/the_hive"
 
 require "mihari/cli"

data/mihari.gemspec

@@ -36,11 +36,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency "email_address", "~> 0.1"
   spec.add_dependency "hachi", "~> 0.1"
   spec.add_dependency "mem", "~> 0.1"
-  spec.add_dependency "net-ping"
+  spec.add_dependency "net-ping", "~> 2.0"
   spec.add_dependency "onyphe", "~> 0.2"
   spec.add_dependency "public_suffix", "~> 3.1"
   spec.add_dependency "shodanx", "~> 0.1"
-  spec.add_dependency "slack-incoming-webhooks", "~> 0.2"
+  spec.add_dependency "slack-notifier", "~> 2.3"
   spec.add_dependency "thor", "~> 0.19"
   spec.add_dependency "urlscan", "~> 0.2"
+  spec.add_dependency "virustotalx", "~> 0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-26 00:00:00.000000000 Z
+date: 2019-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -168,16 +168,16 @@ dependencies:
   name: net-ping
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: onyphe
   requirement: !ruby/object:Gem::Requirement
@@ -221,19 +221,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: slack-incoming-webhooks
+  name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -262,6 +262,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: virustotalx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 description: A framework for continuous malicious hosts monitoring.
 email:
 - manabu.niseki@gmail.com
@@ -289,12 +303,14 @@ files:
 - lib/mihari/analyzers/onyphe.rb
 - lib/mihari/analyzers/shodan.rb
 - lib/mihari/analyzers/urlscan.rb
+- lib/mihari/analyzers/virustotal.rb
 - lib/mihari/artifact.rb
 - lib/mihari/cli.rb
+- lib/mihari/emitters/base.rb
+- lib/mihari/emitters/slack.rb
+- lib/mihari/emitters/stdout.rb
+- lib/mihari/emitters/the_hive.rb
 - lib/mihari/errors.rb
-- lib/mihari/notifiers/base.rb
-- lib/mihari/notifiers/slack.rb
-- lib/mihari/notifiers/the_hive.rb
 - lib/mihari/the_hive.rb
 - lib/mihari/type_checker.rb
 - lib/mihari/version.rb