mihari 0.2.5 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bda91ccfe9d86e0217e40b23fd30cf526f61e7cf9146945939a04c0bd5edaef
-  data.tar.gz: 8e5665d600415ef98cd34ee064b73d740dbe0a3a9c8c2e7e4d1fe5d7f5cf6621
+  metadata.gz: '00748afea4184f8ea3595e822be84acf14a97f78d7d91390013c7bc49d3792bc'
+  data.tar.gz: fb9f40550592648a4c4c9095aadf714efbea3f35de82f26f1fd2648c9c054409
 SHA512:
-  metadata.gz: dfa5b4da08fe3f68f65f61856a28cff31a52e9eafdc3348147b0a7d5b02dc11714a63edf398accd2b0034055b36140c90d741d2b61a4ec2e0e52369bdaba8b2c
-  data.tar.gz: 48a0add497b69ab34f3ed61936464f0667f7f07de62e97b24356a92f2978823d869c163b1c6f58b91f4ba63641ce01617bda63b4804b5a34063df547886dda97
+  metadata.gz: bd032efa642aa96133776a8e1483f7e98485dd5c86395b2eec41c9c8806ebf300131c2a2cafa91c9a0027b08136679f4d6926016ad5d0504514bb657f2440de1
+  data.tar.gz: d77bebb5f059bc55e0a467f3547b5cd3148551d3101037cad0667a1568c8e2cec1f69ea8bee7bf86f71805909ed695073450a19296d39ae0c1c679762a8232f6

data/README.md

@@ -2,6 +2,7 @@
 
 [![Build Status](https://travis-ci.org/ninoseki/mihari.svg?branch=master)](https://travis-ci.org/ninoseki/mihari)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
+[![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 
 mihari(`見張り`) is a framework for continuous malicious hosts (C2 / landing page / phishing, etc.) monitoring backended with [TheHive](https://github.com/TheHive-Project/TheHive).
 
@@ -42,6 +43,7 @@ Commands:
   mihari import_from_json  # Give a JSON input via STDIN
   mihari onyphe [QUERY]    # Onyphe datascan lookup by a given query
   mihari shodan [QUERY]    # Shodan host lookup by a given query
+  mihari urlscan [QUERY]   # urlscan lookup by a given query
 ```
 
 ### Censys
@@ -62,6 +64,12 @@ mihari shodan "YOUR QUERY"
 mihari onyphe "YOUR QUERY"
 ```
 
+### urlscan.io
+
+```bash
+mihari urlscan "YOUR QUERY"
+```
+
 ### Import from JSON
 
 ```bash

data/lib/mihari.rb

@@ -27,6 +27,7 @@ require "mihari/analyzers/basic"
 require "mihari/analyzers/censys"
 require "mihari/analyzers/onyphe"
 require "mihari/analyzers/shodan"
+require "mihari/analyzers/urlscan"
 
 require "mihari/notifiers/base"
 require "mihari/notifiers/slack"

data/lib/mihari/analyzers/urlscan.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "urlscan"
+
+module Mihari
+  module Analyzers
+    class Urlscan < Base
+      attr_reader :api
+      attr_reader :title
+      attr_reader :description
+      attr_reader :query
+      attr_reader :tags
+
+      def initialize(query, tags: [])
+        super()
+
+        @api = ::UrlScan::API.new
+        @query = query
+        @title = "urlscan lookup"
+        @description = "query = #{query}"
+        @tags = tags
+      end
+
+      def artifacts
+        result = search
+        return [] unless result
+
+        results = result.dig("results") || []
+        results.map do |match|
+          match.dig "task", "url"
+        end.compact
+      end
+
+      private
+
+      def search
+        api.search(query)
+      rescue ::UrlScan::ResponseError => _e
+        nil
+      end
+    end
+  end
+end

data/lib/mihari/cli.rb

@@ -9,30 +9,32 @@ module Mihari
     method_option :tags, type: :array, desc: "tags"
     def censys(query)
       tags = options.dig("tags") || []
-      with_error_handling do
-        censys = Analyzers::Censys.new(query, tags: tags)
-        censys.run
-      end
+      censys = Analyzers::Censys.new(query, tags: tags)
+      run_analyzer censys
     end
 
     desc "shodan [QUERY]", "Shodan host lookup by a given query"
     method_option :tags, type: :array, desc: "tags"
     def shodan(query)
       tags = options.dig("tags") || []
-      with_error_handling do
-        shodan = Analyzers::Shodan.new(query, tags: tags)
-        shodan.run
-      end
+      shodan = Analyzers::Shodan.new(query, tags: tags)
+      run_analyzer shodan
     end
 
     desc "onyphe [QUERY]", "Onyphe datascan lookup by a given query"
     method_option :tags, type: :array, desc: "tags"
     def onyphe(query)
       tags = options.dig("tags") || []
-      with_error_handling do
-        onyphe = Analyzers::Onyphe.new(query, tags: tags)
-        onyphe.run
-      end
+      onyphe = Analyzers::Onyphe.new(query, tags: tags)
+      run_analyzer onyphe
+    end
+
+    desc "urlscan [QUERY]", "urlscan lookup by a given query"
+    method_option :tags, type: :array, desc: "tags"
+    def urlscan(query)
+      tags = options.dig("tags") || []
+      urlscan = Analyzers::Urlscan.new(query, tags: tags)
+      run_analyzer urlscan
     end
 
     desc "import_from_json", "Give a JSON input via STDIN"
@@ -48,10 +50,8 @@ module Mihari
       artifacts = json.dig("artifacts")
       tags = json.dig("tags") || []
 
-      with_error_handling do
-        basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, tags: tags)
-        basic.run
-      end
+      basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, tags: tags)
+      run_analyzer basic
     end
 
     no_commands do
@@ -64,6 +64,10 @@ module Mihari
         puts e.backtrace.join('\n')
       end
 
+      def run_analyzer(analyzer)
+        with_error_handling { analyzer.run }
+      end
+
       def parse_as_json(input)
         JSON.parse input
       rescue JSON::ParserError => e

data/lib/mihari/the_hive.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "net/ping"
+require "uri"
+
 module Mihari
   class TheHive
     # @return [true, false]
@@ -14,7 +17,7 @@ module Mihari
 
     # @return [true, false]
     def valid?
-      api_endpont? && api_key?
+      api_endpont? && api_key? && ping?
     end
 
     # @return [Hachi::API]
@@ -44,5 +47,16 @@ module Mihari
         source: "mihari"
       )
     end
+
+    private
+
+    def ping?
+      base_url = ENV.fetch("THEHIVE_API_ENDPOINT")
+      base_url = base_url.end_with?("/") ? base_url[0..-2] : base_url
+      url = "#{base_url}/index.html"
+
+      http = Net::Ping::HTTP.new(url)
+      http.ping?
+    end
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "0.2.5"
+  VERSION = "0.3.0"
 end

data/mihari.gemspec

@@ -36,9 +36,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency "email_address", "~> 0.1"
   spec.add_dependency "hachi", "~> 0.1"
   spec.add_dependency "mem", "~> 0.1"
+  spec.add_dependency "net-ping"
   spec.add_dependency "onyphe", "~> 0.2"
   spec.add_dependency "public_suffix", "~> 3.1"
   spec.add_dependency "shodanx", "~> 0.1"
   spec.add_dependency "slack-incoming-webhooks", "~> 0.2"
   spec.add_dependency "thor", "~> 0.19"
+  spec.add_dependency "urlscan", "~> 0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.3.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-08 00:00:00.000000000 Z
+date: 2019-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: net-ping
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: onyphe
   requirement: !ruby/object:Gem::Requirement
@@ -234,6 +248,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: urlscan
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 description: A framework for continuous malicious hosts monitoring.
 email:
 - manabu.niseki@gmail.com
@@ -260,6 +288,7 @@ files:
 - lib/mihari/analyzers/censys.rb
 - lib/mihari/analyzers/onyphe.rb
 - lib/mihari/analyzers/shodan.rb
+- lib/mihari/analyzers/urlscan.rb
 - lib/mihari/artifact.rb
 - lib/mihari/cli.rb
 - lib/mihari/errors.rb