mihari 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +9 -20
- data/lib/mihari.rb +8 -0
- data/lib/mihari/analyzers/base.rb +16 -7
- data/lib/mihari/analyzers/basic.rb +1 -3
- data/lib/mihari/analyzers/censys.rb +5 -20
- data/lib/mihari/analyzers/circl.rb +4 -0
- data/lib/mihari/analyzers/onyphe.rb +4 -0
- data/lib/mihari/analyzers/passivetotal.rb +83 -0
- data/lib/mihari/analyzers/securitytrails.rb +26 -17
- data/lib/mihari/analyzers/securitytrails_domain_feed.rb +4 -0
- data/lib/mihari/analyzers/shodan.rb +4 -0
- data/lib/mihari/analyzers/virustotal.rb +4 -0
- data/lib/mihari/cli.rb +11 -1
- data/lib/mihari/configurable.rb +22 -0
- data/lib/mihari/emitters/base.rb +2 -0
- data/lib/mihari/emitters/misp.rb +4 -0
- data/lib/mihari/emitters/slack.rb +6 -0
- data/lib/mihari/emitters/the_hive.rb +13 -8
- data/lib/mihari/status.rb +14 -67
- data/lib/mihari/version.rb +1 -1
- data/mihari.gemspec +3 -2
- metadata +23 -7
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA256:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: 4a0fd2207c7c92a24cbeda2fb328ecca19db331aa00132b3f6d361d6f8b4b6fd
         | 
| 4 | 
            +
              data.tar.gz: 77b42d18d1509f6a33be27d38064298b726fa3244364d468714926d2878c3e89
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 14bbf6da4dffb24e0fb3c696bc664970a33784298b7fd09d24c9e792262b27a8eebd229fa1b0d17218e1116590acda49b56d4afff0bbc147c8c661dc545df811
         | 
| 7 | 
            +
              data.tar.gz: 308620e95164284b5370985899ed524705b64808a2ceb073af404e619901a0dc00277653e2e0eb41e83d5a9db4678577436fe28a78c5e8accb2534dd5362ae22
         | 
    
        data/README.md
    CHANGED
    
    | @@ -10,10 +10,10 @@ mihari(`見張り`) is a sidekick tool for [TheHive](https://github.com/TheHive- | |
| 10 10 |  | 
| 11 11 | 
             
            ## How it works
         | 
| 12 12 |  | 
| 13 | 
            -
            - mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts from the  | 
| 14 | 
            -
            - mihari checks whether  | 
| 13 | 
            +
            - mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts from the results.
         | 
| 14 | 
            +
            - mihari checks whether TheHive contains the artifacts or not.
         | 
| 15 15 | 
             
              - If it doesn't contain the artifacts:
         | 
| 16 | 
            -
                - mihari creates an alert  | 
| 16 | 
            +
                - mihari creates an alert on TheHive.
         | 
| 17 17 | 
             
                - mihari sends a notification to Slack. (Optional)
         | 
| 18 18 | 
             
                - mihari creates an event on MISP. (Optional)
         | 
| 19 19 |  | 
| @@ -51,7 +51,7 @@ docker pull ninoseki/mihari | |
| 51 51 |  | 
| 52 52 | 
             
            ## Basic usage
         | 
| 53 53 |  | 
| 54 | 
            -
            mihari supports Censys, Shodan, Onyphe, urlscan, SecurityTrails, crt.sh, CIRCL passive DNS/SSL and VirusTotal by default.
         | 
| 54 | 
            +
            mihari supports Censys, Shodan, Onyphe, urlscan, SecurityTrails, crt.sh, CIRCL passive DNS/SSL, PassiveTotal and VirusTotal by default.
         | 
| 55 55 |  | 
| 56 56 | 
             
            ```bash
         | 
| 57 57 | 
             
            $ mihari
         | 
| @@ -64,7 +64,8 @@ Commands: | |
| 64 64 | 
             
              mihari help [COMMAND]                       # Describe available commands or one specific command
         | 
| 65 65 | 
             
              mihari import_from_json                     # Give a JSON input via STDIN
         | 
| 66 66 | 
             
              mihari onyphe [QUERY]                       # Onyphe datascan lookup by a given query
         | 
| 67 | 
            -
              mihari  | 
| 67 | 
            +
              mihari passivetotal [IP|DOMAIN|EMAIL|SHA1]  # PassiveTotal lookup by a given ip / domain / email / SHA1 certificate fingerprint
         | 
| 68 | 
            +
              mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by a given ip, domain or email
         | 
| 68 69 | 
             
              mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed lookup by a given regexp
         | 
| 69 70 | 
             
              mihari shodan [QUERY]                       # Shodan host lookup by a given query
         | 
| 70 71 | 
             
              mihari status                               # Show the current configuration status
         | 
| @@ -157,9 +158,11 @@ All configuration is done via ENV variables. | |
| 157 158 | 
             
            | SLACK_CHANNEL          | Slack channel name             | Optional (default: `#general`) |
         | 
| 158 159 | 
             
            | CENSYS_ID              | Censys API ID                  | Optional                       |
         | 
| 159 160 | 
             
            | CENSYS_SECRET          | Censys secret                  | Optional                       |
         | 
| 160 | 
            -
            | CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username | Optional                       |
         | 
| 161 161 | 
             
            | CIRCL_PASSIVE_PASSWORD | CIRC_ passive DNS/SSL password | Optional                       |
         | 
| 162 | 
            +
            | CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username | Optional                       |
         | 
| 162 163 | 
             
            | ONYPHE_API_KEY         | Onyphe API key                 | Optional                       |
         | 
| 164 | 
            +
            | PASSIVETOTAL_API_KEY   | PassiveTotal API key           | Optional                       |
         | 
| 165 | 
            +
            | PASSIVETOTAL_USERNAME  | PassiveTotal username          | Optional                       |
         | 
| 163 166 | 
             
            | SECURITYTRAILS_API_KEY | SecurityTrails API key         | Optional                       |
         | 
| 164 167 | 
             
            | SHODAN_API_KEY         | Shodan API key                 | Optional                       |
         | 
| 165 168 | 
             
            | VIRUSTOTAL_API_KEY     | VirusTotal API key             | Optional                       |
         | 
| @@ -220,20 +223,6 @@ mihari caches execution results in `/tmp/mihari` and the default cache duration | |
| 220 223 |  | 
| 221 224 | 
             
            ```bash
         | 
| 222 225 | 
             
            $ docker run --rm ninoseki/mihari
         | 
| 223 | 
            -
            Commands:
         | 
| 224 | 
            -
              mihari alerts                               # Show the alerts on TheHive
         | 
| 225 | 
            -
              mihari censys [QUERY]                       # Censys IPv4 lookup by a given...
         | 
| 226 | 
            -
              mihari crtsh [QUERY]                        # crt.sh lookup by a given query
         | 
| 227 | 
            -
              mihari help [COMMAND]                       # Describe available commands o...
         | 
| 228 | 
            -
              mihari import_from_json                     # Give a JSON input via STDIN
         | 
| 229 | 
            -
              mihari onyphe [QUERY]                       # Onyphe datascan lookup by a g...
         | 
| 230 | 
            -
              mihari securitytrails [IP|DOMAIN]           # SecurityTrails resolutions lo...
         | 
| 231 | 
            -
              mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain fee...
         | 
| 232 | 
            -
              mihari shodan [QUERY]                       # Shodan host lookup by a given...
         | 
| 233 | 
            -
              mihari status                               # Show the current configuratio...
         | 
| 234 | 
            -
              mihari urlscan [QUERY]                      # urlscan lookup by a given query
         | 
| 235 | 
            -
              mihari virustotal [IP|DOMAIN]               # VirusTotal resolutions lookup...
         | 
| 236 | 
            -
             | 
| 237 226 | 
             
            # Note that you should pass configurations via environment variables
         | 
| 238 227 | 
             
            $ docker run --rm ninoseki/mihari -e THEHIVE_API_ENDPOINT="http://THEHIVE_URL" -e THEHIVE_API_KEY="API KEY" mihari
         | 
| 239 228 | 
             
            # or
         | 
    
        data/lib/mihari.rb
    CHANGED
    
    | @@ -10,6 +10,11 @@ module Mihari | |
| 10 10 | 
             
                  []
         | 
| 11 11 | 
             
                end
         | 
| 12 12 | 
             
                memoize :emitters
         | 
| 13 | 
            +
             | 
| 14 | 
            +
                def analyzers
         | 
| 15 | 
            +
                  []
         | 
| 16 | 
            +
                end
         | 
| 17 | 
            +
                memoize :analyzers
         | 
| 13 18 | 
             
              end
         | 
| 14 19 | 
             
            end
         | 
| 15 20 |  | 
| @@ -21,6 +26,8 @@ require "mihari/artifact" | |
| 21 26 | 
             
            require "mihari/cache"
         | 
| 22 27 | 
             
            require "mihari/type_checker"
         | 
| 23 28 |  | 
| 29 | 
            +
            require "mihari/configurable"
         | 
| 30 | 
            +
             | 
| 24 31 | 
             
            require "mihari/the_hive/base"
         | 
| 25 32 | 
             
            require "mihari/the_hive/alert"
         | 
| 26 33 | 
             
            require "mihari/the_hive/artifact"
         | 
| @@ -33,6 +40,7 @@ require "mihari/analyzers/circl" | |
| 33 40 | 
             
            require "mihari/analyzers/crtsh"
         | 
| 34 41 | 
             
            require "mihari/analyzers/dnpedia"
         | 
| 35 42 | 
             
            require "mihari/analyzers/onyphe"
         | 
| 43 | 
            +
            require "mihari/analyzers/passivetotal"
         | 
| 36 44 | 
             
            require "mihari/analyzers/securitytrails_domain_feed"
         | 
| 37 45 | 
             
            require "mihari/analyzers/securitytrails"
         | 
| 38 46 | 
             
            require "mihari/analyzers/shodan"
         | 
| @@ -5,10 +5,7 @@ require "parallel" | |
| 5 5 | 
             
            module Mihari
         | 
| 6 6 | 
             
              module Analyzers
         | 
| 7 7 | 
             
                class Base
         | 
| 8 | 
            -
                   | 
| 9 | 
            -
                    @the_hive = TheHive.new
         | 
| 10 | 
            -
                    @cache = Cache.new
         | 
| 11 | 
            -
                  end
         | 
| 8 | 
            +
                  include Configurable
         | 
| 12 9 |  | 
| 13 10 | 
             
                  # @return [Array<String>, Array<Mihari::Artifact>]
         | 
| 14 11 | 
             
                  def artifacts
         | 
| @@ -47,8 +44,20 @@ module Mihari | |
| 47 44 | 
             
                    puts "Emission by #{emitter.class} is failed: #{e}"
         | 
| 48 45 | 
             
                  end
         | 
| 49 46 |  | 
| 47 | 
            +
                  def self.inherited(child)
         | 
| 48 | 
            +
                    Mihari.analyzers << child
         | 
| 49 | 
            +
                  end
         | 
| 50 | 
            +
             | 
| 50 51 | 
             
                  private
         | 
| 51 52 |  | 
| 53 | 
            +
                  def the_hive
         | 
| 54 | 
            +
                    @the_hive ||= TheHive.new
         | 
| 55 | 
            +
                  end
         | 
| 56 | 
            +
             | 
| 57 | 
            +
                  def cache
         | 
| 58 | 
            +
                    @cache ||= Cache.new
         | 
| 59 | 
            +
                  end
         | 
| 60 | 
            +
             | 
| 52 61 | 
             
                  # @return [Array<Mihari::Artifact>]
         | 
| 53 62 | 
             
                  def normalized_artifacts
         | 
| 54 63 | 
             
                    @normalized_artifacts ||= artifacts.map do |artifact|
         | 
| @@ -58,15 +67,15 @@ module Mihari | |
| 58 67 |  | 
| 59 68 | 
             
                  def uncached_artifacts
         | 
| 60 69 | 
             
                    @uncached_artifacts ||= normalized_artifacts.reject do |artifact|
         | 
| 61 | 
            -
                       | 
| 70 | 
            +
                      cache.cached? artifact.data
         | 
| 62 71 | 
             
                    end
         | 
| 63 72 | 
             
                  end
         | 
| 64 73 |  | 
| 65 74 | 
             
                  # @return [Array<Mihari::Artifact>]
         | 
| 66 75 | 
             
                  def unique_artifacts
         | 
| 67 | 
            -
                    return uncached_artifacts unless  | 
| 76 | 
            +
                    return uncached_artifacts unless the_hive.valid?
         | 
| 68 77 |  | 
| 69 | 
            -
                    @unique_artifacts ||=  | 
| 78 | 
            +
                    @unique_artifacts ||= the_hive.artifact.find_non_existing_artifacts(uncached_artifacts)
         | 
| 70 79 | 
             
                  end
         | 
| 71 80 |  | 
| 72 81 | 
             
                  def set_unique_artifacts
         | 
| @@ -3,14 +3,12 @@ | |
| 3 3 | 
             
            module Mihari
         | 
| 4 4 | 
             
              module Analyzers
         | 
| 5 5 | 
             
                class Basic < Base
         | 
| 6 | 
            -
                   | 
| 6 | 
            +
                  attr_accessor :title
         | 
| 7 7 | 
             
                  attr_reader :description
         | 
| 8 8 | 
             
                  attr_reader :artifacts
         | 
| 9 9 | 
             
                  attr_reader :tags
         | 
| 10 10 |  | 
| 11 11 | 
             
                  def initialize(title:, description:, artifacts:, tags: [])
         | 
| 12 | 
            -
                    super()
         | 
| 13 | 
            -
             | 
| 14 12 | 
             
                    @title = title
         | 
| 15 13 | 
             
                    @description = description
         | 
| 16 14 | 
             
                    @artifacts = artifacts
         | 
| @@ -10,9 +10,6 @@ module Mihari | |
| 10 10 | 
             
                  attr_reader :query
         | 
| 11 11 | 
             
                  attr_reader :tags
         | 
| 12 12 |  | 
| 13 | 
            -
                  CENSYS_ID_KEY = "CENSYS_ID"
         | 
| 14 | 
            -
                  CENSYS_SECRET_KEY = "CENSYS_SECRET"
         | 
| 15 | 
            -
             | 
| 16 13 | 
             
                  def initialize(query, title: nil, description: nil, tags: [])
         | 
| 17 14 | 
             
                    super()
         | 
| 18 15 |  | 
| @@ -24,34 +21,22 @@ module Mihari | |
| 24 21 |  | 
| 25 22 | 
             
                  def artifacts
         | 
| 26 23 | 
             
                    ipv4s = []
         | 
| 24 | 
            +
             | 
| 27 25 | 
             
                    res = api.ipv4.search(query: query)
         | 
| 28 26 | 
             
                    res.each_page do |page|
         | 
| 29 | 
            -
                       | 
| 27 | 
            +
                      ipv4s << page.map(&:ip)
         | 
| 30 28 | 
             
                    end
         | 
| 31 29 |  | 
| 32 | 
            -
                    ipv4s
         | 
| 33 | 
            -
                  end
         | 
| 34 | 
            -
             | 
| 35 | 
            -
                  # @return [true, false]
         | 
| 36 | 
            -
                  def valid?
         | 
| 37 | 
            -
                    censys_id? && censys_secret?
         | 
| 30 | 
            +
                    ipv4s.flatten
         | 
| 38 31 | 
             
                  end
         | 
| 39 32 |  | 
| 40 33 | 
             
                  private
         | 
| 41 34 |  | 
| 42 | 
            -
                   | 
| 43 | 
            -
             | 
| 44 | 
            -
                    ENV.key? CENSYS_ID_KEY
         | 
| 45 | 
            -
                  end
         | 
| 46 | 
            -
             | 
| 47 | 
            -
                  # @return [true, false]
         | 
| 48 | 
            -
                  def censys_secret?
         | 
| 49 | 
            -
                    ENV.key? CENSYS_SECRET_KEY
         | 
| 35 | 
            +
                  def config_keys
         | 
| 36 | 
            +
                    %w(CENSYS_ID CENSYS_SECRET)
         | 
| 50 37 | 
             
                  end
         | 
| 51 38 |  | 
| 52 39 | 
             
                  def api
         | 
| 53 | 
            -
                    raise ArgumentError, "#{CENSYS_ID_KEY} and #{CENSYS_SECRET_KEY} are required" unless valid?
         | 
| 54 | 
            -
             | 
| 55 40 | 
             
                    @api ||= ::Censys::API.new
         | 
| 56 41 | 
             
                  end
         | 
| 57 42 | 
             
                end
         | 
| @@ -0,0 +1,83 @@ | |
| 1 | 
            +
            # frozen_string_literal: true
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            require "passivetotal"
         | 
| 4 | 
            +
             | 
| 5 | 
            +
            module Mihari
         | 
| 6 | 
            +
              module Analyzers
         | 
| 7 | 
            +
                class PassiveTotal < Base
         | 
| 8 | 
            +
                  attr_reader :query
         | 
| 9 | 
            +
                  attr_reader :type
         | 
| 10 | 
            +
             | 
| 11 | 
            +
                  attr_reader :title
         | 
| 12 | 
            +
                  attr_reader :description
         | 
| 13 | 
            +
                  attr_reader :tags
         | 
| 14 | 
            +
             | 
| 15 | 
            +
                  def initialize(query, title: nil, description: nil, tags: [])
         | 
| 16 | 
            +
                    super()
         | 
| 17 | 
            +
             | 
| 18 | 
            +
                    @query = query
         | 
| 19 | 
            +
                    @type = TypeChecker.type(query)
         | 
| 20 | 
            +
             | 
| 21 | 
            +
                    @title = title || "PassiveTotal lookup"
         | 
| 22 | 
            +
                    @description = description || "query = #{query}"
         | 
| 23 | 
            +
                    @tags = tags
         | 
| 24 | 
            +
                  end
         | 
| 25 | 
            +
             | 
| 26 | 
            +
                  def artifacts
         | 
| 27 | 
            +
                    lookup || []
         | 
| 28 | 
            +
                  end
         | 
| 29 | 
            +
             | 
| 30 | 
            +
                  private
         | 
| 31 | 
            +
             | 
| 32 | 
            +
                  def config_keys
         | 
| 33 | 
            +
                    %w(PASSIVETOTAL_USERNAME PASSIVETOTAL_API_KEY)
         | 
| 34 | 
            +
                  end
         | 
| 35 | 
            +
             | 
| 36 | 
            +
                  def api
         | 
| 37 | 
            +
                    @api ||= ::PassiveTotal::API.new
         | 
| 38 | 
            +
                  end
         | 
| 39 | 
            +
             | 
| 40 | 
            +
                  def valid_type?
         | 
| 41 | 
            +
                    %w(ip domain mail).include? type
         | 
| 42 | 
            +
                  end
         | 
| 43 | 
            +
             | 
| 44 | 
            +
                  def lookup
         | 
| 45 | 
            +
                    case type
         | 
| 46 | 
            +
                    when "domain"
         | 
| 47 | 
            +
                      passive_dns_lookup
         | 
| 48 | 
            +
                    when "ip"
         | 
| 49 | 
            +
                      passive_dns_lookup
         | 
| 50 | 
            +
                    when "mail"
         | 
| 51 | 
            +
                      reverse_whois_lookup
         | 
| 52 | 
            +
                    when "hash"
         | 
| 53 | 
            +
                      ssl_lookup
         | 
| 54 | 
            +
                    else
         | 
| 55 | 
            +
                      raise ArgumentError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
         | 
| 56 | 
            +
                    end
         | 
| 57 | 
            +
                  rescue ::PassiveTotal::Error => _e
         | 
| 58 | 
            +
                    nil
         | 
| 59 | 
            +
                  end
         | 
| 60 | 
            +
             | 
| 61 | 
            +
                  def passive_dns_lookup
         | 
| 62 | 
            +
                    res = api.dns.passive_unique(query)
         | 
| 63 | 
            +
                    res.dig("results") || []
         | 
| 64 | 
            +
                  end
         | 
| 65 | 
            +
             | 
| 66 | 
            +
                  def reverse_whois_lookup
         | 
| 67 | 
            +
                    res = api.whois.search(query: query, field: "email")
         | 
| 68 | 
            +
                    results = res.dig("results") || []
         | 
| 69 | 
            +
                    results.map do |result|
         | 
| 70 | 
            +
                      result.dig("domain")
         | 
| 71 | 
            +
                    end.flatten.compact.uniq
         | 
| 72 | 
            +
                  end
         | 
| 73 | 
            +
             | 
| 74 | 
            +
                  def ssl_lookup
         | 
| 75 | 
            +
                    res = api.ssl.history(query)
         | 
| 76 | 
            +
                    results = res.dig("results") || []
         | 
| 77 | 
            +
                    results.map do |result|
         | 
| 78 | 
            +
                      result.dig("ipAddresses")
         | 
| 79 | 
            +
                    end.flatten.compact.uniq
         | 
| 80 | 
            +
                  end
         | 
| 81 | 
            +
                end
         | 
| 82 | 
            +
              end
         | 
| 83 | 
            +
            end
         | 
| @@ -5,21 +5,21 @@ require "securitytrails" | |
| 5 5 | 
             
            module Mihari
         | 
| 6 6 | 
             
              module Analyzers
         | 
| 7 7 | 
             
                class SecurityTrails < Base
         | 
| 8 | 
            -
                  attr_reader : | 
| 8 | 
            +
                  attr_reader :query
         | 
| 9 9 | 
             
                  attr_reader :type
         | 
| 10 10 |  | 
| 11 11 | 
             
                  attr_reader :title
         | 
| 12 12 | 
             
                  attr_reader :description
         | 
| 13 13 | 
             
                  attr_reader :tags
         | 
| 14 14 |  | 
| 15 | 
            -
                  def initialize( | 
| 15 | 
            +
                  def initialize(query, title: nil, description: nil, tags: [])
         | 
| 16 16 | 
             
                    super()
         | 
| 17 17 |  | 
| 18 | 
            -
                    @ | 
| 19 | 
            -
                    @type = TypeChecker.type( | 
| 18 | 
            +
                    @query = query
         | 
| 19 | 
            +
                    @type = TypeChecker.type(query)
         | 
| 20 20 |  | 
| 21 21 | 
             
                    @title = title || "SecurityTrails lookup"
         | 
| 22 | 
            -
                    @description = description || " | 
| 22 | 
            +
                    @description = description || "query = #{query}"
         | 
| 23 23 | 
             
                    @tags = tags
         | 
| 24 24 | 
             
                  end
         | 
| 25 25 |  | 
| @@ -29,12 +29,16 @@ module Mihari | |
| 29 29 |  | 
| 30 30 | 
             
                  private
         | 
| 31 31 |  | 
| 32 | 
            +
                  def config_keys
         | 
| 33 | 
            +
                    %w(SECURITYTRAILS_API_KEY)
         | 
| 34 | 
            +
                  end
         | 
| 35 | 
            +
             | 
| 32 36 | 
             
                  def api
         | 
| 33 37 | 
             
                    @api ||= ::SecurityTrails::API.new
         | 
| 34 38 | 
             
                  end
         | 
| 35 39 |  | 
| 36 40 | 
             
                  def valid_type?
         | 
| 37 | 
            -
                    %w(ip domain).include? type
         | 
| 41 | 
            +
                    %w(ip domain mail).include? type
         | 
| 38 42 | 
             
                  end
         | 
| 39 43 |  | 
| 40 44 | 
             
                  def lookup
         | 
| @@ -43,28 +47,33 @@ module Mihari | |
| 43 47 | 
             
                      domain_lookup
         | 
| 44 48 | 
             
                    when "ip"
         | 
| 45 49 | 
             
                      ip_lookup
         | 
| 50 | 
            +
                    when "mail"
         | 
| 51 | 
            +
                      mail_lookup
         | 
| 46 52 | 
             
                    else
         | 
| 47 | 
            -
                      raise ArgumentError, "#{ | 
| 53 | 
            +
                      raise ArgumentError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
         | 
| 48 54 | 
             
                    end
         | 
| 49 55 | 
             
                  rescue ::SecurityTrails::Error => _e
         | 
| 50 56 | 
             
                    nil
         | 
| 51 57 | 
             
                  end
         | 
| 52 58 |  | 
| 53 59 | 
             
                  def domain_lookup
         | 
| 54 | 
            -
                    result = api.history.get_all_dns_history( | 
| 55 | 
            -
                    records = result. | 
| 60 | 
            +
                    result = api.history.get_all_dns_history(query, "a")
         | 
| 61 | 
            +
                    records = result.records || []
         | 
| 56 62 | 
             
                    records.map do |record|
         | 
| 57 | 
            -
                       | 
| 58 | 
            -
             | 
| 59 | 
            -
                    end.compact.flatten.uniq
         | 
| 63 | 
            +
                      (record.values || []).map(&:ip)
         | 
| 64 | 
            +
                    end.flatten.compact.uniq
         | 
| 60 65 | 
             
                  end
         | 
| 61 66 |  | 
| 62 67 | 
             
                  def ip_lookup
         | 
| 63 | 
            -
                    result = api.domains.search( filter: { ipv4:  | 
| 64 | 
            -
                    records = result. | 
| 65 | 
            -
                    records.map | 
| 66 | 
            -
             | 
| 67 | 
            -
             | 
| 68 | 
            +
                    result = api.domains.search( filter: { ipv4: query })
         | 
| 69 | 
            +
                    records = result.records || []
         | 
| 70 | 
            +
                    records.map(&:hostname).compact.uniq
         | 
| 71 | 
            +
                  end
         | 
| 72 | 
            +
             | 
| 73 | 
            +
                  def mail_lookup
         | 
| 74 | 
            +
                    result = api.domains.search( filter: { whois_email: query })
         | 
| 75 | 
            +
                    records = result.records || []
         | 
| 76 | 
            +
                    records.map(&:hostname).compact.uniq
         | 
| 68 77 | 
             
                  end
         | 
| 69 78 | 
             
                end
         | 
| 70 79 | 
             
              end
         | 
    
        data/lib/mihari/cli.rb
    CHANGED
    
    | @@ -56,7 +56,7 @@ module Mihari | |
| 56 56 | 
             
                  end
         | 
| 57 57 | 
             
                end
         | 
| 58 58 |  | 
| 59 | 
            -
                desc "securitytrails [IP|DOMAIN]", "SecurityTrails  | 
| 59 | 
            +
                desc "securitytrails [IP|DOMAIN|EMAIL]", "SecurityTrails lookup by a given ip, domain or email"
         | 
| 60 60 | 
             
                method_option :title, type: :string, desc: "title"
         | 
| 61 61 | 
             
                method_option :description, type: :string, desc: "description"
         | 
| 62 62 | 
             
                method_option :tags, type: :array, desc: "tags"
         | 
| @@ -109,6 +109,16 @@ module Mihari | |
| 109 109 | 
             
                  end
         | 
| 110 110 | 
             
                end
         | 
| 111 111 |  | 
| 112 | 
            +
                desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal lookup by a given ip / domain / email / SHA1 certificate fingerprint"
         | 
| 113 | 
            +
                method_option :title, type: :string, desc: "title"
         | 
| 114 | 
            +
                method_option :description, type: :string, desc: "description"
         | 
| 115 | 
            +
                method_option :tags, type: :array, desc: "tags"
         | 
| 116 | 
            +
                def passivetotal(query)
         | 
| 117 | 
            +
                  with_error_handling do
         | 
| 118 | 
            +
                    run_analyzer Analyzers::PassiveTotal, query: query, options: options
         | 
| 119 | 
            +
                  end
         | 
| 120 | 
            +
                end
         | 
| 121 | 
            +
             | 
| 112 122 | 
             
                desc "import_from_json", "Give a JSON input via STDIN"
         | 
| 113 123 | 
             
                def import_from_json(input = nil)
         | 
| 114 124 | 
             
                  with_error_handling do
         | 
| @@ -0,0 +1,22 @@ | |
| 1 | 
            +
            # frozen_string_literal: true
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            module Mihari
         | 
| 4 | 
            +
              module Configurable
         | 
| 5 | 
            +
                def configured?
         | 
| 6 | 
            +
                  config_keys.all? { |key| ENV.key? key }
         | 
| 7 | 
            +
                end
         | 
| 8 | 
            +
             | 
| 9 | 
            +
                def configuration_status
         | 
| 10 | 
            +
                  return nil if config_keys.empty?
         | 
| 11 | 
            +
             | 
| 12 | 
            +
                  names = config_keys.join(" and ")
         | 
| 13 | 
            +
                  be_verb = config_keys.length == 1 ? "is" : "are"
         | 
| 14 | 
            +
                  status = configured? ? "found" : "missing"
         | 
| 15 | 
            +
                  "#{names} #{be_verb} #{status}"
         | 
| 16 | 
            +
                end
         | 
| 17 | 
            +
             | 
| 18 | 
            +
                def config_keys
         | 
| 19 | 
            +
                  []
         | 
| 20 | 
            +
                end
         | 
| 21 | 
            +
              end
         | 
| 22 | 
            +
            end
         | 
    
        data/lib/mihari/emitters/base.rb
    CHANGED
    
    
    
        data/lib/mihari/emitters/misp.rb
    CHANGED
    
    
| @@ -3,13 +3,6 @@ | |
| 3 3 | 
             
            module Mihari
         | 
| 4 4 | 
             
              module Emitters
         | 
| 5 5 | 
             
                class TheHive < Base
         | 
| 6 | 
            -
                  attr_reader :the_hive
         | 
| 7 | 
            -
             | 
| 8 | 
            -
                  def initialize
         | 
| 9 | 
            -
                    @the_hive = Mihari::TheHive.new
         | 
| 10 | 
            -
                    @cache = Cache.new
         | 
| 11 | 
            -
                  end
         | 
| 12 | 
            -
             | 
| 13 6 | 
             
                  # @return [true, false]
         | 
| 14 7 | 
             
                  def valid?
         | 
| 15 8 | 
             
                    the_hive.valid?
         | 
| @@ -30,8 +23,20 @@ module Mihari | |
| 30 23 |  | 
| 31 24 | 
             
                  private
         | 
| 32 25 |  | 
| 26 | 
            +
                  def config_keys
         | 
| 27 | 
            +
                    %w(THEHIVE_API_ENDPOINT THEHIVE_API_KEY)
         | 
| 28 | 
            +
                  end
         | 
| 29 | 
            +
             | 
| 30 | 
            +
                  def the_hive
         | 
| 31 | 
            +
                    @the_hive ||= Mihari::TheHive.new
         | 
| 32 | 
            +
                  end
         | 
| 33 | 
            +
             | 
| 34 | 
            +
                  def cache
         | 
| 35 | 
            +
                    @cache ||= Cache.new
         | 
| 36 | 
            +
                  end
         | 
| 37 | 
            +
             | 
| 33 38 | 
             
                  def save_as_cache(data)
         | 
| 34 | 
            -
                     | 
| 39 | 
            +
                    cache.save data
         | 
| 35 40 | 
             
                  end
         | 
| 36 41 | 
             
                end
         | 
| 37 42 | 
             
              end
         | 
    
        data/lib/mihari/status.rb
    CHANGED
    
    | @@ -3,16 +3,7 @@ | |
| 3 3 | 
             
            module Mihari
         | 
| 4 4 | 
             
              class Status
         | 
| 5 5 | 
             
                def check
         | 
| 6 | 
            -
                   | 
| 7 | 
            -
                    censys: { status: censys?, message: censys },
         | 
| 8 | 
            -
                    misp: { status: misp?, message: misp },
         | 
| 9 | 
            -
                    onyphe: { status: onyphe?, message: onyphe },
         | 
| 10 | 
            -
                    securitytrails: { status: securitytrails?, message: securitytrails },
         | 
| 11 | 
            -
                    shodan: { status: shodan?, message: shodan },
         | 
| 12 | 
            -
                    slack: { status: slack?, message: slack },
         | 
| 13 | 
            -
                    the_hive: { status: the_hive?, message: the_hive },
         | 
| 14 | 
            -
                    virustotal: { status: virustotal?, message: virustotal },
         | 
| 15 | 
            -
                  }.map do |key, value|
         | 
| 6 | 
            +
                  statuses.map do |key, value|
         | 
| 16 7 | 
             
                    [key, convert(value)]
         | 
| 17 8 | 
             
                  end.to_h
         | 
| 18 9 | 
             
                end
         | 
| @@ -30,68 +21,24 @@ module Mihari | |
| 30 21 | 
             
                  }
         | 
| 31 22 | 
             
                end
         | 
| 32 23 |  | 
| 33 | 
            -
                def  | 
| 34 | 
            -
                   | 
| 35 | 
            -
             | 
| 36 | 
            -
             | 
| 37 | 
            -
                def securitytrails
         | 
| 38 | 
            -
                  securitytrails? ? "SECURITYTRAILS_API_KEY is found" : "SECURITYTRAILS_API_KEY is missing"
         | 
| 39 | 
            -
                end
         | 
| 40 | 
            -
             | 
| 41 | 
            -
                def virustotal?
         | 
| 42 | 
            -
                  ENV.key?("VIRUSTOTAL_API_KEY")
         | 
| 43 | 
            -
                end
         | 
| 44 | 
            -
             | 
| 45 | 
            -
                def virustotal
         | 
| 46 | 
            -
                  virustotal? ? "VIRUSTOTAL_API_KEY is found" : "VIRUSTOTAL_API_KEY is missing"
         | 
| 47 | 
            -
                end
         | 
| 48 | 
            -
             | 
| 49 | 
            -
                def onyphe?
         | 
| 50 | 
            -
                  ENV.key? "ONYPHE_API_KEY"
         | 
| 51 | 
            -
                end
         | 
| 52 | 
            -
             | 
| 53 | 
            -
                def onyphe
         | 
| 54 | 
            -
                  onyphe? ? "ONYPHE_API_KEY is found" : "ONYPHE_API_KEY is missing"
         | 
| 55 | 
            -
                end
         | 
| 56 | 
            -
             | 
| 57 | 
            -
                def censys?
         | 
| 58 | 
            -
                  ENV.key?("CENSYS_ID") && ENV.key?("CENSYS_SECRET")
         | 
| 59 | 
            -
                end
         | 
| 60 | 
            -
             | 
| 61 | 
            -
                def censys
         | 
| 62 | 
            -
                  censys? ? "CENSYS_ID and CENSYS_SECRET are found" : "CENSYS_ID and CENSYS_SECRET are missing"
         | 
| 63 | 
            -
                end
         | 
| 24 | 
            +
                def statuses
         | 
| 25 | 
            +
                  (Mihari.analyzers + Mihari.emitters).map do |klass|
         | 
| 26 | 
            +
                    name = klass.to_s.downcase.split("::").last.to_s
         | 
| 64 27 |  | 
| 65 | 
            -
             | 
| 66 | 
            -
                   | 
| 28 | 
            +
                    [name, build_status(klass)]
         | 
| 29 | 
            +
                  end.to_h.compact
         | 
| 67 30 | 
             
                end
         | 
| 68 31 |  | 
| 69 | 
            -
                def  | 
| 70 | 
            -
                   | 
| 71 | 
            -
                end
         | 
| 32 | 
            +
                def build_status(klass)
         | 
| 33 | 
            +
                  is_analyzer = klass.ancestors.include?(Mihari::Analyzers::Base)
         | 
| 72 34 |  | 
| 73 | 
            -
             | 
| 74 | 
            -
                   | 
| 75 | 
            -
             | 
| 76 | 
            -
             | 
| 77 | 
            -
                def slack
         | 
| 78 | 
            -
                  slack? ? "SLACK_WEBHOOK_URL is found" : "SLACK_WEBHOOK_URL is missing"
         | 
| 79 | 
            -
                end
         | 
| 80 | 
            -
             | 
| 81 | 
            -
                def the_hive?
         | 
| 82 | 
            -
                  ENV.key?("THEHIVE_API_ENDPOINT") && ENV.key?("THEHIVE_API_KEY")
         | 
| 83 | 
            -
                end
         | 
| 84 | 
            -
             | 
| 85 | 
            -
                def the_hive
         | 
| 86 | 
            -
                  the_hive? ? "THEHIVE_API_ENDPOINT and THEHIVE_API_KEY are found" : "THEHIVE_API_ENDPOINT and THEHIVE_API_KEY are are missing"
         | 
| 87 | 
            -
                end
         | 
| 88 | 
            -
             | 
| 89 | 
            -
                def misp?
         | 
| 90 | 
            -
                  ENV.key?("MISP_API_ENDPOINT") && ENV.key?("MISP_API_KEY")
         | 
| 91 | 
            -
                end
         | 
| 35 | 
            +
                  instance = is_analyzer ? klass.new("dummy") : klass.new
         | 
| 36 | 
            +
                  status = instance.configured?
         | 
| 37 | 
            +
                  message = instance.configuration_status
         | 
| 92 38 |  | 
| 93 | 
            -
             | 
| 94 | 
            -
             | 
| 39 | 
            +
                  message ? { status: status, message: message } : nil
         | 
| 40 | 
            +
                rescue ArgumentError => _e
         | 
| 41 | 
            +
                  nil
         | 
| 95 42 | 
             
                end
         | 
| 96 43 | 
             
              end
         | 
| 97 44 | 
             
            end
         | 
    
        data/lib/mihari/version.rb
    CHANGED
    
    
    
        data/mihari.gemspec
    CHANGED
    
    | @@ -27,7 +27,7 @@ Gem::Specification.new do |spec| | |
| 27 27 | 
             
              spec.add_development_dependency "bundler", "~> 2.0"
         | 
| 28 28 | 
             
              spec.add_development_dependency "coveralls", "~> 0.8"
         | 
| 29 29 | 
             
              spec.add_development_dependency "fakefs", "~> 0.20"
         | 
| 30 | 
            -
              spec.add_development_dependency "rake", "~>  | 
| 30 | 
            +
              spec.add_development_dependency "rake", "~> 13.0"
         | 
| 31 31 | 
             
              spec.add_development_dependency "rspec", "~> 3.8"
         | 
| 32 32 | 
             
              spec.add_development_dependency "timecop", "~> 0.9"
         | 
| 33 33 | 
             
              spec.add_development_dependency "vcr", "~> 5.0"
         | 
| @@ -44,8 +44,9 @@ Gem::Specification.new do |spec| | |
| 44 44 | 
             
              spec.add_dependency "misp", "~> 0.1"
         | 
| 45 45 | 
             
              spec.add_dependency "net-ping", "~> 2.0"
         | 
| 46 46 | 
             
              spec.add_dependency "onyphe", "~> 0.2"
         | 
| 47 | 
            -
              spec.add_dependency "parallel", "~> 1. | 
| 47 | 
            +
              spec.add_dependency "parallel", "~> 1.18"
         | 
| 48 48 | 
             
              spec.add_dependency "passive_circl", "~> 0.1"
         | 
| 49 | 
            +
              spec.add_dependency "passivetotalx", "~> 0.1"
         | 
| 49 50 | 
             
              spec.add_dependency "public_suffix", "~> 4.0"
         | 
| 50 51 | 
             
              spec.add_dependency "securitytrails", "~> 0.2"
         | 
| 51 52 | 
             
              spec.add_dependency "shodanx", "~> 0.2"
         | 
    
        metadata
    CHANGED
    
    | @@ -1,14 +1,14 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: mihari
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0. | 
| 4 | 
            +
              version: 0.11.0
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Manabu Niseki
         | 
| 8 8 | 
             
            autorequire: 
         | 
| 9 9 | 
             
            bindir: exe
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date: 2019-10- | 
| 11 | 
            +
            date: 2019-10-07 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 13 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 14 | 
             
              name: bundler
         | 
| @@ -58,14 +58,14 @@ dependencies: | |
| 58 58 | 
             
                requirements:
         | 
| 59 59 | 
             
                - - "~>"
         | 
| 60 60 | 
             
                  - !ruby/object:Gem::Version
         | 
| 61 | 
            -
                    version: ' | 
| 61 | 
            +
                    version: '13.0'
         | 
| 62 62 | 
             
              type: :development
         | 
| 63 63 | 
             
              prerelease: false
         | 
| 64 64 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 65 65 | 
             
                requirements:
         | 
| 66 66 | 
             
                - - "~>"
         | 
| 67 67 | 
             
                  - !ruby/object:Gem::Version
         | 
| 68 | 
            -
                    version: ' | 
| 68 | 
            +
                    version: '13.0'
         | 
| 69 69 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 70 70 | 
             
              name: rspec
         | 
| 71 71 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| @@ -282,14 +282,14 @@ dependencies: | |
| 282 282 | 
             
                requirements:
         | 
| 283 283 | 
             
                - - "~>"
         | 
| 284 284 | 
             
                  - !ruby/object:Gem::Version
         | 
| 285 | 
            -
                    version: '1. | 
| 285 | 
            +
                    version: '1.18'
         | 
| 286 286 | 
             
              type: :runtime
         | 
| 287 287 | 
             
              prerelease: false
         | 
| 288 288 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 289 289 | 
             
                requirements:
         | 
| 290 290 | 
             
                - - "~>"
         | 
| 291 291 | 
             
                  - !ruby/object:Gem::Version
         | 
| 292 | 
            -
                    version: '1. | 
| 292 | 
            +
                    version: '1.18'
         | 
| 293 293 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 294 294 | 
             
              name: passive_circl
         | 
| 295 295 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| @@ -304,6 +304,20 @@ dependencies: | |
| 304 304 | 
             
                - - "~>"
         | 
| 305 305 | 
             
                  - !ruby/object:Gem::Version
         | 
| 306 306 | 
             
                    version: '0.1'
         | 
| 307 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 308 | 
            +
              name: passivetotalx
         | 
| 309 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 310 | 
            +
                requirements:
         | 
| 311 | 
            +
                - - "~>"
         | 
| 312 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 313 | 
            +
                    version: '0.1'
         | 
| 314 | 
            +
              type: :runtime
         | 
| 315 | 
            +
              prerelease: false
         | 
| 316 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 317 | 
            +
                requirements:
         | 
| 318 | 
            +
                - - "~>"
         | 
| 319 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 320 | 
            +
                    version: '0.1'
         | 
| 307 321 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 308 322 | 
             
              name: public_suffix
         | 
| 309 323 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| @@ -431,6 +445,7 @@ files: | |
| 431 445 | 
             
            - lib/mihari/analyzers/crtsh.rb
         | 
| 432 446 | 
             
            - lib/mihari/analyzers/dnpedia.rb
         | 
| 433 447 | 
             
            - lib/mihari/analyzers/onyphe.rb
         | 
| 448 | 
            +
            - lib/mihari/analyzers/passivetotal.rb
         | 
| 434 449 | 
             
            - lib/mihari/analyzers/securitytrails.rb
         | 
| 435 450 | 
             
            - lib/mihari/analyzers/securitytrails_domain_feed.rb
         | 
| 436 451 | 
             
            - lib/mihari/analyzers/shodan.rb
         | 
| @@ -439,6 +454,7 @@ files: | |
| 439 454 | 
             
            - lib/mihari/artifact.rb
         | 
| 440 455 | 
             
            - lib/mihari/cache.rb
         | 
| 441 456 | 
             
            - lib/mihari/cli.rb
         | 
| 457 | 
            +
            - lib/mihari/configurable.rb
         | 
| 442 458 | 
             
            - lib/mihari/emitters/base.rb
         | 
| 443 459 | 
             
            - lib/mihari/emitters/misp.rb
         | 
| 444 460 | 
             
            - lib/mihari/emitters/slack.rb
         | 
| @@ -479,7 +495,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement | |
| 479 495 | 
             
                - !ruby/object:Gem::Version
         | 
| 480 496 | 
             
                  version: '0'
         | 
| 481 497 | 
             
            requirements: []
         | 
| 482 | 
            -
            rubygems_version: 3.0. | 
| 498 | 
            +
            rubygems_version: 3.0.6
         | 
| 483 499 | 
             
            signing_key: 
         | 
| 484 500 | 
             
            specification_version: 4
         | 
| 485 501 | 
             
            summary: A framework for continuous malicious hosts monitoring.
         |