mihari 1.4.1 → 2.2.0

data/lib/mihari/analyzers/virustotal.rb

@@ -5,12 +5,7 @@ require "virustotal"
 module Mihari
   module Analyzers
     class VirusTotal < Base
-      attr_reader :indicator
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :indicator, :type, :title, :description, :tags
 
       def initialize(indicator, title: nil, description: nil, tags: [])
         super()
@@ -30,7 +25,7 @@ module Mihari
       private
 
       def config_keys
-        %w(virustotal_api_key)
+        %w[virustotal_api_key]
       end
 
       def api
@@ -38,7 +33,7 @@ module Mihari
       end
 
       def valid_type?
-        %w(ip domain).include? type
+        %w[ip domain].include? type
       end
 
       def lookup
@@ -48,14 +43,14 @@ module Mihari
         when "ip"
           ip_lookup
         else
-          raise InvalidInputError, "#{indicator}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+          raise InvalidInputError, "#{indicator}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 
       def domain_lookup
         res = api.domain.resolutions(indicator)
 
-        data = res.dig("data") || []
+        data = res["data"] || []
         data.map do |item|
           item.dig("attributes", "ip_address")
         end.compact.uniq
@@ -64,7 +59,7 @@ module Mihari
       def ip_lookup
         res = api.ip_address.resolutions(indicator)
 
-        data = res.dig("data") || []
+        data = res["data"] || []
         data.map do |item|
           item.dig("attributes", "host_name")
         end.compact.uniq

data/lib/mihari/analyzers/zoomeye.rb

@@ -5,11 +5,7 @@ require "zoomeye"
 module Mihari
   module Analyzers
     class ZoomEye < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
-      attr_reader :type
+      attr_reader :title, :description, :query, :tags, :type
 
       def initialize(query, title: nil, description: nil, tags: [], type: "host")
         super()
@@ -37,11 +33,11 @@ module Mihari
       PAGE_SIZE = 10
 
       def valid_type?
-        %w(host web).include? type
+        %w[host web].include? type
       end
 
       def config_keys
-        %w(zoomeye_password zoomeye_username)
+        %w[zoomeye_password zoomeye_username]
       end
 
       def api
@@ -50,9 +46,9 @@ module Mihari
 
       def convert_responses(responses)
         responses.map do |res|
-          matches = res.dig("matches") || []
+          matches = res["matches"] || []
           matches.map do |match|
-            match.dig "ip"
+            match["ip"]
           end
         end.flatten.compact.uniq
       end
@@ -69,7 +65,7 @@ module Mihari
           res = _host_lookup(query, page: page)
           break unless res
 
-          total = res.dig("total").to_i
+          total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
         end
@@ -88,7 +84,7 @@ module Mihari
           res = _web_lookup(query, page: page)
           break unless res
 
-          total = res.dig("total").to_i
+          total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
         end

data/lib/mihari/cli.rb

@@ -1,301 +1,73 @@
 # frozen_string_literal: true
 
 require "thor"
-require "json"
+
+require "mihari/commands/binaryedge"
+require "mihari/commands/censys"
+require "mihari/commands/circl"
+require "mihari/commands/crtsh"
+require "mihari/commands/dnpedia"
+require "mihari/commands/dnstwister"
+require "mihari/commands/onyphe"
+require "mihari/commands/otx"
+require "mihari/commands/passivetotal"
+require "mihari/commands/pulsedive"
+require "mihari/commands/securitytrails_domain_feed"
+require "mihari/commands/securitytrails"
+require "mihari/commands/shodan"
+require "mihari/commands/spyse"
+require "mihari/commands/urlscan"
+require "mihari/commands/virustotal"
+require "mihari/commands/zoomeye"
+
+require "mihari/commands/free_text"
+require "mihari/commands/http_hash"
+require "mihari/commands/passive_dns"
+require "mihari/commands/passive_ssl"
+require "mihari/commands/reverse_whois"
+require "mihari/commands/ssh_fingerprint"
+
+require "mihari/commands/config"
+require "mihari/commands/json"
+require "mihari/commands/web"
 
 module Mihari
   class CLI < Thor
-    class_option :config, type: :string, desc: "path to config file"
-
-    def self.exit_on_failure?
-      true
-    end
-
-    desc "censys [QUERY]", "Censys IPv4 search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, desc: "type to search (ipv4 / websites / certificates)", default: "ipv4"
-    def censys(query)
-      with_error_handling do
-        run_analyzer Analyzers::Censys, query: query, options: options
-      end
-    end
-
-    desc "shodan [QUERY]", "Shodan host search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def shodan(query)
-      with_error_handling do
-        run_analyzer Analyzers::Shodan, query: query, options: options
-      end
-    end
-
-    desc "onyphe [QUERY]", "Onyphe datascan search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def onyphe(query)
-      with_error_handling do
-        run_analyzer Analyzers::Onyphe, query: query, options: options
-      end
-    end
-
-    desc "urlscan [QUERY]", "urlscan search by a given query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :filter, type: :string, desc: "filter for urlscan pro search"
-    method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
-    method_option :use_pro, type: :boolean, default: false, desc: "use pro search API or not"
-    method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
-    def urlscan(query)
-      with_error_handling do
-        run_analyzer Analyzers::Urlscan, query: query, options: options
-      end
-    end
-
-    desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions lookup by an ip or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def virustotal(indiactor)
-      with_error_handling do
-        run_analyzer Analyzers::VirusTotal, query: refang(indiactor), options: options
-      end
-    end
-
-    desc "securitytrails [IP|DOMAIN|EMAIL]", "SecurityTrails lookup by an ip, domain or email"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def securitytrails(indiactor)
-      with_error_handling do
-        run_analyzer Analyzers::SecurityTrails, query: refang(indiactor), options: options
-      end
-    end
-    map "st" => :securitytrails
-
-    desc "securitytrails_domain_feed [REGEXP]", "SecurityTrails new domain feed search by a regexp"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, default: "registered", desc: "A type of domain feed ('all', 'new' or 'registered')"
-    def securitytrails_domain_feed(regexp)
-      with_error_handling do
-        run_analyzer Analyzers::SecurityTrailsDomainFeed, query: regexp, options: options
-      end
-    end
-    map "st_domain_feed" => :securitytrails_domain_feed
-
-    desc "crtsh [QUERY]", "crt.sh search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :exclude_expired, type: :boolean, desc: "exclude expired certificates"
-    def crtsh(query)
-      with_error_handling do
-        run_analyzer Analyzers::Crtsh, query: query, options: options
-      end
-    end
-
-    desc "dnpedia [QUERY]", "DNPedia domain search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def dnpedia(query)
-      with_error_handling do
-        run_analyzer Analyzers::DNPedia, query: query, options: options
-      end
-    end
-
-    desc "circl [DOMAIN|SHA1]", "CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def circl(query)
-      with_error_handling do
-        run_analyzer Analyzers::CIRCL, query: refang(query), options: options
-      end
-    end
-
-    desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def passivetotal(indicator)
-      with_error_handling do
-        run_analyzer Analyzers::PassiveTotal, query: refang(indicator), options: options
-      end
-    end
-
-    desc "zoomeye [QUERY]", "ZoomEye search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, desc: "type to search(host / web)", default: "host"
-    def zoomeye(query)
-      with_error_handling do
-        run_analyzer Analyzers::ZoomEye, query: query, options: options
-      end
-    end
-
-    desc "binaryedge [QUERY]", "BinaryEdge host search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def binaryedge(query)
-      with_error_handling do
-        run_analyzer Analyzers::BinaryEdge, query: query, options: options
-      end
-    end
-
-    desc "pulsedive [IP|DOMAIN]", "Pulsedive lookup by an ip or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def pulsedive(indiactor)
-      with_error_handling do
-        run_analyzer Analyzers::Pulsedive, query: refang(indiactor), options: options
-      end
-    end
-
-    desc "dnstwister [DOMAIN]", "dnstwister lookup by a domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def dnstwister(domain)
-      with_error_handling do
-        run_analyzer Analyzers::DNSTwister, query: refang(domain), options: options
-      end
-    end
-
-    desc "otx [IP|DOMAIN]", "OTX lookup by an IP or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def otx(domain)
-      with_error_handling do
-        run_analyzer Analyzers::OTX, query: refang(domain), options: options
-      end
-    end
-
-    desc "spyse [QUERY]", "Spyse search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, desc: "type to search (ip or domain)", default: "doamin"
-    def spyse(query)
-      with_error_handling do
-        run_analyzer Analyzers::Spyse, query: query, options: options
-      end
-    end
-
-    desc "passive_dns [IP|DOMAIN]", "Cross search with passive DNS services by an ip or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def passive_dns(query)
-      with_error_handling do
-        run_analyzer Analyzers::PassiveDNS, query: refang(query), options: options
-      end
-    end
-
-    desc "passive_ssl [SHA1]", "Cross search with passive SSL services by an SHA1 certificate fingerprint"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def passive_ssl(query)
-      with_error_handling do
-        run_analyzer Analyzers::PassiveSSL, query: query, options: options
-      end
-    end
-
-    desc "reverse_whois [EMAIL]", "Cross search with reverse whois services by an email"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def reverse_whois(query)
-      with_error_handling do
-        run_analyzer Analyzers::ReveseWhois, query: refang(query), options: options
-      end
-    end
-
-    desc "http_hash", "Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :md5, type: :string, desc: "MD5 hash"
-    method_option :sha256, type: :string, desc: "SHA256 hash"
-    method_option :mmh3, type: :numeric, desc: "MurmurHash3 hash"
-    method_option :html, type: :string, desc: "path to an HTML file"
-    def http_hash
-      with_error_handling do
-        run_analyzer Analyzers::HTTPHash, query: nil, options: options
-      end
-    end
-
-    desc "free_text [TEXT]", "Cross search with search engines by a free text"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def free_text(text)
-      with_error_handling do
-        run_analyzer Analyzers::FreeText, query: text, options: options
-      end
-    end
-
-    desc "ssh_fingerprint [FINGERPRINT]", "Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def ssh_fingerprint(fingerprint)
-      with_error_handling do
-        run_analyzer Analyzers::SSHFingerprint, query: fingerprint, options: options
-      end
-    end
-
-    desc "import_from_json", "Give a JSON input via STDIN"
-    def import_from_json(input = nil)
-      with_error_handling do
-        json = input || STDIN.gets.chomp
-        raise ArgumentError, "Input not found: please give an input in a JSON format" unless json
-
-        json = parse_as_json(json)
-        raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless valid_json?(json)
-
-        title = json.dig("title")
-        description = json.dig("description")
-        artifacts = json.dig("artifacts")
-        tags = json.dig("tags") || []
-
-        basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
-        basic.run
-      end
-    end
-
-    desc "alerts", "Show the alerts on TheHive"
-    method_option :limit, type: :string, default: "5", desc: "Number of alerts to show (or 'all' to show all the alerts)"
-    method_option :title, type: :string, desc: "Title to filter"
-    method_option :source, type: :string, desc: "Source to filter"
-    method_option :tag, type: :string, desc: "Tag to filter"
-    def alerts
-      with_error_handling do
-        load_configuration
-
-        viewer = AlertViewer.new
-        alerts = viewer.list(limit: options["limit"], title: options["title"], source: options["source"], tag: options[:tag])
-        puts JSON.pretty_generate(alerts)
-      end
-    end
-
-    desc "status", "Show the current configuration status"
-    def status
-      with_error_handling do
-        load_configuration
-
-        puts JSON.pretty_generate(Status.check)
+    class_option :config, type: :string, desc: "Path to the config file"
+
+    class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not. Only affects with analyze commands."
+    class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not. Only affects with analyze commands."
+
+    include Mihari::Commands::BinaryEdge
+    include Mihari::Commands::Censys
+    include Mihari::Commands::CIRCL
+    include Mihari::Commands::Config
+    include Mihari::Commands::Crtsh
+    include Mihari::Commands::DNPedia
+    include Mihari::Commands::DNSTwister
+    include Mihari::Commands::FreeText
+    include Mihari::Commands::HTTPHash
+    include Mihari::Commands::JSON
+    include Mihari::Commands::Onyphe
+    include Mihari::Commands::OTX
+    include Mihari::Commands::PassiveDNS
+    include Mihari::Commands::PassiveSSL
+    include Mihari::Commands::PassiveTotal
+    include Mihari::Commands::Pulsedive
+    include Mihari::Commands::ReverseWhois
+    include Mihari::Commands::SecurityTrails
+    include Mihari::Commands::SecurityTrailsDomainFeed
+    include Mihari::Commands::Shodan
+    include Mihari::Commands::Spyse
+    include Mihari::Commands::SSHFingerprint
+    include Mihari::Commands::Urlscan
+    include Mihari::Commands::VirusTotal
+    include Mihari::Commands::Web
+    include Mihari::Commands::ZoomEye
+
+    class << self
+      def exit_on_failure?
+        true
       end
     end
 
@@ -307,15 +79,9 @@ module Mihari
         notifier.notify e
       end
 
-      def parse_as_json(input)
-        JSON.parse input
-      rescue JSON::ParserError => _e
-        nil
-      end
-
       # @return [true, false]
       def valid_json?(json)
-        %w(title description artifacts).all? { |key| json.key? key }
+        %w[title description artifacts].all? { |key| json.key? key }
       end
 
       def load_configuration
@@ -333,11 +99,15 @@ module Mihari
         options = normalize_options(options)
 
         analyzer = analyzer_class.new(query, **options)
+
+        analyzer.ignore_old_artifacts = options["ignore_old_artifacts"] || false
+        analyzer.ignore_threshold = options["ignore_threshold"] || 0
+
         analyzer.run
       end
 
       def symbolize_hash_keys(hash)
-        hash.map { |k, v| [k.to_sym, v] }.to_h
+        hash.transform_keys(&:to_sym)
       end
 
       def normalize_options(options)