mihari 1.3.2 → 2.0.0

data/lib/mihari/analyzers/pulsedive.rb

@@ -5,12 +5,7 @@ require "pulsedive"
 module Mihari
   module Analyzers
     class Pulsedive < Base
-      attr_reader :query
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :type, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -30,7 +25,7 @@ module Mihari
       private
 
       def config_keys
-        %w(pulsedive_api_key)
+        %w[pulsedive_api_key]
       end
 
       def api
@@ -38,18 +33,18 @@ module Mihari
       end
 
       def valid_type?
-        %w(ip domain).include? type
+        %w[ip domain].include? type
       end
 
       def lookup
-        raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         indicator = api.indicator.get_by_value(query)
-        iid = indicator.dig("iid")
+        iid = indicator["iid"]
 
         properties = api.indicator.get_properties_by_id(iid)
-        (properties.dig("dns") || []).map do |property|
-          property.dig("value") if ["A", "PTR"].include?(property.dig("name"))
+        (properties["dns"] || []).map do |property|
+          property["value"] if ["A", "PTR"].include?(property["name"])
         end.compact
       end
     end

data/lib/mihari/analyzers/reverse_whois.rb

@@ -5,16 +5,11 @@ require "parallel"
 module Mihari
   module Analyzers
     class ReveseWhois < Base
-      attr_reader :query
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :type, :title, :description, :tags
 
       ANALYZERS = [
         Mihari::Analyzers::PassiveTotal,
-        Mihari::Analyzers::SecurityTrails,
+        Mihari::Analyzers::SecurityTrails
       ].freeze
 
       def initialize(query, title: nil, description: nil, tags: [])
@@ -37,11 +32,11 @@ module Mihari
       private
 
       def valid_type?
-        %w(mail).include? type
+        %w[mail].include? type
       end
 
       def analyzers
-        raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         ANALYZERS.map do |klass|
           klass.new(query)

data/lib/mihari/analyzers/securitytrails.rb

@@ -5,12 +5,7 @@ require "securitytrails"
 module Mihari
   module Analyzers
     class SecurityTrails < Base
-      attr_reader :query
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :type, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -30,7 +25,7 @@ module Mihari
       private
 
       def config_keys
-        %w(securitytrails_api_key)
+        %w[securitytrails_api_key]
       end
 
       def api
@@ -38,7 +33,7 @@ module Mihari
       end
 
       def valid_type?
-        %w(ip domain mail).include? type
+        %w[ip domain mail].include? type
       end
 
       def lookup
@@ -50,28 +45,28 @@ module Mihari
         when "mail"
           mail_lookup
         else
-          raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 
       def domain_lookup
         result = api.history.get_all_dns_history(query, type: "a")
-        records = result.dig("records") || []
+        records = result["records"] || []
         records.map do |record|
-          (record.dig("values") || []).map { |value| value.dig("ip") }
+          (record["values"] || []).map { |value| value["ip"] }
         end.flatten.compact.uniq
       end
 
       def ip_lookup
-        result = api.domains.search( filter: { ipv4: query })
-        records = result.dig("records") || []
-        records.map { |record| record.dig("hostname") }.compact.uniq
+        result = api.domains.search(filter: {ipv4: query})
+        records = result["records"] || []
+        records.map { |record| record["hostname"] }.compact.uniq
       end
 
       def mail_lookup
-        result = api.domains.search( filter: { whois_email: query })
-        records = result.dig("records") || []
-        records.map { |record| record.dig("hostname") }.compact.uniq
+        result = api.domains.search(filter: {whois_email: query})
+        records = result["records"] || []
+        records.map { |record| record["hostname"] }.compact.uniq
       end
     end
   end

data/lib/mihari/analyzers/securitytrails_domain_feed.rb

@@ -5,11 +5,7 @@ require "securitytrails"
 module Mihari
   module Analyzers
     class SecurityTrailsDomainFeed < Base
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :type, :title, :description, :tags
 
       def initialize(regexp, type: "registered", title: nil, description: nil, tags: [])
         super()
@@ -32,7 +28,7 @@ module Mihari
       private
 
       def config_keys
-        %w(securitytrails_api_key)
+        %w[securitytrails_api_key]
       end
 
       def api
@@ -40,7 +36,7 @@ module Mihari
       end
 
       def valid_type?
-        %w(all new registered).include? type
+        %w[all new registered].include? type
       end
 
       def regexp

data/lib/mihari/analyzers/shodan.rb

@@ -5,10 +5,7 @@ require "shodan"
 module Mihari
   module Analyzers
     class Shodan < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
+      attr_reader :title, :description, :query, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -24,9 +21,9 @@ module Mihari
         return [] unless results || results.empty?
 
         results.map do |result|
-          matches = result.dig("matches") || []
+          matches = result["matches"] || []
           matches.map do |match|
-            match.dig "ip_str"
+            match["ip_str"]
           end.compact
         end.flatten.compact.uniq
       end
@@ -36,7 +33,7 @@ module Mihari
       PAGE_SIZE = 100
 
       def config_keys
-        %w(shodan_api_key)
+        %w[shodan_api_key]
       end
 
       def api
@@ -58,7 +55,11 @@ module Mihari
           break unless res
 
           responses << res
-          break if res.dig("total").to_i <= page * PAGE_SIZE
+          break if res["total"].to_i <= page * PAGE_SIZE
+        rescue JSON::ParserError
+          # ignore JSON::ParserError
+          # ref. https://github.com/ninoseki/mihari/issues/197
+          next
         end
         responses
       end

data/lib/mihari/analyzers/spyse.rb

@@ -6,12 +6,7 @@ require "json"
 module Mihari
   module Analyzers
     class Spyse < Base
-      attr_reader :query
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :type, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [], type: "domain")
         super()
@@ -35,7 +30,7 @@ module Mihari
       end
 
       def config_keys
-        %w(spyse_api_key)
+        %w[spyse_api_key]
       end
 
       def api
@@ -43,14 +38,14 @@ module Mihari
       end
 
       def valid_type?
-        %w(ip domain cert).include? type
+        %w[ip domain cert].include? type
       end
 
       def domain_lookup
         res = api.domain.search(search_params, limit: 100)
         items = res.dig("data", "items") || []
         items.map do |item|
-          item.dig("name")
+          item["name"]
         end.uniq.compact
       end
 
@@ -58,7 +53,7 @@ module Mihari
         res = api.ip.search(search_params, limit: 100)
         items = res.dig("data", "items") || []
         items.map do |item|
-          item.dig("ip")
+          item["ip"]
         end.uniq.compact
       end
 
@@ -69,7 +64,7 @@ module Mihari
         when "ip"
           ip_lookup
         else
-          raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
     end

data/lib/mihari/analyzers/ssh_fingerprint.rb

@@ -5,11 +5,7 @@ require "parallel"
 module Mihari
   module Analyzers
     class SSHFingerprint < Base
-      attr_reader :fingerprint
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :fingerprint, :title, :description, :tags
 
       def initialize(fingerprint, title: nil, description: nil, tags: [])
         super()
@@ -46,7 +42,7 @@ module Mihari
 
         [
           binary_edge,
-          shodan,
+          shodan
         ].compact
       end
 

data/lib/mihari/analyzers/urlscan.rb

@@ -5,20 +5,29 @@ require "urlscan"
 module Mihari
   module Analyzers
     class Urlscan < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
-      attr_reader :target_type
+      attr_reader :title, :description, :query, :tags, :filter, :target_type, :use_pro, :use_similarity
 
-      def initialize(query, title: nil, description: nil, tags: [], target_type: "url")
+      def initialize(
+        query,
+        description: nil,
+        filter: nil,
+        tags: [],
+        target_type: "url",
+        title: nil,
+        use_pro: false,
+        use_similarity: false
+      )
         super()
 
         @query = query
         @title = title || "urlscan lookup"
         @description = description || "query = #{query}"
         @tags = tags
+
+        @filter = filter
         @target_type = target_type
+        @use_pro = use_pro
+        @use_similarity = use_similarity
 
         raise InvalidInputError, "type should be url, domain or ip." unless valid_target_type?
       end
@@ -27,7 +36,7 @@ module Mihari
         result = search
         return [] unless result
 
-        results = result.dig("results") || []
+        results = result["results"] || []
         results.map do |match|
           match.dig "page", target_type
         end.compact.uniq
@@ -36,7 +45,7 @@ module Mihari
       private
 
       def config_keys
-        %w(urlscan_api_key)
+        %w[urlscan_api_key]
       end
 
       def api
@@ -44,11 +53,14 @@ module Mihari
       end
 
       def search
+        return api.pro.similar(query) if use_similarity
+        return api.pro.search(query: query, filter: filter, size: 10_000) if use_pro
+
         api.search(query, size: 10_000)
       end
 
       def valid_target_type?
-        %w(url domain ip).include? target_type
+        %w[url domain ip].include? target_type
       end
     end
   end

data/lib/mihari/analyzers/virustotal.rb

@@ -5,12 +5,7 @@ require "virustotal"
 module Mihari
   module Analyzers
     class VirusTotal < Base
-      attr_reader :indicator
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :indicator, :type, :title, :description, :tags
 
       def initialize(indicator, title: nil, description: nil, tags: [])
         super()
@@ -30,7 +25,7 @@ module Mihari
       private
 
       def config_keys
-        %w(virustotal_api_key)
+        %w[virustotal_api_key]
       end
 
       def api
@@ -38,7 +33,7 @@ module Mihari
       end
 
       def valid_type?
-        %w(ip domain).include? type
+        %w[ip domain].include? type
       end
 
       def lookup
@@ -48,14 +43,14 @@ module Mihari
         when "ip"
           ip_lookup
         else
-          raise InvalidInputError, "#{indicator}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+          raise InvalidInputError, "#{indicator}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end
 
       def domain_lookup
         res = api.domain.resolutions(indicator)
 
-        data = res.dig("data") || []
+        data = res["data"] || []
         data.map do |item|
           item.dig("attributes", "ip_address")
         end.compact.uniq
@@ -64,7 +59,7 @@ module Mihari
       def ip_lookup
         res = api.ip_address.resolutions(indicator)
 
-        data = res.dig("data") || []
+        data = res["data"] || []
         data.map do |item|
           item.dig("attributes", "host_name")
         end.compact.uniq

data/lib/mihari/analyzers/zoomeye.rb

@@ -5,11 +5,7 @@ require "zoomeye"
 module Mihari
   module Analyzers
     class ZoomEye < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
-      attr_reader :type
+      attr_reader :title, :description, :query, :tags, :type
 
       def initialize(query, title: nil, description: nil, tags: [], type: "host")
         super()
@@ -37,11 +33,11 @@ module Mihari
       PAGE_SIZE = 10
 
       def valid_type?
-        %w(host web).include? type
+        %w[host web].include? type
       end
 
       def config_keys
-        %w(zoomeye_password zoomeye_username)
+        %w[zoomeye_password zoomeye_username]
       end
 
       def api
@@ -50,9 +46,9 @@ module Mihari
 
       def convert_responses(responses)
         responses.map do |res|
-          matches = res.dig("matches") || []
+          matches = res["matches"] || []
           matches.map do |match|
-            match.dig "ip"
+            match["ip"]
           end
         end.flatten.compact.uniq
       end
@@ -69,7 +65,7 @@ module Mihari
           res = _host_lookup(query, page: page)
           break unless res
 
-          total = res.dig("total").to_i
+          total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
         end
@@ -88,7 +84,7 @@ module Mihari
           res = _web_lookup(query, page: page)
           break unless res
 
-          total = res.dig("total").to_i
+          total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
         end