mihari 5.1.1 → 5.1.3

data/lib/mihari/clients/securitytrails.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Clients
+    class SecurityTrails < Base
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
+      def initialize(base_url = "https://api.securitytrails.com", api_key:, headers: {})
+        raise(ArgumentError, "'api_key' argument is required") unless api_key
+
+        headers["apikey"] = api_key
+        super(base_url, headers: headers)
+      end
+
+      #
+      # @param [String] mail
+      #
+      # @return [Array<Hash>]
+      #
+      def search_by_mail(mail)
+        res = _post "/v1/domains/list", json: { filter: { whois_email: mail } }
+        res["records"] || []
+      end
+
+      #
+      # @param [String] ip
+      #
+      # @return [Array<Hash>]
+      #
+      def search_by_ip(ip)
+        res = _post "/v1/domains/list", json: { filter: { ipv4: ip } }
+        res["records"] || []
+      end
+
+      #
+      # @param [String] domain
+      # @param [String] type
+      #
+      # @return [Array<Hash>]
+      #
+      def get_all_dns_history(domain, type:)
+        first_page = get_dns_history(domain, type: type, page: 1)
+
+        pages = first_page["pages"].to_i
+        records = first_page["records"] || []
+
+        (2..pages).each do |page_idx|
+          next_page = get_dns_history(domain, type: type, page: page_idx)
+          records << next_page["records"]
+        end
+
+        records.flatten
+      end
+
+      private
+
+      #
+      # @param [String] domain
+      # @param [String] type
+      # @param [Integer] page
+      #
+      # @return [Array<Hash>]
+      #
+      def get_dns_history(domain, type:, page:)
+        _get "/v1/history/#{domain}/dns/#{type}", params: { page: page }
+      end
+
+      #
+      # @param [String] path
+      # @param [Hash, nil] params
+      #
+      # @return [Hash]
+      #
+      def _get(path, params:)
+        res = get(path, params: params)
+        JSON.parse(res.body.to_s)
+      end
+
+      #
+      # @param [String] path
+      # @param [Hash, nil] json
+      #
+      # @return [Hash]
+      #
+      def _post(path, json:)
+        res = post(path, json: json)
+        JSON.parse(res.body.to_s)
+      end
+    end
+  end
+end

data/lib/mihari/clients/shodan.rb

@@ -3,8 +3,14 @@
 module Mihari
   module Clients
     class Shodan < Base
+      # @return [String]
       attr_reader :api_key
 
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
       def initialize(base_url = "https://api.shodan.io", api_key:, headers: {})
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
@@ -13,8 +19,13 @@ module Mihari
         @api_key = api_key
       end
 
-      # Search Shodan using the same query syntax as the website and use facets
-      # to get summary information for different properties.
+      #
+      # @param [String] query
+      # @param [Integer] page
+      # @param [Boolean] minify
+      #
+      # @return [Structs::Shodan::Result]
+      #
       def search(query, page: 1, minify: true)
         params = {
           query: query,
@@ -23,7 +34,7 @@ module Mihari
           key: api_key
         }
         res = get("/shodan/host/search", params: params)
-        JSON.parse(res.body.to_s)
+        Structs::Shodan::Result.from_dynamic! JSON.parse(res.body.to_s)
       end
     end
   end

data/lib/mihari/clients/the_hive.rb

@@ -5,7 +5,7 @@ module Mihari
     class TheHive < Base
       #
       # @param [String] base_url
-      # @param [String] api_key
+      # @param [String, nil] api_key
       # @param [String, nil] api_version
       # @param [Hash] headers
       #
@@ -18,6 +18,11 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # @param [Hash] json
+      #
+      # @return [Hash]
+      #
       def alert(json)
         json = json.to_camelback_keys.compact
         res = post("/alert", json: json)

data/lib/mihari/clients/urlscan.rb

@@ -5,7 +5,7 @@ module Mihari
     class UrlScan < Base
       #
       # @param [String] base_url
-      # @param [String] api_key
+      # @param [String, nil] api_key
       # @param [Hash] headers
       #
       def initialize(base_url = "https://urlscan.io", api_key:, headers: {})
@@ -21,6 +21,8 @@ module Mihari
       # @param [Integer] size
       # @param [String, nil] search_after
       #
+      # @return [Hash]
+      #
       def search(q, size: 100, search_after: nil)
         params = { q: q, size: size, search_after: search_after }.compact
         res = get("/api/v1/search/", params: params)

data/lib/mihari/clients/virustotal.rb

@@ -5,8 +5,7 @@ module Mihari
     class VirusTotal < Base
       #
       # @param [String] base_url
-      # @param [String] id
-      # @param [String] secret
+      # @param [String, nil] api_key
       # @param [Hash] headers
       #
       def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {})
@@ -20,6 +19,8 @@ module Mihari
       #
       # @param [String] query
       #
+      # @return [Hash]
+      #
       def domain_search(query)
         _get("/api/v3/domains/#{query}/resolutions")
       end
@@ -27,6 +28,8 @@ module Mihari
       #
       # @param [String] query
       #
+      # @return [Hash]
+      #
       def ip_search(query)
         _get("/api/v3/ip_addresses/#{query}/resolutions")
       end
@@ -35,6 +38,8 @@ module Mihari
       # @param [String] query
       # @param [String, nil] cursor
       #
+      # @return [Hash]
+      #
       def intel_search(query, cursor: nil)
         params = { query: query, cursor: cursor }.compact
         _get("/api/v3/intelligence/search", params: params)
@@ -42,11 +47,12 @@ module Mihari
 
       private
 
-      #
       #
       # @param [String] path
       # @param [Hash] params
       #
+      # @return [Hash]
+      #
       def _get(path, params: {})
         res = get(path, params: params)
         JSON.parse(res.body.to_s)

data/lib/mihari/clients/zoomeye.rb

@@ -5,6 +5,11 @@ module Mihari
     class ZoomEye < Base
       attr_reader :api_key
 
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
       def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {})
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
@@ -52,11 +57,12 @@ module Mihari
 
       private
 
-      #
       #
       # @param [String] path
       # @param [Hash] params
       #
+      # @return [Hash, nil]
+      #
       def _get(path, params: {})
         res = get(path, params: params)
         JSON.parse(res.body.to_s)

data/lib/mihari/commands/database.rb

@@ -3,8 +3,6 @@
 module Mihari
   module Commands
     module Database
-      include Mixins::Database
-
       def self.included(thor)
         thor.class_eval do
           desc "migrate", "Migrate DB schemas"
@@ -12,14 +10,11 @@ module Mihari
           #
           # @param [String] direction
           #
-          #
           def migrate(direction = "up")
             verbose = options["verbose"]
             ActiveRecord::Migration.verbose = verbose
 
-            with_db_connection do
-              Mihari::Database.migrate(direction.to_sym)
-            end
+            Mihari::Database.with_db_connection { Mihari::Database.migrate(direction.to_sym) }
           end
         end
       end

data/lib/mihari/commands/searcher.rb

@@ -3,7 +3,6 @@
 module Mihari
   module Commands
     module Searcher
-      include Mixins::Database
       include Mixins::ErrorNotification
 
       def self.included(thor)
@@ -16,7 +15,7 @@ module Mihari
           # @param [String] path_or_id
           #
           def search(path_or_id)
-            with_db_connection do
+            Mihari::Database.with_db_connection do
               rule = Structs::Rule.from_path_or_id path_or_id
 
               # validate

data/lib/mihari/database.rb

@@ -164,6 +164,15 @@ module Mihari
 
         ActiveRecord::Base.clear_active_connections!
       end
+
+      def with_db_connection
+        Mihari::Database.connect
+        yield
+      rescue ActiveRecord::StatementInvalid
+        Mihari.logger.error("You haven't finished the DB migration! Please run 'mihari db migrate'.")
+      ensure
+        Mihari::Database.close
+      end
     end
   end
 end

data/lib/mihari/http.rb

@@ -4,7 +4,7 @@ require "insensitive_hash"
 
 module Mihari
   class HTTP
-    # @return [String]
+    # @return [URI]
     attr_reader :url
 
     # @return [Hash]
@@ -26,12 +26,12 @@ module Mihari
       new_url = url.deep_dup
       new_url.query = Addressable::URI.form_encode(params) unless (params || {}).empty?
 
-      get = Net::HTTP::Get.new(new_url)
+      get = Net::HTTP::Get.new(new_url, headers)
       request get
     end
 
     #
-    # Make a POST requesti
+    # Make a POST request
     #
     # @param [Hash, nil] params
     # @param [Hash, nil] json
@@ -43,10 +43,17 @@ module Mihari
       new_url = url.deep_dup
       new_url.query = Addressable::URI.form_encode(params) unless (params || {}).empty?
 
-      post = Net::HTTP::Post.new(new_url)
+      post = Net::HTTP::Post.new(new_url, headers)
 
-      post.body = JSON.generate(json) if json
-      post.set_form_data(data) if data
+      if json
+        post.body = JSON.generate(json) if json
+        post.content_type = "application/json"
+      end
+
+      if data
+        post.set_form_data(data) if data
+        post.content_type = "application/x-www-form-urlencoded"
+      end
 
       request post
     end
@@ -65,10 +72,6 @@ module Mihari
 
     private
 
-    def content_type
-      headers["content-type"] || "application/json"
-    end
-
     #
     # Get options for HTTP request
     #
@@ -89,16 +92,9 @@ module Mihari
     #
     def request(req)
       Net::HTTP.start(url.host, url.port, https_options) do |http|
-        # set headers
-        headers.each do |k, v|
-          req[k] = v
-        end
-
         res = http.request(req)
-
         unless res.is_a?(Net::HTTPSuccess)
-          code = res.code.to_i
-          raise UnsuccessfulStatusCodeError, "Unsuccessful response code returned: #{code}"
+          raise UnsuccessfulStatusCodeError, "Unsuccessful response code returned: #{res.code}"
         end
 
         res

data/lib/mihari/structs/censys.rb

@@ -4,8 +4,17 @@ module Mihari
   module Structs
     module Censys
       class AutonomousSystem < Dry::Struct
+        include Mixins::AutonomousSystem
+
         attribute :asn, Types::Int
 
+        #
+        # @return [Mihari::AutonomousSystem]
+        #
+        def to_as
+          Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -18,6 +27,20 @@ module Mihari
         attribute :country, Types::String.optional
         attribute :country_code, Types::String.optional
 
+        #
+        # @return [Mihari::Geolocation] <description>
+        #
+        def to_geolocation
+          # sometimes Censys overlooks country
+          # then set geolocation as nil
+          return nil if country.nil?
+
+          Mihari::Geolocation.new(
+            country: country,
+            country_code: country_code
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -30,6 +53,13 @@ module Mihari
       class Service < Dry::Struct
         attribute :port, Types::Integer
 
+        #
+        # @return [Mihari::Port]
+        #
+        def to_port
+          Port.new(port: port)
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -45,6 +75,29 @@ module Mihari
         attribute :metadata, Types::Hash
         attribute :services, Types.Array(Service)
 
+        #
+        # @return [Array<Mihari::Port>]
+        #
+        def to_ports
+          services.map(&:to_port)
+        end
+
+        #
+        # @param [String] source
+        #
+        # @return [Mihari::Artifact]
+        #
+        def to_artifact(source = "Censys")
+          Artifact.new(
+            data: ip,
+            source: source,
+            metadata: metadata,
+            autonomous_system: autonomous_system.to_as,
+            geolocation: location.to_geolocation,
+            ports: to_ports
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -76,6 +129,15 @@ module Mihari
         attribute :hits, Types.Array(Hit)
         attribute :links, Links
 
+        #
+        # @param [String] source
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def to_artifacts(source = "Censys")
+          hits.map { |hit| hit.to_artifact(source) }
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(

data/lib/mihari/structs/greynoise.rb

@@ -4,10 +4,29 @@ module Mihari
   module Structs
     module GreyNoise
       class Metadata < Dry::Struct
+        include Mixins::AutonomousSystem
+
         attribute :country, Types::String
         attribute :country_code, Types::String
         attribute :asn, Types::String
 
+        #
+        # @return [Mihari::AutonomousSystem]
+        #
+        def to_as
+          Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
+        end
+
+        #
+        # @return [Mihari::Geolocation]
+        #
+        def to_geolocation
+          Mihari::Geolocation.new(
+            country: country,
+            country_code: country_code
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -23,6 +42,21 @@ module Mihari
         attribute :metadata, Metadata
         attribute :metadata_, Types::Hash
 
+        #
+        # @param [String] source
+        #
+        # @return [Mihari::Artifact]
+        #
+        def to_artifact(source = "GreyNoise")
+          Mihari::Artifact.new(
+            data: ip,
+            source: source,
+            metadata: metadata_,
+            autonomous_system: metadata.to_as,
+            geolocation: metadata.to_geolocation
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -40,6 +74,15 @@ module Mihari
         attribute :message, Types::String
         attribute :query, Types::String
 
+        #
+        # @param [String] source
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def to_artifacts(source = "GreyNoise")
+          data.map { |datum| datum.to_artifact(source) }
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(

data/lib/mihari/structs/onyphe.rb

@@ -4,11 +4,47 @@ module Mihari
   module Structs
     module Onyphe
       class Result < Dry::Struct
+        include Mixins::AutonomousSystem
+
         attribute :asn, Types::String
         attribute :country_code, Types::String.optional
         attribute :ip, Types::String
         attribute :metadata, Types::Hash
 
+        #
+        # @param [String] source
+        #
+        # @return [Mihari::Artifact]
+        #
+        def to_artifact(source = "Onyphe")
+          Mihari::Artifact.new(
+            data: ip,
+            source: source,
+            metadata: metadata,
+            autonomous_system: to_as,
+            geolocation: to_geolocation
+          )
+        end
+
+        #
+        # @return [Mihari::Geolocation, nil]
+        #
+        def to_geolocation
+          return nil if country_code.nil?
+
+          Mihari::Geolocation.new(
+            country: NormalizeCountry(country_code, to: :short),
+            country_code: country_code
+          )
+        end
+
+        #
+        # @return [Mihari::AutonomousSystem]
+        #
+        def to_as
+          Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -30,6 +66,15 @@ module Mihari
         attribute :status, Types::String
         attribute :total, Types::Int
 
+        #
+        # @param [String] source
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def to_artifacts(source = "Onyphe")
+          results.map { |result| result.to_artifact(source) }
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(