mihari 5.1.0 → 5.1.2

data/lib/mihari/clients/zoomeye.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Clients
+    class ZoomEye < Base
+      attr_reader :api_key
+
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
+      def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {})
+        raise(ArgumentError, "'api_key' argument is required") unless api_key
+
+        headers["api-key"] = api_key
+        super(base_url, headers: headers)
+      end
+
+      #
+      # Search the Host devices
+      #
+      # @param [String] query Query string
+      # @param [Integer, nil] page The page number to paging(default:1)
+      # @param [String, nil] facets A comma-separated list of properties to get summary information on query
+      #
+      # @return [Hash]
+      #
+      def host_search(query, page: nil, facets: nil)
+        params = {
+          query: query,
+          page: page,
+          facets: facets
+        }.compact
+
+        _get("/host/search", params: params)
+      end
+
+      #
+      # Search the Web technologies
+      #
+      # @param [String] query Query string
+      # @param [Integer, nil] page The page number to paging(default:1)
+      # @param [String, nil] facets A comma-separated list of properties to get summary information on query
+      #
+      # @return [Hash]
+      #
+      def web_search(query, page: nil, facets: nil)
+        params = {
+          query: query,
+          page: page,
+          facets: facets
+        }.compact
+
+        _get("/web/search", params: params)
+      end
+
+      private
+
+      #
+      # @param [String] path
+      # @param [Hash] params
+      #
+      # @return [Hash, nil]
+      #
+      def _get(path, params: {})
+        res = get(path, params: params)
+        JSON.parse(res.body.to_s)
+      rescue HTTPError
+        nil
+      end
+    end
+  end
+end

data/lib/mihari/commands/database.rb

@@ -3,8 +3,6 @@
 module Mihari
   module Commands
     module Database
-      include Mixins::Database
-
       def self.included(thor)
         thor.class_eval do
           desc "migrate", "Migrate DB schemas"
@@ -12,14 +10,11 @@ module Mihari
           #
           # @param [String] direction
           #
-          #
           def migrate(direction = "up")
             verbose = options["verbose"]
             ActiveRecord::Migration.verbose = verbose
 
-            with_db_connection do
-              Mihari::Database.migrate(direction.to_sym)
-            end
+            Mihari::Database.with_db_connection { Mihari::Database.migrate(direction.to_sym) }
           end
         end
       end

data/lib/mihari/commands/searcher.rb

@@ -3,7 +3,6 @@
 module Mihari
   module Commands
     module Searcher
-      include Mixins::Database
       include Mixins::ErrorNotification
 
       def self.included(thor)
@@ -16,7 +15,7 @@ module Mihari
           # @param [String] path_or_id
           #
           def search(path_or_id)
-            with_db_connection do
+            Mihari::Database.with_db_connection do
               rule = Structs::Rule.from_path_or_id path_or_id
 
               # validate

data/lib/mihari/database.rb

@@ -164,6 +164,15 @@ module Mihari
 
         ActiveRecord::Base.clear_active_connections!
       end
+
+      def with_db_connection
+        Mihari::Database.connect
+        yield
+      rescue ActiveRecord::StatementInvalid
+        Mihari.logger.error("You haven't finished the DB migration! Please run 'mihari db migrate'.")
+      ensure
+        Mihari::Database.close
+      end
     end
   end
 end

data/lib/mihari/emitters/misp.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "misp"
-
 module Mihari
   module Emitters
     class MISP < Base
@@ -16,11 +14,6 @@ module Mihari
 
         @url = kwargs[:url] || Mihari.config.misp_url
         @api_key = kwargs[:api_key] || Mihari.config.misp_api_key
-
-        ::MISP.configure do |config|
-          config.api_endpoint = url
-          config.api_key = api_key
-        end
       end
 
       # @return [Boolean]
@@ -50,17 +43,13 @@ module Mihari
       def emit(rule:, artifacts:, **_options)
         return if artifacts.empty?
 
-        event = ::MISP::Event.new(info: rule.title)
-
-        artifacts.each do |artifact|
-          event.attributes << build_attribute(artifact)
-        end
-
-        rule.tags.each do |tag|
-          event.add_tag name: tag
-        end
-
-        event.create
+        client.create_event({
+          Event: {
+            info: rule.title
+          },
+          Attribute: artifacts.map { |artifact| build_attribute(artifact) },
+          Tag: rule.tags.map { |tag| { name: tag } }
+        })
       end
 
       private
@@ -69,15 +58,19 @@ module Mihari
         %w[misp_url misp_api_key]
       end
 
+      def client
+        @client ||= Clients::MISP.new(url, api_key: api_key)
+      end
+
       #
       # Build a MISP attribute
       #
       # @param [Mihari::Artifact] artifact
       #
-      # @return [::MISP::Attribute]
+      # @return [Hash]
       #
       def build_attribute(artifact)
-        ::MISP::Attribute.new(value: artifact.data, type: to_misp_type(type: artifact.data_type, value: artifact.data))
+        { value: artifact.data, type: to_misp_type(type: artifact.data_type, value: artifact.data) }
       end
 
       #

data/lib/mihari/emitters/the_hive.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "hachi"
-
 module Mihari
   module Emitters
     class TheHive < Base
@@ -50,7 +48,7 @@ module Mihari
         return if artifacts.empty?
 
         payload = payload(rule: rule, artifacts: artifacts)
-        api.alert.create(**payload)
+        client.alert(payload)
       end
 
       #
@@ -79,8 +77,8 @@ module Mihari
         %w[thehive_url thehive_api_key]
       end
 
-      def api
-        @api ||= Hachi::API.new(api_endpoint: url, api_key: api_key, api_version: normalized_api_version)
+      def client
+        @client ||= Clients::TheHive.new(url, api_key: api_key, api_version: normalized_api_version)
       end
 
       #

data/lib/mihari/emitters/webhook.rb

@@ -77,13 +77,13 @@ module Mihari
         payload_ = payload_as_string(artifacts: artifacts, rule: rule)
         payload = JSON.parse(payload_)
 
-        client = Mihari::HTTP.new(url, headers: headers, payload: payload)
+        client = Mihari::HTTP.new(url, headers: headers)
 
         case method
         when "GET"
           res = client.get
         when "POST"
-          res = client.post
+          res = client.post(json: payload)
         end
 
         res

data/lib/mihari/feed/reader.rb

@@ -6,25 +6,28 @@ require "insensitive_hash"
 module Mihari
   module Feed
     class Reader
-      attr_reader :uri, :http_request_headers, :http_request_method, :http_request_payload
+      attr_reader :url, :headers, :params, :json, :data, :method
 
-      def initialize(uri, http_request_headers: {}, http_request_method: "GET", http_request_payload_type: nil, http_request_payload: {})
-        @uri = Addressable::URI.parse(uri)
-        @http_request_headers = http_request_headers.insensitive
-        @http_request_method = http_request_method
-        @http_request_payload = http_request_payload
+      def initialize(url, headers: {}, method: "GET", params: nil, json: nil, data: nil)
+        @url = Addressable::URI.parse(url)
+        @headers = headers.insensitive
+        @method = method
 
-        http_request_headers["content-type"] = http_request_payload_type if http_request_payload_type
+        @params = params
+        @json = json
+        @data = data
+
+        headers["content-type"] = "application/json" unless json.nil?
       end
 
       def read
-        return read_file(uri.path) if uri.scheme == "file"
+        return read_file(url.path) if url.scheme == "file"
 
         res = nil
-        client = HTTP.new(uri, headers: http_request_headers, payload: http_request_payload)
+        client = HTTP.new(url, headers: headers)
 
-        res = client.get if http_request_method == "GET"
-        res = client.post if http_request_method == "POST"
+        res = client.get(params: params) if method == "GET"
+        res = client.post(params: params, json: json, data: data) if method == "POST"
 
         return [] if res.nil?
 

data/lib/mihari/http.rb

@@ -4,54 +4,62 @@ require "insensitive_hash"
 
 module Mihari
   class HTTP
-    attr_reader :url, :headers, :payload
+    # @return [String]
+    attr_reader :url
 
-    def initialize(url, headers: {}, payload: {})
+    # @return [Hash]
+    attr_reader :headers
+
+    def initialize(url, headers: {})
       @url = url.is_a?(URI) ? url : URI(url.to_s)
       @headers = headers.insensitive
-      @payload = payload
     end
 
     #
     # Make a GET request
     #
+    # @param [Hash, nil] params
+    #
     # @return [Net::HTTPResponse]
     #
-    def get
+    def get(params: nil)
       new_url = url.deep_dup
-      new_url.query = Addressable::URI.form_encode(payload) unless payload.empty?
+      new_url.query = Addressable::URI.form_encode(params) unless (params || {}).empty?
 
       get = Net::HTTP::Get.new(new_url)
       request get
     end
 
     #
-    # Make a POST request
+    # Make a POST requesti
+    #
+    # @param [Hash, nil] params
+    # @param [Hash, nil] json
+    # @param [Hash, nil] data
     #
     # @return [Net::HTTPResponse]
     #
-    def post
-      post = Net::HTTP::Post.new(url)
-
-      case content_type
-      when "application/json"
-        post.body = JSON.generate(payload)
-      when "application/x-www-form-urlencoded"
-        post.set_form_data(payload)
-      end
+    def post(params: nil, json: nil, data: nil)
+      new_url = url.deep_dup
+      new_url.query = Addressable::URI.form_encode(params) unless (params || {}).empty?
+
+      post = Net::HTTP::Post.new(new_url)
+
+      post.body = JSON.generate(json) if json
+      post.set_form_data(data) if data
 
       request post
     end
 
     class << self
-      def get(url, headers: {}, params: {})
-        client = new(url, headers: headers, payload: params)
-        client.get
+      def get(url, headers: {}, params: nil)
+        client = new(url, headers: headers)
+        client.get(params: params)
       end
 
-      def post(url, headers: {}, payload: {})
-        client = new(url, headers: headers, payload: payload)
-        client.post
+      def post(url, headers: {}, params: nil, json: nil, data: nil)
+        client = new(url, headers: headers)
+        client.post(params: params, json: json, data: data)
       end
     end
 

data/lib/mihari/mixins/retriable.rb

@@ -9,7 +9,9 @@ module Mihari
         Errno::EPIPE,
         OpenSSL::SSL::SSLError,
         Timeout::Error,
-        RetryableError
+        RetryableError,
+        NetworkError,
+        TimeoutError
       ]
 
       #

data/lib/mihari/schemas/analyzer.rb

@@ -82,10 +82,11 @@ module Mihari
       required(:analyzer).value(Types::String.enum("feed"))
       required(:query).value(:string)
       required(:selector).value(:string)
-      optional(:http_request_method).value(Types::HTTPRequestMethods).default("GET")
-      optional(:http_request_headers).value(:hash).default({})
-      optional(:http_request_payload).value(:hash).default({})
-      optional(:http_request_payload_type).value(Types::HTTPRequestPayloadTypes)
+      optional(:method).value(Types::HTTPRequestMethods).default("GET")
+      optional(:headers).value(:hash).default({})
+      optional(:params).value(:hash)
+      optional(:data).value(:hash)
+      optional(:json).value(:hash)
       optional(:options).hash(AnalyzerOptions)
     end
   end

data/lib/mihari/structs/censys.rb

@@ -4,8 +4,17 @@ module Mihari
   module Structs
     module Censys
       class AutonomousSystem < Dry::Struct
+        include Mixins::AutonomousSystem
+
         attribute :asn, Types::Int
 
+        #
+        # @return [Mihari::AutonomousSystem]
+        #
+        def to_as
+          Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -18,6 +27,20 @@ module Mihari
         attribute :country, Types::String.optional
         attribute :country_code, Types::String.optional
 
+        #
+        # @return [Mihari::Geolocation] <description>
+        #
+        def to_geolocation
+          # sometimes Censys overlooks country
+          # then set geolocation as nil
+          return nil if country.nil?
+
+          Mihari::Geolocation.new(
+            country: country,
+            country_code: country_code
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -30,6 +53,13 @@ module Mihari
       class Service < Dry::Struct
         attribute :port, Types::Integer
 
+        #
+        # @return [Mihari::Port]
+        #
+        def to_port
+          Port.new(port: port)
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -45,6 +75,29 @@ module Mihari
         attribute :metadata, Types::Hash
         attribute :services, Types.Array(Service)
 
+        #
+        # @return [Array<Mihari::Port>]
+        #
+        def to_ports
+          services.map(&:to_port)
+        end
+
+        #
+        # @param [String] source
+        #
+        # @return [Mihari::Artifact]
+        #
+        def to_artifact(source = "Censys")
+          Artifact.new(
+            data: ip,
+            source: source,
+            metadata: metadata,
+            autonomous_system: autonomous_system.to_as,
+            geolocation: location.to_geolocation,
+            ports: to_ports
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -76,6 +129,15 @@ module Mihari
         attribute :hits, Types.Array(Hit)
         attribute :links, Links
 
+        #
+        # @param [String] source
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def to_artifacts(source = "Censys")
+          hits.map { |hit| hit.to_artifact(source) }
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(

data/lib/mihari/structs/greynoise.rb

@@ -4,10 +4,29 @@ module Mihari
   module Structs
     module GreyNoise
       class Metadata < Dry::Struct
+        include Mixins::AutonomousSystem
+
         attribute :country, Types::String
         attribute :country_code, Types::String
         attribute :asn, Types::String
 
+        #
+        # @return [Mihari::AutonomousSystem]
+        #
+        def to_as
+          Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
+        end
+
+        #
+        # @return [Mihari::Geolocation]
+        #
+        def to_geolocation
+          Mihari::Geolocation.new(
+            country: country,
+            country_code: country_code
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -23,6 +42,21 @@ module Mihari
         attribute :metadata, Metadata
         attribute :metadata_, Types::Hash
 
+        #
+        # @param [String] source
+        #
+        # @return [Mihari::Artifact]
+        #
+        def to_artifact(source = "GreyNoise")
+          Mihari::Artifact.new(
+            data: ip,
+            source: source,
+            metadata: metadata_,
+            autonomous_system: metadata.to_as,
+            geolocation: metadata.to_geolocation
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -40,6 +74,15 @@ module Mihari
         attribute :message, Types::String
         attribute :query, Types::String
 
+        #
+        # @param [String] source
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def to_artifacts(source = "GreyNoise")
+          data.map { |datum| datum.to_artifact(source) }
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(

data/lib/mihari/structs/onyphe.rb

@@ -4,11 +4,47 @@ module Mihari
   module Structs
     module Onyphe
       class Result < Dry::Struct
+        include Mixins::AutonomousSystem
+
         attribute :asn, Types::String
         attribute :country_code, Types::String.optional
         attribute :ip, Types::String
         attribute :metadata, Types::Hash
 
+        #
+        # @param [String] source
+        #
+        # @return [Mihari::Artifact]
+        #
+        def to_artifact(source = "Onyphe")
+          Mihari::Artifact.new(
+            data: ip,
+            source: source,
+            metadata: metadata,
+            autonomous_system: to_as,
+            geolocation: to_geolocation
+          )
+        end
+
+        #
+        # @return [Mihari::Geolocation, nil]
+        #
+        def to_geolocation
+          return nil if country_code.nil?
+
+          Mihari::Geolocation.new(
+            country: NormalizeCountry(country_code, to: :short),
+            country_code: country_code
+          )
+        end
+
+        #
+        # @return [Mihari::AutonomousSystem]
+        #
+        def to_as
+          Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -30,6 +66,15 @@ module Mihari
         attribute :status, Types::String
         attribute :total, Types::Int
 
+        #
+        # @param [String] source
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def to_artifacts(source = "Onyphe")
+          results.map { |result| result.to_artifact(source) }
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(