mihari 5.0.1 → 5.1.1

data/lib/mihari/clients/zoomeye.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Clients
+    class ZoomEye < Base
+      attr_reader :api_key
+
+      def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {})
+        raise(ArgumentError, "'api_key' argument is required") unless api_key
+
+        headers["api-key"] = api_key
+        super(base_url, headers: headers)
+      end
+
+      #
+      # Search the Host devices
+      #
+      # @param [String] query Query string
+      # @param [Integer, nil] page The page number to paging(default:1)
+      # @param [String, nil] facets A comma-separated list of properties to get summary information on query
+      #
+      # @return [Hash]
+      #
+      def host_search(query, page: nil, facets: nil)
+        params = {
+          query: query,
+          page: page,
+          facets: facets
+        }.compact
+
+        _get("/host/search", params: params)
+      end
+
+      #
+      # Search the Web technologies
+      #
+      # @param [String] query Query string
+      # @param [Integer, nil] page The page number to paging(default:1)
+      # @param [String, nil] facets A comma-separated list of properties to get summary information on query
+      #
+      # @return [Hash]
+      #
+      def web_search(query, page: nil, facets: nil)
+        params = {
+          query: query,
+          page: page,
+          facets: facets
+        }.compact
+
+        _get("/web/search", params: params)
+      end
+
+      private
+
+      #
+      #
+      # @param [String] path
+      # @param [Hash] params
+      #
+      def _get(path, params: {})
+        res = get(path, params: params)
+        JSON.parse(res.body.to_s)
+      rescue HTTPError
+        nil
+      end
+    end
+  end
+end

data/lib/mihari/commands/database.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Database
+      include Mixins::Database
+
+      def self.included(thor)
+        thor.class_eval do
+          desc "migrate", "Migrate DB schemas"
+          method_option :verbose, type: :boolean, default: true
+          #
+          # @param [String] direction
+          #
+          #
+          def migrate(direction = "up")
+            verbose = options["verbose"]
+            ActiveRecord::Migration.verbose = verbose
+
+            with_db_connection do
+              Mihari::Database.migrate(direction.to_sym)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/{initializer.rb → rule.rb}

@@ -4,20 +4,41 @@ require "pathname"
 
 module Mihari
   module Commands
-    module Initializer
+    module Rule
       def self.included(thor)
         thor.class_eval do
-          desc "init", "Initialize a new rule"
-          method_option :path, type: :string, default: "./rule.yml"
-          def init
-            path = options["path"]
+          desc "validate [PATH]", "Validate a rule file"
+          #
+          # Validate format of a rule
+          #
+          # @param [String] path
+          #
+          def validate(path)
+            rule = Structs::Rule.from_path_or_id(path)
 
+            begin
+              rule.validate!
+              Mihari.logger.info "Valid format. The input is parsed as the following:"
+              Mihari.logger.info rule.data.to_yaml
+            rescue RuleValidationError
+              nil
+            end
+          end
+
+          desc "init [PATH]", "Initialize a new rule file"
+          #
+          # Initialize a new rule file
+          #
+          # @param [String] path
+          #
+          #
+          def init(path = "./rule.yml")
             warning = "#{path} exists. Do you want to overwrite it? (y/n)"
             return if Pathname(path).exist? && !(yes? warning)
 
             initialize_rule path
 
-            Mihari.logger.info "A new rule is initialized as #{path}."
+            Mihari.logger.info "A new rule is initialized: #{path}."
           end
 
           no_commands do

data/lib/mihari/commands/searcher.rb

@@ -10,6 +10,11 @@ module Mihari
         thor.class_eval do
           desc "search [PATH]", "Search by a rule"
           method_option :force_overwrite, type: :boolean, aliases: "-f", desc: "Force an overwrite the rule"
+          #
+          # Search by a rule
+          #
+          # @param [String] path_or_id
+          #
           def search(path_or_id)
             with_db_connection do
               rule = Structs::Rule.from_path_or_id path_or_id

data/lib/mihari/database.rb

@@ -9,10 +9,6 @@ def env
   ENV["APP_ENV"] || ENV["RACK_ENV"]
 end
 
-def test_env?
-  env == "test"
-end
-
 def development_env?
   env == "development"
 end
@@ -121,22 +117,24 @@ def adapter
   "sqlite3"
 end
 
+#
+# @return [Array<ActiveRecord::Migration>] schemas
+#
+def schemas
+  [V5Schema]
+end
+
 module Mihari
   class Database
     class << self
-      include Memist::Memoizable
-
       #
       # DB migraration
       #
       # @param [Symbol] direction
       #
       def migrate(direction)
-        ActiveRecord::Migration.verbose = false
-
-        [V5Schema].each { |schema| schema.migrate direction }
+        schemas.each { |schema| schema.migrate direction }
       end
-      memoize :migrate unless test_env?
 
       #
       # Establish DB connection
@@ -153,10 +151,7 @@ module Mihari
             database: Mihari.config.database_url.path[1..]
           )
         end
-
         ActiveRecord::Base.logger = Logger.new($stdout) if development_env?
-
-        migrate :up
       rescue StandardError => e
         Mihari.logger.error e
       end
@@ -169,15 +164,6 @@ module Mihari
 
         ActiveRecord::Base.clear_active_connections!
       end
-
-      #
-      # Destory DB
-      #
-      def destroy!
-        return unless ActiveRecord::Base.connected?
-
-        migrate :down
-      end
     end
   end
 end

data/lib/mihari/emitters/misp.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "misp"
-
 module Mihari
   module Emitters
     class MISP < Base
@@ -16,11 +14,6 @@ module Mihari
 
         @url = kwargs[:url] || Mihari.config.misp_url
         @api_key = kwargs[:api_key] || Mihari.config.misp_api_key
-
-        ::MISP.configure do |config|
-          config.api_endpoint = url
-          config.api_key = api_key
-        end
       end
 
       # @return [Boolean]
@@ -50,17 +43,13 @@ module Mihari
       def emit(rule:, artifacts:, **_options)
         return if artifacts.empty?
 
-        event = ::MISP::Event.new(info: rule.title)
-
-        artifacts.each do |artifact|
-          event.attributes << build_attribute(artifact)
-        end
-
-        rule.tags.each do |tag|
-          event.add_tag name: tag
-        end
-
-        event.create
+        client.create_event({
+          Event: {
+            info: rule.title
+          },
+          Attribute: artifacts.map { |artifact| build_attribute(artifact) },
+          Tag: rule.tags.map { |tag| { name: tag } }
+        })
       end
 
       private
@@ -69,15 +58,19 @@ module Mihari
         %w[misp_url misp_api_key]
       end
 
+      def client
+        @client ||= Clients::MISP.new(url, api_key: api_key)
+      end
+
       #
       # Build a MISP attribute
       #
       # @param [Mihari::Artifact] artifact
       #
-      # @return [::MISP::Attribute]
+      # @return [Hash]
       #
       def build_attribute(artifact)
-        ::MISP::Attribute.new(value: artifact.data, type: to_misp_type(type: artifact.data_type, value: artifact.data))
+        { value: artifact.data, type: to_misp_type(type: artifact.data_type, value: artifact.data) }
       end
 
       #

data/lib/mihari/emitters/the_hive.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "hachi"
-
 module Mihari
   module Emitters
     class TheHive < Base
@@ -50,7 +48,7 @@ module Mihari
         return if artifacts.empty?
 
         payload = payload(rule: rule, artifacts: artifacts)
-        api.alert.create(**payload)
+        client.alert(payload)
       end
 
       #
@@ -79,8 +77,8 @@ module Mihari
         %w[thehive_url thehive_api_key]
       end
 
-      def api
-        @api ||= Hachi::API.new(api_endpoint: url, api_key: api_key, api_version: normalized_api_version)
+      def client
+        @client ||= Clients::TheHive.new(url, api_key: api_key, api_version: normalized_api_version)
       end
 
       #

data/lib/mihari/emitters/webhook.rb

@@ -77,13 +77,13 @@ module Mihari
         payload_ = payload_as_string(artifacts: artifacts, rule: rule)
         payload = JSON.parse(payload_)
 
-        client = Mihari::HTTP.new(url, headers: headers, payload: payload)
+        client = Mihari::HTTP.new(url, headers: headers)
 
         case method
         when "GET"
           res = client.get
         when "POST"
-          res = client.post
+          res = client.post(json: payload)
         end
 
         res

data/lib/mihari/feed/reader.rb

@@ -6,25 +6,28 @@ require "insensitive_hash"
 module Mihari
   module Feed
     class Reader
-      attr_reader :uri, :http_request_headers, :http_request_method, :http_request_payload
+      attr_reader :url, :headers, :params, :json, :data, :method
 
-      def initialize(uri, http_request_headers: {}, http_request_method: "GET", http_request_payload_type: nil, http_request_payload: {})
-        @uri = Addressable::URI.parse(uri)
-        @http_request_headers = http_request_headers.insensitive
-        @http_request_method = http_request_method
-        @http_request_payload = http_request_payload
+      def initialize(url, headers: {}, method: "GET", params: nil, json: nil, data: nil)
+        @url = Addressable::URI.parse(url)
+        @headers = headers.insensitive
+        @method = method
 
-        http_request_headers["content-type"] = http_request_payload_type if http_request_payload_type
+        @params = params
+        @json = json
+        @data = data
+
+        headers["content-type"] = "application/json" unless json.nil?
       end
 
       def read
-        return read_file(uri.path) if uri.scheme == "file"
+        return read_file(url.path) if url.scheme == "file"
 
         res = nil
-        client = HTTP.new(uri, headers: http_request_headers, payload: http_request_payload)
+        client = HTTP.new(url, headers: headers)
 
-        res = client.get if http_request_method == "GET"
-        res = client.post if http_request_method == "POST"
+        res = client.get(params: params) if method == "GET"
+        res = client.post(params: params, json: json, data: data) if method == "POST"
 
         return [] if res.nil?
 

data/lib/mihari/http.rb

@@ -4,54 +4,62 @@ require "insensitive_hash"
 
 module Mihari
   class HTTP
-    attr_reader :url, :headers, :payload
+    # @return [String]
+    attr_reader :url
 
-    def initialize(url, headers: {}, payload: {})
+    # @return [Hash]
+    attr_reader :headers
+
+    def initialize(url, headers: {})
       @url = url.is_a?(URI) ? url : URI(url.to_s)
       @headers = headers.insensitive
-      @payload = payload
     end
 
     #
     # Make a GET request
     #
+    # @param [Hash, nil] params
+    #
     # @return [Net::HTTPResponse]
     #
-    def get
+    def get(params: nil)
       new_url = url.deep_dup
-      new_url.query = Addressable::URI.form_encode(payload) unless payload.empty?
+      new_url.query = Addressable::URI.form_encode(params) unless (params || {}).empty?
 
       get = Net::HTTP::Get.new(new_url)
       request get
     end
 
     #
-    # Make a POST request
+    # Make a POST requesti
+    #
+    # @param [Hash, nil] params
+    # @param [Hash, nil] json
+    # @param [Hash, nil] data
     #
     # @return [Net::HTTPResponse]
     #
-    def post
-      post = Net::HTTP::Post.new(url)
-
-      case content_type
-      when "application/json"
-        post.body = JSON.generate(payload)
-      when "application/x-www-form-urlencoded"
-        post.set_form_data(payload)
-      end
+    def post(params: nil, json: nil, data: nil)
+      new_url = url.deep_dup
+      new_url.query = Addressable::URI.form_encode(params) unless (params || {}).empty?
+
+      post = Net::HTTP::Post.new(new_url)
+
+      post.body = JSON.generate(json) if json
+      post.set_form_data(data) if data
 
       request post
     end
 
     class << self
-      def get(url, headers: {}, params: {})
-        client = new(url, headers: headers, payload: params)
-        client.get
+      def get(url, headers: {}, params: nil)
+        client = new(url, headers: headers)
+        client.get(params: params)
       end
 
-      def post(url, headers: {}, payload: {})
-        client = new(url, headers: headers, payload: payload)
-        client.post
+      def post(url, headers: {}, params: nil, json: nil, data: nil)
+        client = new(url, headers: headers)
+        client.post(params: params, json: json, data: data)
       end
     end
 

data/lib/mihari/mixins/database.rb

@@ -6,6 +6,8 @@ module Mihari
       def with_db_connection
         Mihari::Database.connect
         yield
+      rescue ActiveRecord::StatementInvalid
+        Mihari.logger.error("You haven't finished the DB migration! Please run 'mihari db migrate'.")
       ensure
         Mihari::Database.close
       end

data/lib/mihari/mixins/retriable.rb

@@ -9,7 +9,9 @@ module Mihari
         Errno::EPIPE,
         OpenSSL::SSL::SSLError,
         Timeout::Error,
-        RetryableError
+        RetryableError,
+        NetworkError,
+        TimeoutError
       ]
 
       #

data/lib/mihari/schemas/analyzer.rb

@@ -82,10 +82,11 @@ module Mihari
       required(:analyzer).value(Types::String.enum("feed"))
       required(:query).value(:string)
       required(:selector).value(:string)
-      optional(:http_request_method).value(Types::HTTPRequestMethods).default("GET")
-      optional(:http_request_headers).value(:hash).default({})
-      optional(:http_request_payload).value(:hash).default({})
-      optional(:http_request_payload_type).value(Types::HTTPRequestPayloadTypes)
+      optional(:method).value(Types::HTTPRequestMethods).default("GET")
+      optional(:headers).value(:hash).default({})
+      optional(:params).value(:hash)
+      optional(:data).value(:hash)
+      optional(:json).value(:hash)
       optional(:options).hash(AnalyzerOptions)
     end
   end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "5.0.1"
+  VERSION = "5.1.1"
 end

data/lib/mihari.rb

@@ -212,6 +212,27 @@ require "mihari/emitters/slack"
 require "mihari/emitters/the_hive"
 require "mihari/emitters/webhook"
 
+# Clients
+require "mihari/clients/base"
+
+require "mihari/clients/binaryedge"
+require "mihari/clients/censys"
+require "mihari/clients/circl"
+require "mihari/clients/crtsh"
+require "mihari/clients/dnpedia"
+require "mihari/clients/dnstwister"
+require "mihari/clients/greynoise"
+require "mihari/clients/misp"
+require "mihari/clients/onyphe"
+require "mihari/clients/otx"
+require "mihari/clients/passivetotal"
+require "mihari/clients/publsedive"
+require "mihari/clients/shodan"
+require "mihari/clients/the_hive"
+require "mihari/clients/urlscan"
+require "mihari/clients/virustotal"
+require "mihari/clients/zoomeye"
+
 # Analyzers
 require "mihari/analyzers/base"
 

data/mihari.gemspec

@@ -4,6 +4,11 @@ lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "mihari/version"
 
+def ci_env?
+  # CI=true in GitHub Actions
+  ENV["CI"]
+end
+
 Gem::Specification.new do |spec|
   spec.name = "mihari"
   spec.version = Mihari::VERSION
@@ -30,8 +35,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 2.4"
   spec.add_development_dependency "coveralls_reborn", "~> 0.27"
   spec.add_development_dependency "fakefs", "~> 2.4"
+  spec.add_development_dependency "fuubar", "~> 2.5"
   spec.add_development_dependency "mysql2", "~> 0.5"
-  spec.add_development_dependency "overcommit", "~> 0.60"
   spec.add_development_dependency "pg", "~> 1.4"
   spec.add_development_dependency "rack-test", "~> 2.0"
   spec.add_development_dependency "rake", "~> 13.0"
@@ -40,19 +45,19 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 3.12"
   spec.add_development_dependency "simplecov-lcov", "~> 0.8.0"
   spec.add_development_dependency "standard", "~> 1.24"
-  spec.add_development_dependency "steep", "~> 1.3"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.1"
   spec.add_development_dependency "webmock", "~> 3.18"
 
+  unless ci_env?
+    spec.add_development_dependency "overcommit", "~> 0.60"
+    spec.add_development_dependency "ruby-lsp", "~> 0.4"
+    spec.add_development_dependency "steep", "~> 1.3"
+  end
+
   spec.add_dependency "activerecord", "7.0.4.2"
   spec.add_dependency "addressable", "2.8.1"
   spec.add_dependency "awrence", "2.0.1"
-  spec.add_dependency "binaryedge", "0.1.0"
-  spec.add_dependency "censysx", "0.1.1"
-  spec.add_dependency "crtsh-rb", "0.3.1"
-  spec.add_dependency "dnpedia", "0.1.0"
-  spec.add_dependency "dnstwister", "0.1.0"
   spec.add_dependency "dotenv", "2.8.1"
   spec.add_dependency "dry-configurable", "1.0.1"
   spec.add_dependency "dry-container", "0.11.0"
@@ -66,37 +71,26 @@ Gem::Specification.new do |spec|
   spec.add_dependency "grape-entity", "1.0.0"
   spec.add_dependency "grape-swagger", "1.5.0"
   spec.add_dependency "grape-swagger-entity", "0.5.1"
-  spec.add_dependency "greynoise", "0.1.1"
-  spec.add_dependency "hachi", "2.0.0"
   spec.add_dependency "insensitive_hash", "0.3.3"
   spec.add_dependency "jr-cli", "0.6.0"
   spec.add_dependency "launchy", "2.5.2"
   spec.add_dependency "memist", "2.0.2"
-  spec.add_dependency "misp", "0.1.4"
   spec.add_dependency "net-ping", "2.0.8"
   spec.add_dependency "normalize_country", "0.3.2"
-  spec.add_dependency "onyphe", "2.0.0"
   spec.add_dependency "parallel", "1.22.1"
-  spec.add_dependency "passive_circl", "0.1.0"
-  spec.add_dependency "passivetotalx", "0.1.1"
   spec.add_dependency "plissken", "2.0.1"
   spec.add_dependency "public_suffix", "5.0.1"
-  spec.add_dependency "pulsedive", "0.1.5"
   spec.add_dependency "puma", "6.0.2"
   spec.add_dependency "rack", "2.2.4"
   spec.add_dependency "rack-contrib", "2.3.0"
   spec.add_dependency "rack-cors", "1.1.1"
   spec.add_dependency "securitytrails", "1.0.0"
   spec.add_dependency "semantic_logger", "4.12.0"
-  spec.add_dependency "sentry-ruby", "5.7.0"
-  spec.add_dependency "shodanx", "0.2.1"
+  spec.add_dependency "sentry-ruby", "5.8.0"
   spec.add_dependency "slack-notifier", "2.4.0"
-  spec.add_dependency "sqlite3", "1.6.0"
+  spec.add_dependency "sqlite3", "1.6.1"
   spec.add_dependency "thor", "1.2.1"
-  spec.add_dependency "urlscan", "0.8.0"
   spec.add_dependency "uuidtools", "2.2.0"
-  spec.add_dependency "virustotalx", "1.2.0"
   spec.add_dependency "whois", "5.1.0"
   spec.add_dependency "whois-parser", "2.0.0"
-  spec.add_dependency "zoomeye-rb", "0.2.0"
 end