mihari 2.4.0 → 3.0.0

data/lib/mihari/commands/securitytrails.rb

@@ -5,13 +5,13 @@ module Mihari
     module SecurityTrails
       def self.included(thor)
         thor.class_eval do
-          desc "securitytrails [IP|DOMAIN|EMAIL]", "SecurityTrails lookup by an ip, domain or email"
+          desc "securitytrails [IP|DOMAIN|EMAIL]", "SecurityTrails search by an ip, domain or email"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
           def securitytrails(indiactor)
             with_error_handling do
-              run_analyzer Analyzers::SecurityTrails, query: refang(indiactor), options: options
+              run_analyzer Analyzers::SecurityTrails, query: indiactor, options: options
             end
           end
           map "st" => :securitytrails

data/lib/mihari/commands/shodan.rb

@@ -5,7 +5,7 @@ module Mihari
     module Shodan
       def self.included(thor)
         thor.class_eval do
-          desc "shodan [QUERY]", "Shodan host search by a query"
+          desc "shodan [QUERY]", "Shodan host search"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"

data/lib/mihari/commands/spyse.rb

@@ -5,7 +5,7 @@ module Mihari
     module Spyse
       def self.included(thor)
         thor.class_eval do
-          desc "spyse [QUERY]", "Spyse search by a query"
+          desc "spyse [QUERY]", "Spyse search"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"

data/lib/mihari/commands/urlscan.rb

@@ -5,11 +5,11 @@ module Mihari
     module Urlscan
       def self.included(thor)
         thor.class_eval do
-          desc "urlscan [QUERY]", "urlscan search by a given query"
+          desc "urlscan [QUERY]", "urlscan search"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
-          method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
+          method_option :target_type, type: :string, default: "url", desc: "target type to fetch from search results (target type should be 'url', 'domain' or 'ip')"
           method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
           def urlscan(query)
             with_error_handling do

data/lib/mihari/commands/validator.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Validator
+      include Mixins::Rule
+      include Mixins::Configuration
+
+      def self.included(thor)
+        thor.class_eval do
+          desc "rule [PATH]", "Validate format of a rule file"
+          def rule(path)
+            # convert str(YAML) to hash or str(path/YAML file) to hash
+            rule = load_rule(path)
+
+            # validate rule schema
+            validate_rule rule
+
+            puts "Valid format. The input is parsed as the following:"
+            puts rule.to_yaml
+          end
+
+          desc "config [PATH]", "Validate format of a config file"
+          def config(path)
+            # convert str(YAML) to hash or str(path/YAML file) to hash
+            config = load_config(path)
+
+            # validate config schema
+            validate_config config
+
+            puts "Valid format. The input is parsed as the following:"
+            puts config.to_yaml
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/virustotal.rb

@@ -5,13 +5,13 @@ module Mihari
     module VirusTotal
       def self.included(thor)
         thor.class_eval do
-          desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions lookup by an ip or domain"
+          desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions search by an ip or domain"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
           def virustotal(indiactor)
             with_error_handling do
-              run_analyzer Analyzers::VirusTotal, query: refang(indiactor), options: options
+              run_analyzer Analyzers::VirusTotal, query: indiactor, options: options
             end
           end
         end

data/lib/mihari/commands/zoomeye.rb

@@ -5,7 +5,7 @@ module Mihari
     module ZoomEye
       def self.included(thor)
         thor.class_eval do
-          desc "zoomeye [QUERY]", "ZoomEye search by a query"
+          desc "zoomeye [QUERY]", "ZoomEye search"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"

data/lib/mihari/constraints.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Mihari
+  ALLOWED_DATA_TYPES = ["hash", "ip", "domain", "url", "mail"].freeze
+end

data/lib/mihari/database.rb

@@ -2,7 +2,7 @@
 
 require "active_record"
 
-class InitialSchema < ActiveRecord::Migration[6.0]
+class InitialSchema < ActiveRecord::Migration[6.1]
   def change
     create_table :tags, if_not_exists: true do |t|
       t.string :name, null: false
@@ -32,6 +32,12 @@ class InitialSchema < ActiveRecord::Migration[6.0]
   end
 end
 
+class V3Schema < ActiveRecord::Migration[6.1]
+  def change
+    add_column :artifacts, :source, :string, if_not_exists: true
+  end
+end
+
 def adapter
   return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
   return "mysql2" if Mihari.config.database.start_with?("mysql2://")
@@ -54,7 +60,9 @@ module Mihari
         end
 
         ActiveRecord::Migration.verbose = false
+
         InitialSchema.migrate(:up)
+        V3Schema.migrate(:up)
       rescue StandardError
         # Do nothing
       end
@@ -65,7 +73,10 @@ module Mihari
       end
 
       def destroy!
-        InitialSchema.migrate(:down) if ActiveRecord::Base.connected?
+        return unless ActiveRecord::Base.connected?
+
+        InitialSchema.migrate(:down)
+        V3Schema.migrate(:down)
       end
     end
   end

data/lib/mihari/emitters/base.rb

@@ -3,8 +3,8 @@
 module Mihari
   module Emitters
     class Base
-      include Configurable
-      include Retriable
+      include Mixins::Configurable
+      include Mixins::Retriable
 
       def self.inherited(child)
         Mihari.emitters << child

data/lib/mihari/emitters/database.rb

@@ -10,7 +10,7 @@ module Mihari
       def emit(title:, description:, artifacts:, source:, tags: [])
         return if artifacts.empty?
 
-        tags = tags.map { |name| Tag.find_or_create_by(name: name) }.compact.uniq
+        tags = tags.filter_map { |name| Tag.find_or_create_by(name: name) }.uniq
         taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }
 
         alert = Alert.new(

data/lib/mihari/emitters/misp.rb

@@ -36,7 +36,7 @@ module Mihari
 
       private
 
-      def config_keys
+      def configuration_keys
         %w[misp_api_endpoint misp_api_key]
       end
 

data/lib/mihari/emitters/slack.rb

@@ -2,18 +2,17 @@
 
 require "slack-notifier"
 require "digest/sha2"
+require "dry-initializer"
 
 module Mihari
   module Emitters
     class Attachment
       include Mem
 
-      attr_reader :data, :data_type
+      extend Dry::Initializer
 
-      def initialize(data:, data_type:)
-        @data = data
-        @data_type = data_type
-      end
+      option :data
+      option :data_type
 
       def actions
         [vt_link, urlscan_link, censys_link, shodan_link].compact
@@ -144,7 +143,7 @@ module Mihari
 
       private
 
-      def config_keys
+      def configuration_keys
         %w[slack_webhook_url]
       end
     end

data/lib/mihari/emitters/the_hive.rb

@@ -26,7 +26,7 @@ module Mihari
 
       private
 
-      def config_keys
+      def configuration_keys
         %w[thehive_api_endpoint thehive_api_key]
       end
 

data/lib/mihari/emitters/webhook.rb

@@ -33,7 +33,7 @@ module Mihari
 
       private
 
-      def config_keys
+      def configuration_keys
         %w[webhook_url]
       end
 
@@ -46,14 +46,7 @@ module Mihari
       end
 
       def use_json_body
-        @use_json_body ||= truthy?(Mihari.config.webhook_use_json_body || 'false')
-      end
-
-      def truthy?(value)
-        return true if value == "true"
-        return true if value == true
-
-        false
+        @use_json_body ||= Mihari.config.webhook_use_json_body
       end
     end
   end

data/lib/mihari/mixins/configurable.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Mixins
+    module Configurable
+      #
+      # Check whether it is configured or not
+      #
+      # @return [Boolean]
+      #
+      def configured?
+        configuration_keys.all? { |key| Mihari.config.send(key) }
+      end
+
+      #
+      # Configuration values
+      #
+      # @return [Array<Hash>, nil] Configuration values as a list of hash. Returns nil if there is any keys.
+      #
+      def configuration_values
+        return nil if configuration_keys.empty?
+
+        configuration_keys.map do |key|
+          { key: key.upcase, value: Mihari.config.send(key) }
+        end
+      end
+
+      #
+      # Configuration keys
+      #
+      # @return [Array<String>] A list of cofiguration keys
+      #
+      def configuration_keys
+        []
+      end
+    end
+  end
+end

data/lib/mihari/mixins/configuration.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "colorize"
+require "yaml"
+
+module Mihari
+  module Mixins
+    module Configuration
+      #
+      # Load config file into hash
+      #
+      # @param [String] path Path to YAML file
+      #
+      # @return [Hash]
+      #
+      def load_config(path)
+        return YAML.safe_load(File.read(path), symbolize_names: true) if Pathname(path).exist?
+
+        YAML.safe_load(path, symbolize_names: true)
+      end
+
+      #
+      # Validate config schema
+      #
+      # @param [Hash] config
+      #
+      def validate_config(config)
+        error_message = "Failed to parse the input as a config!"
+
+        contract = Schemas::ConfigurationContract.new
+        result = contract.call(config)
+        unless result.errors.empty?
+          puts error_message.colorize(:red)
+          show_validation_errors result.errors
+          raise ArgumentError, "Invalid config schema"
+        end
+
+        # check keys
+        # TODO: check keys with dry-schema
+        valid_keys = Mihari.config.values.keys
+        config.each_key do |key|
+          unless valid_keys.include?(key)
+            puts error_message.colorize(:red)
+            raise ArgumentError, "#{key} is not a valid key."
+          end
+        end
+      end
+
+      #
+      # Returns a template for config
+      #
+      # @return [String] A template for config
+      #
+      def config_template
+        config = Mihari.config.values.keys.map do |key|
+          [key.to_s, nil]
+        end.to_h
+
+        YAML.dump(config)
+      end
+
+      #
+      # Create (blank) config file
+      #
+      # @param [String] filename
+      # @param [Dry::Files] files
+      # @param [String] template
+      #
+      # @return [nil]
+      #
+      def initialize_config_yaml(filename, files = Dry::Files.new, template: config_template)
+        files.write(filename, template)
+      end
+
+      private
+
+      def show_validation_errors(errors)
+        errors.messages.each do |message|
+          path = message.path.map(&:to_s).join
+          puts "- #{path} #{message.text}".colorize(:red)
+        end
+      end
+    end
+  end
+end

data/lib/mihari/mixins/hash.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "cymbal"
+
+module Mihari
+  module Mixins
+    module Hash
+      #
+      # Symbolize hash keys
+      #
+      # @param [Hash] hash
+      #
+      # @return [Hash]
+      #
+      def symbolize_hash(hash)
+        Cymbal.symbolize hash
+      end
+    end
+  end
+end

data/lib/mihari/mixins/refang.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Mixins
+    module Refang
+      #
+      # Refang defanged indicator
+      #
+      # @param [String] indicator
+      #
+      # @return [String]
+      #
+      def refang(indicator)
+        return indicator.gsub("[.]", ".").gsub("(.)", ".") if indicator.is_a?(String)
+
+        # for RSpec & Ruby 2.7
+        indicator
+      end
+    end
+  end
+end

data/lib/mihari/mixins/retriable.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Mixins
+    module Retriable
+      #
+      # Retry on error
+      #
+      # @param [Integer] times
+      # @param [Integer] interval
+      #
+      # @return [nil]
+      #
+      def retry_on_error(times: 3, interval: 10)
+        try = 0
+        begin
+          try += 1
+          yield
+        rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error, RetryableError => e
+          sleep interval
+          retry if try < times
+          raise e
+        end
+      end
+    end
+  end
+end