mihari 2.2.1 → 3.0.1

data/lib/mihari/cli/analyzer.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "mihari/commands/binaryedge"
+require "mihari/commands/censys"
+require "mihari/commands/circl"
+require "mihari/commands/crtsh"
+require "mihari/commands/dnpedia"
+require "mihari/commands/dnstwister"
+require "mihari/commands/onyphe"
+require "mihari/commands/otx"
+require "mihari/commands/passivetotal"
+require "mihari/commands/pulsedive"
+require "mihari/commands/securitytrails"
+require "mihari/commands/shodan"
+require "mihari/commands/spyse"
+require "mihari/commands/urlscan"
+require "mihari/commands/virustotal"
+require "mihari/commands/zoomeye"
+
+require "mihari/commands/json"
+
+module Mihari
+  module CLI
+    class Analyzer < Base
+      include Mihari::Commands::BinaryEdge
+      include Mihari::Commands::Censys
+      include Mihari::Commands::CIRCL
+      include Mihari::Commands::Crtsh
+      include Mihari::Commands::DNPedia
+      include Mihari::Commands::DNSTwister
+      include Mihari::Commands::JSON
+      include Mihari::Commands::Onyphe
+      include Mihari::Commands::OTX
+      include Mihari::Commands::PassiveTotal
+      include Mihari::Commands::Pulsedive
+      include Mihari::Commands::SecurityTrails
+      include Mihari::Commands::Shodan
+      include Mihari::Commands::Spyse
+      include Mihari::Commands::Urlscan
+      include Mihari::Commands::VirusTotal
+      include Mihari::Commands::ZoomEye
+    end
+  end
+end

data/lib/mihari/cli/base.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "thor"
+
+require "mihari/mixins/hash"
+
+require "mihari/cli/mixins/utils"
+
+module Mihari
+  module CLI
+    class Base < Thor
+      include Mihari::Mixins::Hash
+      include Mixins::Utils
+
+      class_option :config, type: :string, desc: "Path to the config file"
+
+      class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not. Only affects with analyze commands."
+      class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not. Only affects with analyze commands."
+
+      class << self
+        def exit_on_failure?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/mihari/cli/init.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "thor"
+
+require "mihari/commands/init"
+
+module Mihari
+  module CLI
+    class Initialization < Base
+      include Mihari::Commands::Initialization
+    end
+  end
+end

data/lib/mihari/cli/main.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# Commands
+require "mihari/commands/search"
+require "mihari/commands/web"
+
+# CLIs
+require "mihari/cli/base"
+
+require "mihari/cli/analyzer"
+require "mihari/cli/init"
+require "mihari/cli/validator"
+
+module Mihari
+  module CLI
+    class Main < Base
+      include Mihari::Commands::Search
+      include Mihari::Commands::Web
+
+      desc "analyze", "Sub commands to run an analyzer"
+      subcommand "analyze", Analyzer
+
+      desc "init", "Sub commands to initialize config & rule"
+      subcommand "init", Initialization
+
+      desc "validate", "Sub commands to validate format of config & rule"
+      subcommand "validate", Validator
+    end
+  end
+end

data/lib/mihari/cli/mixins/utils.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Mihari
+  module CLI
+    module Mixins
+      module Utils
+        #
+        # Send an exception notification if there is any error in a block
+        #
+        # @return [Nil]
+        #
+        def with_error_handling
+          yield
+        rescue StandardError => e
+          notifier = Notifiers::ExceptionNotifier.new
+          notifier.notify e
+        end
+
+        #
+        # Check required keys in JSON
+        #
+        # @param [Hash] json
+        #
+        # @return [Boolean]
+        #
+        def required_alert_keys?(json)
+          %w[title description artifacts].all? { |key| json.key? key }
+        end
+
+        #
+        # Load configuration and establish DB connection
+        #
+        # @return [Hash]
+        #
+        def load_configuration
+          config = options["config"]
+          return unless config
+
+          Mihari.load_config_from_yaml config
+          Database.connect
+        end
+
+        #
+        # Run analyzer
+        #
+        # @param [Class<Mihari::Analyzers::Base>] analyzer_class
+        # @param [String] query
+        # @param [Hash] options
+        #
+        # @return [nil]
+        #
+        def run_analyzer(analyzer_class, query:, options:)
+          load_configuration
+
+          # options = Thor::CoreExt::HashWithIndifferentAccess
+          # ref. https://www.rubydoc.info/github/wycats/thor/Thor/CoreExt/HashWithIndifferentAccess
+          # so need to covert it to a plain hash
+          hash_options = options.to_hash
+
+          hash_options = symbolize_hash(hash_options)
+          hash_options = normalize_options(hash_options)
+
+          analyzer = analyzer_class.new(query, **hash_options)
+
+          analyzer.ignore_old_artifacts = options[:ignore_old_artifacts] || false
+          analyzer.ignore_threshold = options[:ignore_threshold] || 0
+
+          analyzer.run
+        end
+
+        #
+        # Normalize options (reject keys not for analyzers)
+        #
+        # @param [Hash] options
+        #
+        # @return [Hash]
+        #
+        def normalize_options(options)
+          # Delete :config because it is not intended to use for running an analyzer
+          [:config, :ignore_old_artifacts, :ignore_threshold].each do |ignore_key|
+            options.delete(ignore_key)
+          end
+          options
+        end
+      end
+    end
+  end
+end

data/lib/mihari/cli/validator.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "mihari/commands/validator"
+
+module Mihari
+  module CLI
+    class Validator < Base
+      include Mihari::Commands::Validator
+    end
+  end
+end

data/lib/mihari/commands/binaryedge.rb

@@ -5,7 +5,7 @@ module Mihari
     module BinaryEdge
       def self.included(thor)
         thor.class_eval do
-          desc "binaryedge [QUERY]", "BinaryEdge host search by a query"
+          desc "binaryedge [QUERY]", "BinaryEdge host search"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"

data/lib/mihari/commands/censys.rb

@@ -5,7 +5,7 @@ module Mihari
     module Censys
       def self.included(thor)
         thor.class_eval do
-          desc "censys [QUERY]", "Censys IPv4 search by a query"
+          desc "censys [QUERY]", "Censys IPv4 search"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"

data/lib/mihari/commands/circl.rb

@@ -5,13 +5,13 @@ module Mihari
     module CIRCL
       def self.included(thor)
         thor.class_eval do
-          desc "circl [DOMAIN|SHA1]", "CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint"
+          desc "circl [DOMAIN|SHA1]", "CIRCL passive DNS/SSL search by a domain or SHA1 certificate fingerprint"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
           def circl(query)
             with_error_handling do
-              run_analyzer Analyzers::CIRCL, query: refang(query), options: options
+              run_analyzer Analyzers::CIRCL, query: query, options: options
             end
           end
         end

data/lib/mihari/commands/crtsh.rb

@@ -5,7 +5,7 @@ module Mihari
     module Crtsh
       def self.included(thor)
         thor.class_eval do
-          desc "crtsh [QUERY]", "crt.sh search by a query"
+          desc "crtsh [QUERY]", "crt.sh search"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"

data/lib/mihari/commands/dnpedia.rb

@@ -5,7 +5,7 @@ module Mihari
     module DNPedia
       def self.included(thor)
         thor.class_eval do
-          desc "dnpedia [QUERY]", "DNPedia domain search by a query"
+          desc "dnpedia [QUERY]", "DNPedia domain search"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"

data/lib/mihari/commands/dnstwister.rb

@@ -5,13 +5,13 @@ module Mihari
     module DNSTwister
       def self.included(thor)
         thor.class_eval do
-          desc "dnstwister [DOMAIN]", "dnstwister lookup by a domain"
+          desc "dnstwister [DOMAIN]", "dnstwister search"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
           def dnstwister(domain)
             with_error_handling do
-              run_analyzer Analyzers::DNSTwister, query: refang(domain), options: options
+              run_analyzer Analyzers::DNSTwister, query: domain, options: options
             end
           end
         end

data/lib/mihari/commands/init.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "colorize"
+
+module Mihari
+  module Commands
+    module Initialization
+      def self.included(thor)
+        include Mixins::Configuration
+        include Mixins::Rule
+
+        thor.class_eval do
+          desc "config", "Create a config file"
+          method_option :filename, type: :string, default: "mihari.yml"
+          def config
+            filename = options["filename"]
+
+            warning = "#{filename} exists. Do you want to overwrite it? (y/n)"
+            if File.exist?(filename) && !(yes? warning)
+              return
+            end
+
+            initialize_config_yaml filename
+
+            puts "The config file is initialized as #{filename}.".colorize(:blue)
+          end
+
+          desc "rule", "Create a rule file"
+          method_option :filename, type: :string, default: "rule.yml"
+          def rule
+            filename = options["filename"]
+
+            warning = "#{filename} exists. Do you want to overwrite it? (y/n)"
+            if File.exist?(filename) && !(yes? warning)
+              return
+            end
+
+            initialize_rule_yaml filename
+
+            puts "The rule file is created as #{filename}.".colorize(:blue)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/json.rb

@@ -12,7 +12,7 @@ module Mihari
               raise ArgumentError, "Input not found: please give an input in a JSON format" unless json
 
               json = parse_as_json(json)
-              raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless valid_json?(json)
+              raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless required_alert_keys?(json)
 
               title = json["title"]
               description = json["description"]

data/lib/mihari/commands/onyphe.rb

@@ -5,7 +5,7 @@ module Mihari
     module Onyphe
       def self.included(thor)
         thor.class_eval do
-          desc "onyphe [QUERY]", "Onyphe datascan search by a query"
+          desc "onyphe [QUERY]", "Onyphe datascan search"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"

data/lib/mihari/commands/otx.rb

@@ -5,13 +5,13 @@ module Mihari
     module OTX
       def self.included(thor)
         thor.class_eval do
-          desc "otx [IP|DOMAIN]", "OTX lookup by an IP or domain"
+          desc "otx [IP|DOMAIN]", "OTX search by an IP or domain"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
           def otx(domain)
             with_error_handling do
-              run_analyzer Analyzers::OTX, query: refang(domain), options: options
+              run_analyzer Analyzers::OTX, query: domain, options: options
             end
           end
         end

data/lib/mihari/commands/passivetotal.rb

@@ -5,13 +5,13 @@ module Mihari
     module PassiveTotal
       def self.included(thor)
         thor.class_eval do
-          desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint"
+          desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal search by an ip, domain, email or SHA1 certificate fingerprint"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
           def passivetotal(indicator)
             with_error_handling do
-              run_analyzer Analyzers::PassiveTotal, query: refang(indicator), options: options
+              run_analyzer Analyzers::PassiveTotal, query: indicator, options: options
             end
           end
         end

data/lib/mihari/commands/pulsedive.rb

@@ -5,13 +5,13 @@ module Mihari
     module Pulsedive
       def self.included(thor)
         thor.class_eval do
-          desc "pulsedive [IP|DOMAIN]", "Pulsedive lookup by an ip or domain"
+          desc "pulsedive [IP|DOMAIN]", "Pulsedive search by an ip or domain"
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
           def pulsedive(indiactor)
             with_error_handling do
-              run_analyzer Analyzers::Pulsedive, query: refang(indiactor), options: options
+              run_analyzer Analyzers::Pulsedive, query: indiactor, options: options
             end
           end
         end

data/lib/mihari/commands/search.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Search
+      include Mixins::Rule
+
+      def self.included(thor)
+        thor.class_eval do
+          desc "search [RULE]", "Search by a rule"
+          def search_by_rule(rule)
+            # convert str(YAML) to hash or str(path/YAML file) to hash
+            rule = load_rule(rule)
+
+            # validate rule schema
+            validate_rule rule
+
+            analyzer = build_rule_analyzer(**rule)
+
+            ignore_old_artifacts = options["ignore_old_artifacts"] || false
+            ignore_threshold = options["ignore_threshold"] || 0
+
+            with_error_handling do
+              run_rule_analyzer analyzer, ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold
+            end
+          end
+        end
+      end
+
+      private
+
+      #
+      # Build a rule analyzer
+      #
+      # @param [String] title
+      # @param [String] description
+      # @param [Array<Hash>] queries
+      # @param [Array<String>, nil] tags
+      # @param [Array<String>, nil] allowed_data_types
+      # @param [String, nil] source
+      #
+      # @return [Mihari::Analyzers::Rule]
+      #
+      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, source: nil)
+        tags = [] if tags.nil?
+        allowed_data_types = ALLOWED_DATA_TYPES if allowed_data_types.nil?
+
+        Analyzers::Rule.new(
+          title: title,
+          description: description,
+          tags: tags,
+          queries: queries,
+          allowed_data_types: allowed_data_types,
+          source: source
+        )
+      end
+
+      #
+      # Run rule analyzer
+      #
+      # @param [Mihari::Analyzer::Rule] analyzer
+      # @param [Boolean] ignore_old_artifacts
+      # @param [Integer] ignore_threshold
+      #
+      # @return [nil]
+      #
+      def run_rule_analyzer(analyzer, ignore_old_artifacts: false, ignore_threshold: 0)
+        load_configuration
+
+        analyzer.ignore_old_artifacts = ignore_old_artifacts
+        analyzer.ignore_threshold = ignore_threshold
+
+        analyzer.run
+      end
+    end
+  end
+end