mihari 3.10.0 → 4.0.0

data/lib/mihari/analyzers/urlscan.rb

@@ -2,17 +2,17 @@
 
 require "urlscan"
 
-SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
-
 module Mihari
   module Analyzers
     class Urlscan < Base
       param :query
-      option :title, default: proc { "urlscan search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
       option :allowed_data_types, default: proc { SUPPORTED_DATA_TYPES }
-      option :use_similarity, default: proc { false }
+
+      option :interval, default: proc { 0 }
+
+      SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
+      SIZE = 1000
 
       def initialize(*args, **kwargs)
         super
@@ -21,15 +21,15 @@ module Mihari
       end
 
       def artifacts
-        result = search
-        return [] unless result
-
-        results = result["results"] || []
+        responses = search
+        results = responses.map(&:results).flatten
 
         allowed_data_types.map do |type|
-          results.filter_map do |match|
-            match.dig "page", type
-          end.uniq
+          results.filter_map do |result|
+            page = result.page
+            data = page.send(type.to_sym)
+            data.nil? ? nil : Artifact.new(data: data, source: source, metadata: result)
+          end
         end.flatten
       end
 
@@ -43,15 +43,38 @@ module Mihari
         @api ||= ::UrlScan::API.new(Mihari.config.urlscan_api_key)
       end
 
+      #
+      # Search with search_after option
+      #
+      # @return [Structs::Urlscan::Response]
+      #
+      def search_with_search_after(search_after: nil)
+        res = api.search(query, size: SIZE, search_after: search_after)
+        Structs::Urlscan::Response.from_dynamic! res
+      end
+
       #
       # Search
       #
-      # @return [Array<Hash>]
+      # @return [Array<Structs::Urlscan::Response>]
       #
       def search
-        return api.pro.similar(query) if use_similarity
+        responses = []
+
+        search_after = nil
+        loop do
+          res = search_with_search_after(search_after: search_after)
+          responses << res
+
+          break if res.results.length < SIZE
+
+          search_after = res.results.last.sort.join(",")
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
+        end
 
-        api.search(query, size: 10_000)
+        responses
       end
 
       #

data/lib/mihari/analyzers/virustotal.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "VirusTotal search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -47,7 +44,7 @@ module Mihari
       #
       # Search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         case type
@@ -63,28 +60,30 @@ module Mihari
       #
       # Domain search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def domain_search
         res = api.domain.resolutions(query)
 
         data = res["data"] || []
         data.filter_map do |item|
-          item.dig("attributes", "ip_address")
-        end.uniq
+          data = item.dig("attributes", "ip_address")
+          data.nil? ? nil : Artifact.new(data: data, source: source, metadata: item)
+        end
       end
 
       #
       # IP search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ip_search
         res = api.ip_address.resolutions(query)
 
         data = res["data"] || []
         data.filter_map do |item|
-          item.dig("attributes", "host_name")
+          data = item.dig("attributes", "host_name")
+          Artifact.new(data: data, source: source, metadata: item)
         end.uniq
       end
     end

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -6,9 +6,8 @@ module Mihari
   module Analyzers
     class VirusTotalIntelligence < Base
       param :query
-      option :title, default: proc { "VirusTotal Intelligence search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
+      option :interval, default: proc { 0 }
 
       def initialize(*args, **kwargs)
         super
@@ -19,8 +18,10 @@ module Mihari
       def artifacts
         responses = search_witgh_cursor
         responses.map do |response|
-          response.data.map(&:value)
-        end.flatten.compact.uniq
+          response.data.map do |datum|
+            Artifact.new(data: datum.value, source: source, metadata: datum.metadata)
+          end
+        end.flatten
       end
 
       private
@@ -54,6 +55,9 @@ module Mihari
           break if response.meta.cursor.nil?
 
           cursor = response.meta.cursor
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
 
         responses

data/lib/mihari/analyzers/zoomeye.rb

@@ -6,11 +6,11 @@ module Mihari
   module Analyzers
     class ZoomEye < Base
       param :query
-      option :title, default: proc { "ZoomEye search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
       option :type, default: proc { "host" }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         case type
         when "host"
@@ -48,13 +48,14 @@ module Mihari
       #
       # @param [Array<Hash>] responses
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def convert_responses(responses)
         responses.map do |res|
           matches = res["matches"] || []
           matches.map do |match|
-            match["ip"]
+            data = match["ip"]
+            Artifact.new(data: data, source: source, metadata: match)
           end
         end.flatten.compact.uniq
       end
@@ -87,6 +88,9 @@ module Mihari
           total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         convert_responses responses.compact
       end
@@ -119,6 +123,9 @@ module Mihari
           total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         convert_responses responses.compact
       end

data/lib/mihari/cli/base.rb

@@ -1,15 +1,10 @@
 # frozen_string_literal: true
 
-require "thor"
-
-require "mihari/mixins/hash"
-
 require "mihari/cli/mixins/utils"
 
 module Mihari
   module CLI
     class Base < Thor
-      include Mihari::Mixins::Hash
       include Mixins::Utils
 
       class << self

data/lib/mihari/cli/init.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "thor"
-
 require "mihari/commands/init"
 
 module Mihari

data/lib/mihari/cli/main.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "thor"
+
 # Commands
 require "mihari/commands/search"
 require "mihari/commands/web"
@@ -7,7 +9,6 @@ require "mihari/commands/web"
 # CLIs
 require "mihari/cli/base"
 
-require "mihari/cli/analyzer"
 require "mihari/cli/init"
 require "mihari/cli/validator"
 
@@ -17,13 +18,10 @@ module Mihari
       include Mihari::Commands::Search
       include Mihari::Commands::Web
 
-      desc "analyze", "Sub commands to run an analyzer"
-      subcommand "analyze", Analyzer
-
-      desc "init", "Sub commands to initialize config & rule"
+      desc "init", "Sub commands to initialize a rule"
       subcommand "init", Initialization
 
-      desc "validate", "Sub commands to validate format of config & rule"
+      desc "validate", "Sub commands to validate format of a rule"
       subcommand "validate", Validator
     end
   end

data/lib/mihari/cli/mixins/utils.rb

@@ -27,19 +27,6 @@ module Mihari
           %w[title description artifacts].all? { |key| json.key? key }
         end
 
-        #
-        # Load configuration and establish DB connection
-        #
-        # @return [Hash]
-        #
-        def load_configuration
-          config = options["config"]
-          return unless config
-
-          Mihari.load_config_from_yaml config
-          Database.connect
-        end
-
         #
         # Run analyzer
         #
@@ -50,14 +37,12 @@ module Mihari
         # @return [nil]
         #
         def run_analyzer(analyzer_class, query:, options:)
-          load_configuration
-
           # options = Thor::CoreExt::HashWithIndifferentAccess
           # ref. https://www.rubydoc.info/github/wycats/thor/Thor/CoreExt/HashWithIndifferentAccess
           # so need to covert it to a plain hash
           hash_options = options.to_hash
 
-          hash_options = symbolize_hash(hash_options)
+          hash_options = hash_options.symbolize_keys
           hash_options = normalize_options(hash_options)
 
           analyzer = analyzer_class.new(query, **hash_options)
@@ -76,8 +61,7 @@ module Mihari
         # @return [Hash]
         #
         def normalize_options(options)
-          # Delete :config because it is not intended to use for running an analyzer
-          [:config, :ignore_old_artifacts, :ignore_threshold].each do |ignore_key|
+          [:ignore_old_artifacts, :ignore_threshold].each do |ignore_key|
             options.delete(ignore_key)
           end
           options

data/lib/mihari/commands/init.rb

@@ -1,30 +1,12 @@
 # frozen_string_literal: true
 
-require "colorize"
-
 module Mihari
   module Commands
     module Initialization
-      include Mixins::Configuration
       include Mixins::Rule
 
       def self.included(thor)
         thor.class_eval do
-          desc "config", "Create a config file"
-          method_option :filename, type: :string, default: "mihari.yml"
-          def config
-            filename = options["filename"]
-
-            warning = "#{filename} exists. Do you want to overwrite it? (y/n)"
-            if File.exist?(filename) && !(yes? warning)
-              return
-            end
-
-            initialize_config_yaml filename
-
-            puts "The config file is initialized as #{filename}.".colorize(:blue)
-          end
-
           desc "rule", "Create a rule file"
           method_option :filename, type: :string, default: "rule.yml"
           def rule

data/lib/mihari/commands/search.rb

@@ -3,22 +3,22 @@
 module Mihari
   module Commands
     module Search
+      include Mixins::Database
       include Mixins::Rule
 
       def self.included(thor)
         thor.class_eval do
           desc "search [RULE]", "Search by a rule"
-          method_option :config, type: :string, desc: "Path to the config file"
-          def search_by_rule(rule)
-            # load configuration
-            load_configuration
-
-            # convert str(YAML) to hash or str(path/YAML file) to hash
-            rule = load_rule(rule)
-
-            # validate rule schema
-            rule = validate_rule(rule)
+          def search_by_rule(path_or_id)
+            rule = load_rule(path_or_id)
+            # validate
+            begin
+              validate_rule! rule
+            rescue RuleValidationError => e
+              raise e
+            end
 
+            # build and run the analyzer
             analyzer = build_rule_analyzer(
               title: rule[:title],
               description: rule[:description],
@@ -26,8 +26,7 @@ module Mihari
               tags: rule[:tags],
               allowed_data_types: rule[:allowed_data_types],
               disallowed_data_values: rule[:disallowed_data_values],
-              source: rule[:source],
-              id: rule[:id]
+              id: rule.id
             )
 
             ignore_old_artifacts = rule[:ignore_old_artifacts]
@@ -35,6 +34,14 @@ module Mihari
 
             with_error_handling do
               run_rule_analyzer analyzer, ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold
+
+              # record a rule
+              with_db_connection do
+                model = rule.to_model
+                model.save
+              rescue ActiveRecord::RecordNotUnique
+                nil
+              end
             end
           end
         end
@@ -51,11 +58,10 @@ module Mihari
       # @param [Array<String>, nil] tags
       # @param [Array<String>, nil] allowed_data_types
       # @param [Array<String>, nil] disallowed_data_values
-      # @param [String, nil] source
       #
       # @return [Mihari::Analyzers::Rule]
       #
-      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, source: nil, id: nil)
+      def build_rule_analyzer(title:, description:, queries:, tags: nil, allowed_data_types: nil, disallowed_data_values: nil, id: nil)
         tags = [] if tags.nil?
         allowed_data_types = ALLOWED_DATA_TYPES if allowed_data_types.nil?
         disallowed_data_values = [] if disallowed_data_values.nil?
@@ -67,7 +73,6 @@ module Mihari
           queries: queries,
           allowed_data_types: allowed_data_types,
           disallowed_data_values: disallowed_data_values,
-          source: source,
           id: id
         )
       end

data/lib/mihari/commands/validator.rb

@@ -4,32 +4,20 @@ module Mihari
   module Commands
     module Validator
       include Mixins::Rule
-      include Mixins::Configuration
 
       def self.included(thor)
         thor.class_eval do
           desc "rule [PATH]", "Validate format of a rule file"
           def rule(path)
-            # convert str(YAML) to hash or str(path/YAML file) to hash
             rule = load_rule(path)
 
-            # validate rule schema
-            validate_rule rule
-
-            puts "Valid format. The input is parsed as the following:"
-            puts rule.to_yaml
-          end
-
-          desc "config [PATH]", "Validate format of a config file"
-          def config(path)
-            # convert str(YAML) to hash or str(path/YAML file) to hash
-            config = load_config(path)
-
-            # validate config schema
-            validate_config config
-
-            puts "Valid format. The input is parsed as the following:"
-            puts config.to_yaml
+            begin
+              validate_rule! rule
+              puts "Valid format. The input is parsed as the following:"
+              puts rule.data.to_yaml
+            rescue RuleValidationError
+              nil
+            end
           end
         end
       end

data/lib/mihari/commands/web.rb

@@ -10,15 +10,12 @@ module Mihari
           method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
           method_option :threads, type: :string, default: "0:16", desc: "min:max threads to use"
           method_option :verbose, type: :boolean, default: true, desc: "Report each request"
-          method_option :config, type: :string, desc: "Path to the config file"
           def web
             port = options["port"]
             host = options["host"]
             threads = options["threads"]
             verbose = options["verbose"]
 
-            load_configuration
-
             # set rack env as production
             ENV["RACK_ENV"] ||= "production"
 

data/lib/mihari/database.rb

@@ -1,8 +1,18 @@
 # frozen_string_literal: true
 
-require "active_record"
+def env
+  ENV["APP_ENV"] || ENV["RACK_ENV"]
+end
+
+def test_env?
+  env == "test"
+end
 
-class InitialSchema < ActiveRecord::Migration[6.1]
+def development_env?
+  env == "development"
+end
+
+class InitialSchema < ActiveRecord::Migration[7.0]
   def change
     create_table :tags, if_not_exists: true do |t|
       t.string :name, null: false
@@ -32,13 +42,13 @@ class InitialSchema < ActiveRecord::Migration[6.1]
   end
 end
 
-class AddeSourceToArtifactSchema < ActiveRecord::Migration[6.1]
+class AddeSourceToArtifactSchema < ActiveRecord::Migration[7.0]
   def change
     add_column :artifacts, :source, :string, if_not_exists: true
   end
 end
 
-class EnrichmentsSchema < ActiveRecord::Migration[6.1]
+class EnrichmentsSchema < ActiveRecord::Migration[7.0]
   def change
     create_table :autonomous_systems, if_not_exists: true do |t|
       t.integer :asn, null: false
@@ -74,7 +84,7 @@ class EnrichmentsSchema < ActiveRecord::Migration[6.1]
   end
 end
 
-class EnrichmentCreatedAtSchema < ActiveRecord::Migration[6.1]
+class EnrichmentCreatedAtSchema < ActiveRecord::Migration[7.0]
   def change
     # Add created_at column because now it is able to enrich an atrifact after the creation
     add_column :autonomous_systems, :created_at, :datetime, if_not_exists: true
@@ -85,6 +95,23 @@ class EnrichmentCreatedAtSchema < ActiveRecord::Migration[6.1]
   end
 end
 
+class RuleSchema < ActiveRecord::Migration[7.0]
+  def change
+    create_table :rules, id: :string, if_not_exists: true do |t|
+      t.string :title, null: false
+      t.string :description, null: false
+      t.json :data, null: false
+      t.timestamps
+    end
+  end
+end
+
+class AddeMetadataToArtifactSchema < ActiveRecord::Migration[7.0]
+  def change
+    add_column :artifacts, :metadata, :json, if_not_exists: true
+  end
+end
+
 def adapter
   return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
   return "mysql2" if Mihari.config.database.start_with?("mysql2://")
@@ -95,7 +122,34 @@ end
 module Mihari
   class Database
     class << self
+      include Memist::Memoizable
+
+      #
+      # DB migraration
+      #
+      # @param [Symbol] direction
+      #
+      def migrate(direction)
+        ActiveRecord::Migration.verbose = false
+
+        [
+          InitialSchema,
+          AddeSourceToArtifactSchema,
+          EnrichmentsSchema,
+          EnrichmentCreatedAtSchema,
+          # v4
+          RuleSchema,
+          AddeMetadataToArtifactSchema
+        ].each { |schema| schema.migrate direction }
+      end
+      memoize :migrate unless test_env?
+
+      #
+      # Establish DB connection
+      #
       def connect
+        return if ActiveRecord::Base.connected?
+
         case adapter
         when "postgresql", "mysql2"
           ActiveRecord::Base.establish_connection(Mihari.config.database)
@@ -106,32 +160,30 @@ module Mihari
           )
         end
 
-        ActiveRecord::Base.logger = Logger.new($stdout) if ENV["RACK_ENV"] == "development"
-        ActiveRecord::Migration.verbose = false
+        ActiveRecord::Base.logger = Logger.new($stdout) if development_env?
 
-        InitialSchema.migrate(:up)
-        AddeSourceToArtifactSchema.migrate(:up)
-        EnrichmentsSchema.migrate(:up)
-        EnrichmentCreatedAtSchema.migrate(:up)
+        migrate :up
       rescue StandardError
         # Do nothing
       end
 
+      #
+      # Close DB connection(s)
+      #
       def close
-        ActiveRecord::Base.clear_active_connections!
-        ActiveRecord::Base.connection.close
+        return unless ActiveRecord::Base.connected?
+
+        ActiveRecord::Base.clear_all_connections!
       end
 
+      #
+      # Destory DB
+      #
       def destroy!
         return unless ActiveRecord::Base.connected?
 
-        InitialSchema.migrate(:down)
-        AddeSourceToArtifactSchema.migrate(:down)
-        EnrichmentsSchema.migrate(:down)
-        EnrichmentCreatedAtSchema.migrate(:down)
+        migrate :down
       end
     end
   end
 end
-
-Mihari::Database.connect

data/lib/mihari/emitters/misp.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "misp"
-require "net/ping"
 
 module Mihari
   module Emitters

data/lib/mihari/emitters/slack.rb

@@ -1,13 +1,12 @@
 # frozen_string_literal: true
 
-require "slack-notifier"
 require "digest/sha2"
-require "dry-initializer"
+require "slack-notifier"
 
 module Mihari
   module Emitters
     class Attachment
-      include Mem
+      include Memist::Memoizable
 
       extend Dry::Initializer
 
@@ -63,7 +62,7 @@ module Mihari
         when "domain"
           "https://urlscan.io/domain/#{data}"
         when "url"
-          uri = URI(data)
+          uri = Addressable::URI.parse(data)
           "https://urlscan.io/domain/#{uri.hostname}"
         end
       end