mihari 3.10.0 → 4.0.0

data/lib/mihari/emitters/stdout.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-
 module Mihari
   module Emitters
     class StandardOutput < Base

data/lib/mihari/emitters/the_hive.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "hachi"
-require "net/ping"
 
 module Mihari
   module Emitters

data/lib/mihari/emitters/webhook.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "net/http"
-require "uri"
-
 module Mihari
   module Emitters
     class Webhook < Base
@@ -15,7 +11,7 @@ module Mihari
       def emit(title:, description:, artifacts:, source:, tags:)
         return if artifacts.empty?
 
-        uri = URI(Mihari.config.webhook_url)
+        uri = Addressable::URI.parse(Mihari.config.webhook_url)
         data = {
           title: title,
           description: description,

data/lib/mihari/enrichers/ipinfo.rb

@@ -1,6 +1,6 @@
+# frozen_string_literal: true
+
 require "http"
-require "json"
-require "memist"
 
 module Mihari
   module Enrichers

data/lib/mihari/errors.rb

@@ -8,4 +8,10 @@ module Mihari
   class RetryableError < Error; end
 
   class FileNotFoundError < Error; end
+
+  class HttpError < Error; end
+
+  class FeedParseError < Error; end
+
+  class RuleValidationError < Error; end
 end

data/lib/mihari/feed/parser.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "jr/cli/core_ext"
+
+module Mihari
+  module Feed
+    class Parser
+      attr_reader :data
+
+      #
+      # @param [Array<Hash>, Array<Array<String>>] data
+      #
+      def initialize(data)
+        @data = data
+      end
+
+      #
+      # Parse data by selector
+      #
+      # @param [String] selector
+      #
+      # @return [Array<String>]
+      #
+      def parse(selector)
+        parsed = data.instance_eval(selector)
+
+        raise FeedParseError unless parsed.is_a?(Array) || parsed.is_a?(Enumerator)
+        raise FeedParseError unless parsed.all?(String)
+
+        parsed.to_a
+      end
+    end
+  end
+end

data/lib/mihari/feed/reader.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Mihari
+  module Feed
+    class Reader
+      attr_reader :uri, :http_request_headers, :http_request_method, :http_request_payload_type, :http_request_payload
+
+      def initialize(uri, http_request_headers: {}, http_request_method: "GET", http_request_payload_type: nil, http_request_payload: {})
+        @uri = Addressable::URI.parse(uri)
+        @http_request_headers = http_request_headers
+        @http_request_method = http_request_method
+        @http_request_payload_type = http_request_payload_type
+        @http_request_payload = http_request_payload
+      end
+
+      def read
+        return read_file(uri.path) if uri.scheme == "file"
+
+        return get if http_request_method == "GET"
+
+        post
+      end
+
+      def get
+        uri.query = Addressable::URI.form_encode(http_request_payload)
+        get = Net::HTTP::Get.new(uri)
+
+        request(get)
+      end
+
+      def post
+        post = Net::HTTP::Post.new(uri)
+
+        case http_request_payload_type
+        when "application/json"
+          post.body = JSON.generate(http_request_payload)
+        when "application/x-www-form-urlencoded"
+          post.set_form_data(http_request_payload)
+        end
+
+        request(post)
+      end
+
+      #
+      # Convert text as JSON
+      #
+      # @param [String] text
+      #
+      # @return [Array<Hash>]
+      #
+      def convert_as_json(text)
+        data = JSON.parse(text, symbolize_names: true)
+        return data if data.is_a?(Array)
+
+        [data]
+      end
+
+      #
+      # Convert text as CSV
+      #
+      # @param [String] text
+      #
+      # @return [Array<Hash>]
+      #
+      def convert_as_csv(text)
+        text_without_comments = text.lines.reject { |line| line.start_with? "#" }.join("\n")
+
+        CSV.new(text_without_comments).to_a.reject(&:empty?)
+      end
+
+      def https_options
+        return { use_ssl: true } if uri.scheme == "https"
+
+        {}
+      end
+
+      #
+      # Make a HTTP request
+      #
+      # @param [Net::HTTPRequest] req
+      #
+      # @return [Array<Hash>]
+      #
+      def request(req)
+        Net::HTTP.start(uri.host, uri.port, https_options) do |http|
+          # set headers
+          http_request_headers.each do |k, v|
+            req[k] = v
+          end
+
+          response = http.request(req)
+
+          code = response.code.to_i
+          raise HttpError, "Unsupported response code returned: #{code}" if code != 200
+
+          body = response.body
+
+          content_type = response["Content-Type"].to_s
+          if content_type.include?("application/json")
+            convert_as_json(body)
+          else
+            convert_as_csv(body)
+          end
+        end
+      end
+
+      #
+      # Read & convert a file
+      #
+      # @param [String] path
+      #
+      # @return [Array<Hash>]
+      #
+      def read_file(path)
+        text = File.read(path)
+
+        if path.end_with?(".json")
+          convert_as_json text
+        else
+          convert_as_csv text
+        end
+      end
+    end
+  end
+end

data/lib/mihari/mixins/autonomous_system.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Mixins
     module AutonomousSystem

data/lib/mihari/mixins/database.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Mixins
+    module Database
+      def with_db_connection
+        Mihari::Database.connect
+        yield
+      ensure
+        Mihari::Database.close
+      end
+    end
+  end
+end

data/lib/mihari/mixins/disallowed_data_value.rb

@@ -1,9 +1,9 @@
-require "mem"
+# frozen_string_literal: true
 
 module Mihari
   module Mixins
     module DisallowedDataValue
-      include Mem
+      include Memist::Memoizable
 
       #
       # Normalize a value as a disallowed data value
@@ -19,7 +19,6 @@ module Mihari
         value_without_slashes = value[1..-2]
         Regexp.compile value_without_slashes.to_s
       end
-
       memoize :normalize_disallowed_data_value
 
       #

data/lib/mihari/mixins/rule.rb

@@ -1,51 +1,56 @@
+# frozen_string_literal: true
+
 require "date"
-require "pathname"
-require "yaml"
 require "erb"
+require "pathname"
 
 module Mihari
   module Mixins
     module Rule
+      include Mixins::Database
+
       #
       # Load rule into hash
       #
-      # @param [String] path Path to YAML file or YAML string
+      # @param [String] path_or_id Path to YAML file or YAML string or ID of a rule in the database
       #
-      # @return [Hash]
+      # @return [Mihari::Structs::Rule::Rule]
       #
-      def load_rule(path)
+      def load_rule(path_or_id)
+        data = nil
+
+        data = load_data_from_file(path_or_id) if File.exist?(path_or_id)
+        data = load_data_from_db(path_or_id) if data.nil?
+
+        Structs::Rule::Rule.new(data)
+      end
+
+      def load_data_from_file(path)
         return YAML.safe_load(File.read(path), permitted_classes: [Date], symbolize_names: true) if Pathname(path).exist?
 
         YAML.safe_load(path, symbolize_names: true)
       end
 
+      def load_data_from_db(id)
+        with_db_connection do
+          rule = Mihari::Rule.find(id)
+          rule.data
+        rescue ActiveRecord::RecordNotFound
+          raise ArgumentError, "ID:#{id} is not found in the database"
+        end
+      end
+
       #
-      # Validate rule schema and return a normalized rule
-      #
-      # @param [Hash] rule
+      # Validate a rule
       #
-      # @return [Hash]
+      # @param [Mihari::Structs::Rule::Rule] rule
       #
-      def validate_rule(rule)
-        error_message = "Failed to parse the input as a rule!"
-
-        contract = Schemas::RuleContract.new
-        begin
-          result = contract.call(rule)
-          unless result.errors.empty?
-            messages = result.errors.messages.map do |message|
-              path = message.path.map(&:to_s).join
-              "#{path} #{message.text}"
-            end
-            puts error_message.colorize(:red)
-            raise ArgumentError, messages.join("\n")
-          end
-        rescue NoMethodError
-          puts error_message.colorize(:red)
-          raise ArgumentError, "Invalid rule schema"
-        end
-
-        result.to_h
+      def validate_rule!(rule)
+        rule.validate!
+      rescue RuleValidationError => e
+        error_message = "Failed to parse the input as a rule:\n#{e.message}"
+        puts error_message.colorize(:red)
+        raise e
       end
 
       #
@@ -60,8 +65,8 @@ module Mihari
         data = template.result
 
         # validate the template of rule for just in case
-        rule = YAML.safe_load(data, permitted_classes: [Date], symbolize_names: true)
-        validate_rule rule
+        hashed_data = YAML.safe_load(data, permitted_classes: [Date], symbolize_names: true)
+        Structs::Rule::Rule.new(hashed_data)
 
         data
       end

data/lib/mihari/models/alert.rb

@@ -1,21 +1,20 @@
 # frozen_string_literal: true
 
-require "active_record"
-require "active_record/filter"
-
 module Mihari
   class Alert < ActiveRecord::Base
     has_many :taggings, dependent: :destroy
     has_many :artifacts, dependent: :destroy
     has_many :tags, through: :taggings
 
+    belongs_to :rule
+
     class << self
       #
       # Search alerts
       #
       # @param [Structs::Alert::SearchFilterWithPagination] filter
       #
-      # @return [Array<Hash>]
+      # @return [Array<Alert>]
       #
       def search(filter)
         limit = filter.limit.to_i
@@ -55,7 +54,7 @@ module Mihari
         artifact = artifact.where(dns_records: { value: filter.dns_record }) if filter.dns_record
         artifact = artifact.where(reverse_dns_names: { name: filter.reverse_dns_name }) if filter.reverse_dns_name
         # get artifact ids if there is any valid filter for artifact
-        if filter.has_valid_artifact_filters
+        if filter.valid_artifact_filters?
           artifact_ids = artifact.pluck(:id)
           # set invalid ID if nothing is matched with the filters
           artifact_ids = [-1] if artifact_ids.empty?
@@ -70,10 +69,10 @@ module Mihari
         relation = relation.where(source: filter.source) if filter.source
         relation = relation.where(title: filter.title) if filter.title
 
-        relation = relation.filter(description: { like: "%#{filter.description}%" }) if filter.description
+        relation = relation.where("description LIKE ?", "%#{filter.description}%") if filter.description
 
-        relation = relation.filter(created_at: { gte: filter.from_at }) if filter.from_at
-        relation = relation.filter(created_at: { lte: filter.to_at }) if filter.to_at
+        relation = relation.where("alerts.created_at >= ?", filter.from_at) if filter.from_at
+        relation = relation.where("alerts.created_at <= ?", filter.to_at) if filter.to_at
 
         relation
       end

data/lib/mihari/models/artifact.rb

@@ -1,11 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-require "active_record/filter"
-require "active_support/core_ext/integer/time"
-require "active_support/core_ext/numeric/time"
-require "uri"
-
 class ArtifactValidator < ActiveModel::Validator
   def validate(record)
     return if record.data_type
@@ -119,7 +113,7 @@ module Mihari
     def normalize_as_domain(url_or_domain)
       return url_or_domain if data_type == "domain"
 
-      URI.parse(url_or_domain).host
+      Addressable::URI.parse(url_or_domain).host
     end
 
     def can_enrich_whois?

data/lib/mihari/models/autonomous_system.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-
 module Mihari
   class AutonomousSystem < ActiveRecord::Base
     belongs_to :artifact

data/lib/mihari/models/dns.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-require "resolv"
-
 module Mihari
   class DnsRecord < ActiveRecord::Base
     belongs_to :artifact
@@ -37,7 +34,7 @@ module Mihari
         resources = Resolv::DNS.new.getresources(domain, resource_type)
         resource_name = resource_type.to_s.split("::").last
 
-        resources.map do |resource|
+        resources.filter_map do |resource|
           # A, AAAA
           if resource.respond_to?(:address)
             new(resource: resource_name, value: resource.address.to_s)
@@ -48,7 +45,7 @@ module Mihari
           elsif resource.respond_to?(:data)
             new(resource: resource_name, value: resource.data.to_s)
           end
-        end.compact
+        end
       end
     end
   end

data/lib/mihari/models/geolocation.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
 require "normalize_country"
 
 module Mihari

data/lib/mihari/models/reverse_dns.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-require "resolv"
-
 module Mihari
   class ReverseDnsName < ActiveRecord::Base
     belongs_to :artifact

data/lib/mihari/models/rule.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Mihari
+  class Rule < ActiveRecord::Base
+    has_many :alerts, foreign_key: :source
+
+    #
+    # Returns a hash representative
+    #
+    # @return [Hash]
+    #
+    def to_h
+      symbolized_data = data.deep_symbolize_keys
+      h = { id: id, created_at: created_at, yaml: symbolized_data.to_yaml }
+      h.merge symbolized_data
+    end
+
+    class << self
+      #
+      # Search rules
+      #
+      # @param [Structs::Rule::SearchFilterWithPagination] filter
+      #
+      # @return [Array<Rule>]
+      #
+      def search(filter)
+        limit = filter.limit.to_i
+        raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
+
+        page = filter.page.to_i
+        raise ArgumentError, "page should be bigger than zero" unless page.positive?
+
+        offset = (page - 1) * limit
+
+        relation = build_relation(filter.without_pagination)
+
+        # TODO: improve queires
+        rule_ids = relation.limit(limit).offset(offset).order(created_at: :desc).pluck(:id).uniq
+        where(id: [rule_ids]).order(created_at: :desc)
+      end
+
+      #
+      # Count alerts
+      #
+      # @param [Structs::Rule::SearchFilterWithPagination] filter
+      #
+      # @return [Integer]
+      #
+      def count(filter)
+        relation = build_relation(filter)
+        relation.distinct("rules.id").count
+      end
+
+      private
+
+      def build_relation(filter)
+        relation = self
+        relation = relation.includes(alerts: :tags)
+
+        relation = relation.where(alerts: { tags: { name: filter.tag_name } }) if filter.tag_name
+
+        relation = relation.where(title: filter.title) if filter.title
+
+        relation = relation.where("rules.description LIKE ?", "%#{filter.description}%") if filter.description
+
+        relation = relation.where("rules.created_at >= ?", filter.from_at) if filter.from_at
+        relation = relation.where("rules.created_at <= ?", filter.to_at) if filter.to_at
+
+        relation
+      end
+    end
+  end
+end

data/lib/mihari/models/tag.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-
 module Mihari
   class Tag < ActiveRecord::Base
     has_many :taggings, dependent: :destroy

data/lib/mihari/models/tagging.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-
 module Mihari
   class Tagging < ActiveRecord::Base
     belongs_to :alert

data/lib/mihari/models/whois.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-require "active_record"
 require "whois-parser"
-require "public_suffix"
 
 module Mihari
   class WhoisRecord < ActiveRecord::Base

data/lib/mihari/notifiers/exception_notifier.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-
 module Mihari
   module Notifiers
     class ExceptionNotifier

data/lib/mihari/schemas/analyzer.rb

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require "dry/schema"
-require "dry/validation"
-
-require "mihari/schemas/macros"
-
 module Mihari
   module Schemas
     AnalyzerRun = Dry::Schema.Params do

data/lib/mihari/schemas/macros.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "dry/types"
-
 module Dry
   module Schema
     module Macros

data/lib/mihari/schemas/rule.rb

@@ -1,39 +1,53 @@
 # frozen_string_literal: true
 
-require "dry/schema"
-require "dry/validation"
-
-require "mihari/schemas/macros"
-
 module Mihari
   module Schemas
+    AnalyzerOptions = Dry::Schema.Params do
+      optional(:interval).value(:integer)
+    end
+
     Analyzer = Dry::Schema.Params do
       required(:analyzer).value(Types::AnalyzerTypes)
       required(:query).value(:string)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Spyse = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("spyse"))
       required(:query).value(:string)
       required(:type).value(Types::String.enum("ip", "domain"))
+      optional(:options).hash(AnalyzerOptions)
     end
 
     ZoomEye = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("zoomeye"))
       required(:query).value(:string)
       required(:type).value(Types::String.enum("host", "web"))
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Crtsh = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("crtsh"))
       required(:query).value(:string)
       optional(:exclude_expired).value(:bool).default(true)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Urlscan = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("urlscan"))
       required(:query).value(:string)
-      optional(:use_similarity).value(:bool).default(true)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
+    Feed = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("feed"))
+      required(:query).value(:string)
+      required(:http_request_method).value(Types::FeedHttpRequestMethods).default("GET")
+      required(:http_request_headers).value(:hash).default({})
+      required(:http_request_payload).value(:hash).default({})
+      required(:selector).value(:string)
+      optional(:http_request_payload_type).value(Types::FeedHttpRequestPayloadTypes)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Rule = Dry::Schema.Params do
@@ -47,7 +61,7 @@ module Mihari
       optional(:created_on).value(:date)
       optional(:updated_on).value(:date)
 
-      required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh }
+      required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh | Feed }
 
       optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
       optional(:disallowed_data_values).value(array[:string]).default([])