mihari 3.12.0 → 4.0.0

data/lib/mihari/models/geolocation.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
 require "normalize_country"
 
 module Mihari

data/lib/mihari/models/reverse_dns.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-require "resolv"
-
 module Mihari
   class ReverseDnsName < ActiveRecord::Base
     belongs_to :artifact

data/lib/mihari/models/rule.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Mihari
+  class Rule < ActiveRecord::Base
+    has_many :alerts, foreign_key: :source
+
+    #
+    # Returns a hash representative
+    #
+    # @return [Hash]
+    #
+    def to_h
+      symbolized_data = data.deep_symbolize_keys
+      h = { id: id, created_at: created_at, yaml: symbolized_data.to_yaml }
+      h.merge symbolized_data
+    end
+
+    class << self
+      #
+      # Search rules
+      #
+      # @param [Structs::Rule::SearchFilterWithPagination] filter
+      #
+      # @return [Array<Rule>]
+      #
+      def search(filter)
+        limit = filter.limit.to_i
+        raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
+
+        page = filter.page.to_i
+        raise ArgumentError, "page should be bigger than zero" unless page.positive?
+
+        offset = (page - 1) * limit
+
+        relation = build_relation(filter.without_pagination)
+
+        # TODO: improve queires
+        rule_ids = relation.limit(limit).offset(offset).order(created_at: :desc).pluck(:id).uniq
+        where(id: [rule_ids]).order(created_at: :desc)
+      end
+
+      #
+      # Count alerts
+      #
+      # @param [Structs::Rule::SearchFilterWithPagination] filter
+      #
+      # @return [Integer]
+      #
+      def count(filter)
+        relation = build_relation(filter)
+        relation.distinct("rules.id").count
+      end
+
+      private
+
+      def build_relation(filter)
+        relation = self
+        relation = relation.includes(alerts: :tags)
+
+        relation = relation.where(alerts: { tags: { name: filter.tag_name } }) if filter.tag_name
+
+        relation = relation.where(title: filter.title) if filter.title
+
+        relation = relation.where("rules.description LIKE ?", "%#{filter.description}%") if filter.description
+
+        relation = relation.where("rules.created_at >= ?", filter.from_at) if filter.from_at
+        relation = relation.where("rules.created_at <= ?", filter.to_at) if filter.to_at
+
+        relation
+      end
+    end
+  end
+end

data/lib/mihari/models/tag.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-
 module Mihari
   class Tag < ActiveRecord::Base
     has_many :taggings, dependent: :destroy

data/lib/mihari/models/tagging.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "active_record"
-
 module Mihari
   class Tagging < ActiveRecord::Base
     belongs_to :alert

data/lib/mihari/models/whois.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-require "active_record"
 require "whois-parser"
-require "public_suffix"
 
 module Mihari
   class WhoisRecord < ActiveRecord::Base

data/lib/mihari/notifiers/exception_notifier.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-
 module Mihari
   module Notifiers
     class ExceptionNotifier

data/lib/mihari/schemas/analyzer.rb

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require "dry/schema"
-require "dry/validation"
-
-require "mihari/schemas/macros"
-
 module Mihari
   module Schemas
     AnalyzerRun = Dry::Schema.Params do

data/lib/mihari/schemas/macros.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "dry/types"
-
 module Dry
   module Schema
     module Macros

data/lib/mihari/schemas/rule.rb

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require "dry/schema"
-require "dry/validation"
-
-require "mihari/schemas/macros"
-
 module Mihari
   module Schemas
     AnalyzerOptions = Dry::Schema.Params do

data/lib/mihari/structs/alert.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "dry/struct"
-
 module Mihari
   module Structs
     module Alert

data/lib/mihari/structs/censys.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "dry/struct"
-
 module Mihari
   module Structs
     module Censys
@@ -34,13 +31,15 @@ module Mihari
         attribute :ip, Types::String
         attribute :location, Location
         attribute :autonomous_system, AutonomousSystem
+        attribute :metadata, Types::Hash
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
             ip: d.fetch("ip"),
             location: Location.from_dynamic!(d.fetch("location")),
-            autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system"))
+            autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system")),
+            metadata: d
           )
         end
       end

data/lib/mihari/structs/greynoise.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "dry/struct"
-
 module Mihari
   module Structs
     module GreyNoise
@@ -24,12 +21,14 @@ module Mihari
       class Datum < Dry::Struct
         attribute :ip, Types::String
         attribute :metadata, Metadata
+        attribute :metadata_, Types::Hash
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
             ip: d.fetch("ip"),
-            metadata: Metadata.from_dynamic!(d.fetch("metadata"))
+            metadata: Metadata.from_dynamic!(d.fetch("metadata")),
+            metadata_: d
           )
         end
       end

data/lib/mihari/structs/ipinfo.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "dry/struct"
-
 module Mihari
   module Structs
     module IPInfo

data/lib/mihari/structs/onyphe.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "dry/struct"
-
 module Mihari
   module Structs
     module Onyphe
@@ -10,6 +7,7 @@ module Mihari
         attribute :asn, Types::String
         attribute :country_code, Types::String.optional
         attribute :ip, Types::String
+        attribute :metadata, Types::Hash
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
@@ -17,7 +15,8 @@ module Mihari
             asn: d.fetch("asn"),
             ip: d.fetch("ip"),
             # Onyphe's country = 2-letter country code
-            country_code: d["country"]
+            country_code: d["country"],
+            metadata: d
           )
         end
       end
@@ -26,7 +25,7 @@ module Mihari
         attribute :count, Types::Int
         attribute :error, Types::Int
         attribute :max_page, Types::Int
-        attribute :page, Types::String
+        attribute :page, Types::Int
         attribute :results, Types.Array(Result)
         attribute :status, Types::String
         attribute :total, Types::Int
@@ -37,7 +36,7 @@ module Mihari
             count: d.fetch("count"),
             error: d.fetch("error"),
             max_page: d.fetch("max_page"),
-            page: d.fetch("page"),
+            page: d.fetch("page").to_i,
             results: d.fetch("results").map { |x| Result.from_dynamic!(x) },
             status: d.fetch("status"),
             total: d.fetch("total")

data/lib/mihari/structs/rule.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    module Rule
+      class SearchFilter < Dry::Struct
+        attribute? :description, Types::String.optional
+        attribute? :tag_name, Types::String.optional
+        attribute? :title, Types::String.optional
+        attribute? :from_at, Types::DateTime.optional
+        attribute? :to_at, Types::DateTime.optional
+      end
+
+      class SearchFilterWithPagination < SearchFilter
+        attribute? :page, Types::Int.default(1)
+        attribute? :limit, Types::Int.default(10)
+
+        def without_pagination
+          SearchFilter.new(
+            description: description,
+            from_at: from_at,
+            tag_name: tag_name,
+            title: title,
+            to_at: to_at
+          )
+        end
+      end
+
+      class Rule
+        attr_reader :data, :errors
+
+        def initialize(data)
+          @data = data.deep_symbolize_keys
+          @errors = nil
+          @no_method_error = nil
+
+          validate
+        end
+
+        #
+        # @return [Boolean]
+        #
+        def errors?
+          return false if @errors.nil?
+
+          !@errors.empty?
+        end
+
+        #
+        # @return [Array<String>]
+        #
+        def error_messages
+          return [] if @errors.nil?
+
+          @errors.messages.map do |message|
+            path = message.path.map(&:to_s).join
+            "#{path} #{message.text}"
+          end
+        end
+
+        def validate
+          begin
+            contract = Schemas::RuleContract.new
+            result = contract.call(data)
+          rescue NoMethodError => e
+            @no_method_error = e
+            return
+          end
+
+          @data = result.to_h
+          @errors = result.errors
+        end
+
+        def validate!
+          raise RuleValidationError, "Data should be a hash" unless data.is_a?(Hash)
+
+          raise RuleValidationError, error_messages.join("\n") if errors?
+
+          raise RuleValidationError, "Something wrong with queries" unless @no_method_error.nil?
+        end
+
+        def [](key)
+          data[key.to_sym]
+        end
+
+        #
+        # @return [String]
+        #
+        def id
+          @id ||= data[:id] || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, data.to_yaml).to_s
+        end
+
+        #
+        # @return [String]
+        #
+        def title
+          @title ||= data[:title]
+        end
+
+        #
+        # @return [String]
+        #
+        def description
+          @description ||= data[:description]
+        end
+
+        #
+        # @return [Mihari::Rule]
+        #
+        def to_model
+          Mihari::Rule.new(
+            id: id,
+            title: title,
+            description: description,
+            data: data
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/shodan.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "dry/struct"
-
 module Mihari
   module Structs
     module Shodan
@@ -25,6 +22,7 @@ module Mihari
         attribute :location, Location
         attribute :domains, Types.Array(Types::String)
         attribute :ip_str, Types::String
+        attribute :metadata, Types::Hash
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
@@ -33,7 +31,8 @@ module Mihari
             hostnames: d.fetch("hostnames"),
             location: Location.from_dynamic!(d.fetch("location")),
             domains: d.fetch("domains"),
-            ip_str: d.fetch("ip_str")
+            ip_str: d.fetch("ip_str"),
+            metadata: d
           )
         end
       end

data/lib/mihari/structs/urlscan.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "dry/struct"
-
 module Mihari
   module Structs
     module Urlscan

data/lib/mihari/structs/virustotal_intelligence.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "dry/struct"
-
 module Mihari
   module Structs
     module VirusTotalIntelligence
@@ -21,6 +18,7 @@ module Mihari
         attribute :type, Types::String
         attribute :id, Types::String
         attribute :context_attributes, ContextAttributes.optional
+        attribute :metadata, Types::Hash
 
         def value
           case type
@@ -44,7 +42,8 @@ module Mihari
           new(
             type: d.fetch("type"),
             id: d.fetch("id"),
-            context_attributes: context_attributes
+            context_attributes: context_attributes,
+            metadata: d
           )
         end
       end

data/lib/mihari/type_checker.rb

@@ -1,11 +1,5 @@
 # frozen_string_literal: true
 
-require "addressable/uri"
-require "dry-initializer"
-require "email_address"
-require "ipaddr"
-require "public_suffix"
-
 module Mihari
   class TypeChecker
     extend Dry::Initializer
@@ -16,6 +10,8 @@ module Mihari
       super
 
       raise ArgumentError if data.is_a?(Hash)
+
+      @data = data.to_s
     end
 
     # @return [Boolean]

data/lib/mihari/types.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "dry/types"
-
 module Mihari
   module Types
     include Dry.Types()

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.12.0"
+  VERSION = "4.0.0"
 end

data/lib/mihari/web/api.rb

@@ -18,12 +18,15 @@ require "mihari/web/entities/artifact"
 
 require "mihari/web/entities/alert"
 
+require "mihari/web/entities/rule"
+
 # Endpoints
 require "mihari/web/endpoints/alerts"
 require "mihari/web/endpoints/artifacts"
 require "mihari/web/endpoints/command"
 require "mihari/web/endpoints/configs"
 require "mihari/web/endpoints/ip_addresses"
+require "mihari/web/endpoints/rules"
 require "mihari/web/endpoints/sources"
 require "mihari/web/endpoints/tags"
 
@@ -37,6 +40,7 @@ module Mihari
     mount Endpoints::Command
     mount Endpoints::Configs
     mount Endpoints::IPAddresses
+    mount Endpoints::Rules
     mount Endpoints::Sources
     mount Endpoints::Tags
 

data/lib/mihari/web/app.rb

@@ -11,6 +11,8 @@ require "grape-entity"
 require "grape-swagger"
 require "grape-swagger-entity"
 
+require "mihari/web/middleware/connection_adapter"
+
 require "mihari/web/api"
 
 module Mihari
@@ -34,6 +36,8 @@ module Mihari
             end
           end
 
+          use Middleware::ConnectionAdapter
+
           run App.new
         end.to_app
       end
@@ -41,14 +45,8 @@ module Mihari
       def run!(port: 9292, host: "localhost", threads: "0:16", verbose: false)
         url = "http://#{host}:#{port}"
 
-        Rack::Handler::Puma.run(instance, Port: port, Host: host, Threads: threads, Verbose: verbose) do |server|
+        Rack::Handler::Puma.run(instance, Port: port, Host: host, Threads: threads, Verbose: verbose) do |_launcher|
           Launchy.open(url) if ENV["RACK_ENV"] != "development"
-
-          [:INT, :TERM].each do |sig|
-            trap(sig) do
-              server.shutdown
-            end
-          end
         end
       end
     end

data/lib/mihari/web/endpoints/alerts.rb

@@ -7,14 +7,17 @@ module Mihari
         desc "Search alerts", {
           is_array: true,
           success: Entities::Alert,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Search alerts"
         }
         params do
           optional :page, type: Integer
+
           optional :artifact, type: String
           optional :description, type: String
           optional :source, type: String
           optional :tag, type: String
+          optional :title, type: String
 
           optional :fromAt, type: DateTime
           optional :toAt, type: DateTime
@@ -38,7 +41,7 @@ module Mihari
           filter["tag_name"] = filter["tag"]
 
           # symbolize hash keys
-          filter = filter.to_h.transform_keys(&:to_sym)
+          filter = filter.to_h.symbolize_keys
 
           search_filter_with_pagenation = Structs::Alert::SearchFilterWithPagination.new(**filter)
           alerts = Mihari::Alert.search(search_filter_with_pagenation)
@@ -49,7 +52,8 @@ module Mihari
 
         desc "Delete an alert", {
           success: Entities::Message,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Delete an alert"
         }
         params do
           requires :id, type: Integer

data/lib/mihari/web/endpoints/artifacts.rb

@@ -6,7 +6,8 @@ module Mihari
       namespace :artifacts do
         desc "Get an artifact", {
           success: Entities::Artifact,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Get an artifact"
         }
         params do
           requires :id, type: Integer
@@ -38,7 +39,8 @@ module Mihari
 
         desc "Enrich an artifact", {
           success: Entities::Message,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Enrich an artifact"
         }
         params do
           requires :id, type: Integer
@@ -67,7 +69,8 @@ module Mihari
 
         desc "Delete an artifact", {
           success: Entities::Message,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Delete an artifact"
         }
         params do
           requires :id, type: Integer

data/lib/mihari/web/endpoints/command.rb

@@ -8,7 +8,8 @@ module Mihari
       namespace :command do
         desc "Run a command", {
           success: Entities::CommandResult,
-          failure: [{ code: 400, message: "Bad request", model: Entities::Message }]
+          failure: [{ code: 400, message: "Bad request", model: Entities::Message }],
+          summary: "Run a command"
         }
         params do
           requires :command, type: String, documentation: { param_type: "body" }

data/lib/mihari/web/endpoints/configs.rb

@@ -6,7 +6,8 @@ module Mihari
       namespace :configs do
         desc "Get configs", {
           is_array: true,
-          success: Entities::Config
+          success: Entities::Config,
+          summary: "Get configs"
         }
         get "/" do
           statuses = Status.check

data/lib/mihari/web/endpoints/ip_addresses.rb

@@ -6,7 +6,8 @@ module Mihari
       namespace :ip_addresses do
         desc "Get an IP address", {
           success: Entities::IPAddress,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Get an IP address"
         }
         params do
           requires :ip, type: String, regexp: /\A[0-9.]+\z/