mihari 4.11.0 → 5.0.0

data/lib/mihari/structs/rule.rb

@@ -4,6 +4,7 @@ require "date"
 require "erb"
 require "json"
 require "pathname"
+require "securerandom"
 require "yaml"
 
 module Mihari
@@ -12,24 +13,16 @@ module Mihari
       # @return [Hash]
       attr_reader :data
 
-      # @return [String]
-      attr_reader :yaml
-
       # @return [Array, nil]
       attr_reader :errors
 
-      # @return [String]
-      attr_writer :id
-
       #
       # Initialize
       #
       # @param [Hash] data
-      # @param [String] yaml
       #
-      def initialize(data, yaml)
+      def initialize(data)
         @data = data.deep_symbolize_keys
-        @yaml = yaml
 
         @errors = nil
 
@@ -70,7 +63,7 @@ module Mihari
       # @return [String]
       #
       def id
-        @id ||= data[:id] || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, data.to_yaml).to_s
+        @id ||= data[:id]
       end
 
       #
@@ -87,16 +80,71 @@ module Mihari
         @description ||= data[:description]
       end
 
+      #
+      # @return [String]
+      #
+      def yaml
+        @yaml ||= data.deep_stringify_keys.to_yaml
+      end
+
+      #
+      # @return [Array<Hash>]
+      #
+      def queries
+        @queries ||= data[:queries]
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def data_types
+        @data_types ||= data[:data_types]
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def tags
+        @tags ||= data[:tags]
+      end
+
+      #
+      # @return [Array<String>]
+      #
+      def falsepositives
+        @falsepositives ||= data[:falsepositives]
+      end
+
+      #
+      # @return [Array<Hash>]
+      #
+      def emitters
+        @emitters ||= data[:emitters]
+      end
+
+      #
+      # @return [Array<Hash>]
+      #
+      def enrichers
+        @enrichers ||= data[:enrichers]
+      end
+
+      #
+      # @return [Integer, nil]
+      #
+      def artifact_lifetime
+        @artifact_lifetime ||= data[:artifact_lifetime]
+      end
+
       #
       # @return [Mihari::Rule]
       #
-      def to_model
+      def model
         rule = Mihari::Rule.find(id)
 
         rule.title = title
         rule.description = description
         rule.data = data
-        rule.yaml = yaml
 
         rule
       rescue ActiveRecord::RecordNotFound
@@ -104,118 +152,83 @@ module Mihari
           id: id,
           title: title,
           description: description,
-          data: data,
-          yaml: yaml
+          data: data
         )
       end
 
       #
       # @return [Mihari::Analyzers::Rule]
       #
-      def to_analyzer
-        analyzer = Mihari::Analyzers::Rule.new(
-          title: self[:title],
-          description: self[:description],
-          tags: self[:tags],
-          queries: self[:queries],
-          allowed_data_types: self[:allowed_data_types],
-          disallowed_data_values: self[:disallowed_data_values],
-          emitters: self[:emitters],
-          enrichers: self[:enrichers],
-          id: id
-        )
-        analyzer.ignore_old_artifacts = self[:ignore_old_artifacts]
-        analyzer.ignore_threshold = self[:ignore_threshold]
-
-        analyzer
+      def analyzer
+        Mihari::Analyzers::Rule.new(rule: self)
       end
 
       class << self
         include Mixins::Database
 
         #
-        # @param [Mihari::Rule] model
-        #
-        # @return [Mihari::Structs::Rule]
-        #
-        def from_model(model)
-          data = model.data.deep_symbolize_keys
-          # set ID if YAML data do not have ID
-          data[:id] = model.id unless data.key?(:id)
-
-          Structs::Rule.new(data, model.yaml)
-        end
-
+        # Load rule from YAML string
         #
         # @param [String] yaml
         #
         # @return [Mihari::Structs::Rule]
-        # @param [String, nil] id
         #
-        def from_yaml(yaml, id: nil)
-          data = load_erb_yaml(yaml)
-          # set ID if id is given & YAML data do not have ID
-          data[:id] = id if !id.nil? && !data.key?(:id)
-
-          Structs::Rule.new(data, yaml)
+        def from_yaml(yaml)
+          Structs::Rule.new YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
+        rescue Psych::SyntaxError => e
+          raise YAMLSyntaxError, e.message
         end
 
         #
-        # @param [String] path_or_id Path to YAML file or YAML string or ID of a rule in the database
+        # @param [Mihari::Rule] model
         #
         # @return [Mihari::Structs::Rule]
         #
-        def from_path_or_id(path_or_id)
-          yaml = nil
-
-          yaml = load_yaml_from_file(path_or_id) if File.exist?(path_or_id)
-          yaml = load_yaml_from_db(path_or_id) if yaml.nil?
-
-          Structs::Rule.from_yaml yaml
+        def from_model(model)
+          Structs::Rule.new(model.data)
         end
 
-        private
-
         #
-        # Load ERR templated YAML
+        # Load a rule from path
         #
-        # @param [String] yaml
+        # @param [String] path
         #
-        # @return [Hash]
+        # @return [Mihari::Structs::Rule, nil]
         #
-        def load_erb_yaml(yaml)
-          YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol], symbolize_names: true)
-        rescue Psych::SyntaxError => e
-          raise YAMLSyntaxError, e.message
+        def from_path(path)
+          return nil unless Pathname(path).exist?
+
+          from_yaml File.read(path)
         end
 
         #
-        # Load YAML string from path
+        # Load a rule from DB
         #
-        # @param [String] path
+        # @param [String] id
         #
-        # @return [String, nil]
+        # @return [Mihari::Structs::Rule, nil]
         #
-        def load_yaml_from_file(path)
-          return nil unless Pathname(path).exist?
+        def from_id(id)
+          with_db_connection do
+            return nil unless Mihari::Rule.exists?(id)
 
-          File.read path
+            Structs::Rule.from_model Mihari::Rule.find(id)
+          end
         end
 
         #
-        # Load YAML string from the database
-        #
-        # @param [String] id <description>
+        # @param [String] path_or_id Path to YAML file or YAML string or ID of a rule in the database
         #
-        # @return [Hash]
+        # @return [Mihari::Structs::Rule]
         #
-        def load_yaml_from_db(id)
-          with_db_connection do
-            rule = Mihari::Rule.find(id)
-            rule.yaml || rule.symbolized_data.to_yaml
-          rescue ActiveRecord::RecordNotFound
-            raise ArgumentError, "ID:#{id} is not found in the database"
-          end
+        def from_path_or_id(path_or_id)
+          rule = from_path(path_or_id)
+          return rule unless rule.nil?
+
+          rule = from_id(path_or_id)
+          return rule unless rule.nil?
+
+          raise ArgumentError, "#{path_or_id} does not exist"
         end
       end
     end

data/lib/mihari/templates/rule.yml.erb

@@ -1,23 +1,5 @@
-title: ... # String (required)
-description: ... # String (required)
-
-id: ... # String (optional)
-author: ... # String (optional)
-created_on: <%= Date.today %> # Date (optional)
-updated_on: <%= Date.today %> # Date (optional)
-
-tags: [] # Array<String> (Optional, defaults to [])
-allowed_data_types: # Array<String> (Optional, defaults to ["hash", "ip", "domain", "url", "mail"])
-  - hash
-  - ip
-  - domain
-  - url
-  - mail
-disallowed_data_values: [] # Array<String> (Optional, defaults to [])
-
-ignore_old_artifacts: true # Whether to ignore old artifacts from checking or not (Optional, defaults to true)
-ignore_threshold: 0 # Number of days to define whether an artifact is old or not (Optional, defaults to 0)
-
-queries: # Array<Hash> (required)
-  - analyzer: shodan # String (required)
-    query: ... # String (required)
+id: <%= SecureRandom.uuid %>
+title: Title goes here
+description: Description goes here
+created_on: <%= Date.today %>
+queries: []

data/lib/mihari/types.rb

@@ -12,7 +12,7 @@ module Mihari
     Double = Strict::Float | Strict::Integer
     DateTime = Strict::DateTime
 
-    DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
+    DataTypes = Types::String.enum(*DEFAULT_DATA_TYPES)
 
     HTTPRequestMethods = Types::String.enum("GET", "POST")
     HTTPRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "4.11.0"
+  VERSION = "5.0.0"
 end

data/lib/mihari/web/api.rb

@@ -6,7 +6,6 @@ require "mihari/web/endpoints/artifacts"
 require "mihari/web/endpoints/configs"
 require "mihari/web/endpoints/ip_addresses"
 require "mihari/web/endpoints/rules"
-require "mihari/web/endpoints/sources"
 require "mihari/web/endpoints/tags"
 
 module Mihari
@@ -19,7 +18,6 @@ module Mihari
     mount Endpoints::Configs
     mount Endpoints::IPAddresses
     mount Endpoints::Rules
-    mount Endpoints::Sources
     mount Endpoints::Tags
 
     add_swagger_documentation(api_version: "v1", info: { title: "Mihari API" })

data/lib/mihari/web/app.rb

@@ -31,7 +31,7 @@ module Mihari
           use Rack::Cors do
             allow do
               origins "*"
-              resource "*", headers: :any, methods: [:get, :post, :put, :delete, :options]
+              resource "*", headers: :any, methods: %i[get post put delete options]
             end
           end
 
@@ -42,7 +42,7 @@ module Mihari
         end.to_app
       end
 
-      def run!(port: 9292, host: "localhost", threads: "1:1", verbose: false)
+      def run!(port: 9292, host: "localhost", threads: "0:5", verbose: false, worker_timeout: 60)
         url = "http://#{host}:#{port}"
 
         # set maximum number of threads to use as PARALLEL_PROCESSOR_COUNT (if it is not set)
@@ -50,8 +50,14 @@ module Mihari
         # TODO: is this the best way?
         _min_thread, max_thread = threads.split(":")
         ENV["PARALLEL_PROCESSOR_COUNT"] = max_thread if ENV["PARALLEL_PROCESSOR_COUNT"].nil?
-
-        Rack::Handler::Puma.run(instance, Port: port, Host: host, Threads: threads, Verbose: verbose) do |_launcher|
+        Rack::Handler::Puma.run(
+          instance,
+          Port: port,
+          Host: host,
+          Threads: threads,
+          Verbose: verbose,
+          worker_timeout: worker_timeout
+        ) do |_launcher|
           Launchy.open(url) if ENV["RACK_ENV"] != "development"
         rescue Launchy::CommandNotFoundError
           # ref. https://github.com/ninoseki/mihari/issues/477

data/lib/mihari/web/endpoints/alerts.rb

@@ -6,7 +6,7 @@ module Mihari
       namespace :alerts do
         desc "Search alerts", {
           is_array: true,
-          success: Entities::Alert,
+          success: Entities::AlertsWithPagination,
           failure: [{ code: 404, message: "Not found", model: Entities::Message }],
           summary: "Search alerts"
         }
@@ -15,7 +15,7 @@ module Mihari
 
           optional :artifact, type: String
           optional :description, type: String
-          optional :source, type: String
+          optional :rule_id, type: String
           optional :tag, type: String
           optional :title, type: String
 
@@ -47,7 +47,15 @@ module Mihari
           alerts = Mihari::Alert.search(search_filter_with_pagenation)
           total = Mihari::Alert.count(search_filter_with_pagenation.without_pagination)
 
-          present({ alerts: alerts, total: total, current_page: page, page_size: limit }, with: Entities::AlertsWithPagination)
+          present(
+            {
+              alerts: alerts,
+              total: total,
+              current_page: page,
+              page_size: limit
+            },
+            with: Entities::AlertsWithPagination
+          )
         end
 
         desc "Delete an alert", {

data/lib/mihari/web/endpoints/configs.rb

@@ -10,12 +10,7 @@ module Mihari
           summary: "Get configs"
         }
         get "/" do
-          statuses = Status.check
-
-          configs = statuses.map do |key, value|
-            { name: key, status: value }
-          end
-          present(configs, with: Entities::Config)
+          present(Mihari.configs, with: Entities::Config)
         end
       end
     end

data/lib/mihari/web/endpoints/rules.rb

@@ -4,9 +4,19 @@ module Mihari
   module Endpoints
     class Rules < Grape::API
       namespace :rules do
+        desc "Get Rule IDs", {
+          is_array: true,
+          success: Entities::RuleIDs,
+          summary: "Get rule IDs"
+        }
+        get "/ids" do
+          rule_ids = Mihari::Rule.distinct.pluck(:id)
+          present({ rule_ids: rule_ids }, with: Entities::RuleIDs)
+        end
+
         desc "Search rules", {
           is_array: true,
-          success: Entities::Rule,
+          success: Entities::RulesWithPagination,
           failure: [{ code: 404, message: "Not found", model: Entities::Message }],
           summary: "Search rules"
         }
@@ -40,7 +50,13 @@ module Mihari
           rules = Mihari::Rule.search(search_filter_with_pagenation)
           total = Mihari::Rule.count(search_filter_with_pagenation.without_pagination)
 
-          present({ rules: rules.map(&:to_h), total: total, current_page: page, page_size: limit }, with: Entities::RulesWithPagination)
+          present(
+            { rules: rules,
+              total: total,
+              current_page: page,
+              page_size: limit },
+            with: Entities::RulesWithPagination
+          )
         end
 
         desc "Get a rule", {
@@ -60,7 +76,7 @@ module Mihari
             error!({ message: "ID:#{id} is not found" }, 404)
           end
 
-          present rule.to_h, with: Entities::Rule
+          present rule, with: Entities::Rule
         end
 
         desc "Run a rule", {
@@ -74,14 +90,12 @@ module Mihari
           id = params["id"].to_s
 
           begin
-            rule = Mihari::Rule.find(id)
+            rule = Mihari::Structs::Rule.from_model(Mihari::Rule.find(id))
           rescue ActiveRecord::RecordNotFound
             error!({ message: "ID:#{id} is not found" }, 404)
           end
 
-          struct = Mihari::Structs::Rule.from_model(rule)
-          analyzer = struct.to_analyzer
-          analyzer.run
+          rule.analyzer.run
 
           status 201
           present({ message: "ID:#{id} is ran successfully" }, with: Entities::Message)
@@ -121,14 +135,13 @@ module Mihari
           end
 
           begin
-            model = rule.to_model
-            model.save
+            rule.model.save
           rescue ActiveRecord::RecordNotUnique
             error!({ message: "ID:#{rule.id} is already registered" }, 400)
           end
 
           status 201
-          present model.to_h, with: Entities::Rule
+          present rule.model, with: Entities::Rule
         end
 
         desc "Update a rule", {
@@ -150,7 +163,7 @@ module Mihari
           end
 
           begin
-            rule = Structs::Rule.from_yaml(yaml, id: id)
+            rule = Structs::Rule.from_yaml(yaml)
           rescue YAMLSyntaxError => e
             error!({ message: e.message }, 400)
           end
@@ -165,14 +178,13 @@ module Mihari
           end
 
           begin
-            model = rule.to_model
-            model.save
+            rule.model.save
           rescue ActiveRecord::RecordNotUnique
-            error!({ message: "ID:#{model.id} is already registered" }, 400)
+            error!({ message: "ID:#{id} is already registered" }, 400)
           end
 
           status 201
-          present model.to_h, with: Entities::Rule
+          present rule.model, with: Entities::Rule
         end
 
         desc "Delete a rule", {