mihari 6.2.0 → 7.0.0

data/lib/mihari/rule.rb

@@ -2,7 +2,7 @@
 
 module Mihari
   class Rule < Service
-    include Mixins::FalsePositive
+    include Concerns::FalsePositiveValidatable
 
     # @return [Hash]
     attr_reader :data
@@ -84,10 +84,19 @@ module Mihari
     end
 
     #
-    # @return [Array<String>]
+    # @return [Array<Mihari::Models::Tag>]
     #
     def tags
-      data[:tags]
+      data[:tags].uniq.filter_map do |name|
+        Models::Tag.find_or_create_by(name: name)
+      end
+    end
+
+    #
+    # @return [Array<Mihari::Models::Tagging>]
+    #
+    def taggings
+      tags.map { |tag| Models::Tagging.find_or_create_by(tag_id: tag.id, rule_id: id) }
     end
 
     #
@@ -100,8 +109,8 @@ module Mihari
     #
     # @return [Integer, nil]
     #
-    def artifact_lifetime
-      data[:artifact_lifetime] || data[:artifact_ttl]
+    def artifact_ttl
+      data[:artifact_ttl]
     end
 
     #
@@ -111,15 +120,10 @@ module Mihari
     #
     def artifacts
       analyzer_results.flat_map do |result|
-        case result
-        when Success
-          artifacts = result.value!
-          artifacts.map do |artifact|
-            artifact.rule_id = id
-            artifact
-          end
-        else
-          raise result.failure unless analyzer.ignore_error?
+        artifacts = result.value!
+        artifacts.map do |artifact|
+          artifact.rule_id = id
+          artifact
         end
       end
     end
@@ -146,7 +150,7 @@ module Mihari
     #
     def unique_artifacts
       normalized_artifacts.select do |artifact|
-        artifact.unique?(base_time: base_time, artifact_lifetime: artifact_lifetime)
+        artifact.unique?(base_time: base_time, artifact_ttl: artifact_ttl)
       end
     end
 
@@ -206,19 +210,19 @@ module Mihari
     # @return [Mihari::Models::Rule]
     #
     def model
-      rule = Mihari::Models::Rule.find(id)
-
-      rule.title = title
-      rule.description = description
-      rule.data = data
-
-      rule
+      Mihari::Models::Rule.find(id).tap do |rule|
+        rule.title = title
+        rule.description = description
+        rule.data = data
+        rule.taggings = taggings
+      end
     rescue ActiveRecord::RecordNotFound
       Mihari::Models::Rule.new(
         id: id,
         title: title,
         description: description,
-        data: data
+        data: data,
+        taggings: taggings
       )
     end
 
@@ -232,6 +236,13 @@ module Mihari
       false
     end
 
+    #
+    # @return [Boolean]
+    #
+    def exists?
+      Mihari::Models::Rule.exists? id
+    end
+
     def update_or_create
       model.save
     end
@@ -309,7 +320,7 @@ module Mihari
 
     # @return [Array<Dry::Monads::Result::Success<Array<Mihari::Models::Artifact>>, Dry::Monads::Result::Failure>]
     def analyzer_results
-      parallel_results = Parallel.map(parallel_analyzers) { |analyzer| analyzer.result }
+      parallel_results = Parallel.map(parallel_analyzers, &:result)
       serial_results = serial_analyzers.map(&:result)
       parallel_results + serial_results
     end

data/lib/mihari/schemas/alert.rb

@@ -5,6 +5,7 @@ module Mihari
     Alert = Dry::Schema.Params do
       required(:rule_id).value(:string)
       required(:artifacts).value(array[:string])
+      optional(:source).value(:string)
     end
 
     #

data/lib/mihari/schemas/analyzer.rb

@@ -6,7 +6,7 @@ module Mihari
     # Analyzer schemas
     #
     module Analyzers
-      extend Schemas::Mixins
+      extend Concerns::Orrable
 
       # Analyzer with API key and pagination
       [
@@ -110,7 +110,7 @@ module Mihari
         optional(:method).value(Types::HTTPRequestMethods).default("GET")
         optional(:headers).value(:hash).default({})
         optional(:params).value(:hash)
-        optional(:data).value(:hash)
+        optional(:form).value(:hash)
         optional(:json).value(:hash)
         optional(:options).hash(AnalyzerOptions)
       end

data/lib/mihari/schemas/concerns/orrable.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Schemas
+    module Concerns
+      #
+      # OR-rable concern
+      #
+      module Orrable
+        extend ActiveSupport::Concern
+
+        def get_or_composition
+          schemas = constants.map { |sym| const_get sym }
+          return schemas.first if schemas.length <= 1
+
+          base, *others = schemas
+          others.each { |other| base = base.or(other) }
+
+          base
+        end
+      end
+    end
+  end
+end

data/lib/mihari/schemas/emitter.rb

@@ -6,7 +6,7 @@ module Mihari
     # Emitter schemas
     #
     module Emitters
-      extend Schemas::Mixins
+      extend Concerns::Orrable
 
       Database = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::Database.class_keys))
@@ -24,7 +24,6 @@ module Mihari
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::TheHive.class_keys))
         optional(:url).value(:string)
         optional(:api_key).value(:string)
-        optional(:api_version).value(Types::String.enum("v4", "v5")).default("v4")
         optional(:options).hash(Options)
       end
 

data/lib/mihari/schemas/enricher.rb

@@ -6,11 +6,10 @@ module Mihari
     # Enricher schemas
     #
     module Enrichers
-      extend Schemas::Mixins
+      extend Concerns::Orrable
 
-      IPInfo = Dry::Schema.Params do
-        required(:enricher).value(Types::String.enum(*Mihari::Enrichers::IPInfo.class_keys))
-        optional(:api_key).value(:string)
+      MMDB = Dry::Schema.Params do
+        required(:enricher).value(Types::String.enum(*Mihari::Enrichers::MMDB.class_keys))
         optional(:options).hash(Options)
       end
 

data/lib/mihari/schemas/macros.rb

@@ -10,7 +10,7 @@ module Dry
       class DSL
         def default(value)
           schema_dsl.before(:rule_applier) do |result|
-            result.update(name => value) unless result[name]
+            result.update(name => value) if result.output && !result[name]
           end
         end
       end

data/lib/mihari/schemas/options.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "dry/schema"
-
 module Mihari
   module Schemas
     Options = Dry::Schema.Params do

data/lib/mihari/schemas/rule.rb

@@ -29,7 +29,6 @@ module Mihari
       optional(:data_types).value(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
       optional(:falsepositives).value(array[:string]).default([])
 
-      optional(:artifact_lifetime).value(:integer)
       optional(:artifact_ttl).value(:integer)
     end
 
@@ -37,7 +36,7 @@ module Mihari
     # Rule schema contract
     #
     class RuleContract < Dry::Validation::Contract
-      include Mihari::Mixins::FalsePositive
+      include Mihari::Concerns::FalsePositiveValidatable
 
       params(Rule)
 

data/lib/mihari/services/{rule_builder.rb → builders.rb}

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require "date"
-require "erb"
-require "pathname"
-require "yaml"
-
 module Mihari
   module Services
     #
@@ -23,7 +18,7 @@ module Mihari
         result = Try { Mihari::Models::Rule.find path_or_id }.to_result
         return result.value! if result.success?
 
-        raise ArgumentError, "#{path_or_id} does not exist" unless Pathname(path_or_id).exist?
+        raise ArgumentError, "#{path_or_id} not found" unless Pathname(path_or_id).exist?
 
         YAML.safe_load(
           ERB.new(File.read(path_or_id)).result,

data/lib/mihari/services/creators.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    #
+    # Alert creator
+    #
+    class AlertCreator < Service
+      #
+      # @param [Hash] params
+      #
+      # @return [Mihari::Models::Alert]
+      #
+      def call(params)
+        normalized = params.to_snake_keys
+        proxy = Services::AlertProxy.new(**normalized)
+        emitter = Emitters::Database.new(rule: proxy.rule)
+        emitter.call proxy.artifacts
+      end
+    end
+  end
+end

data/lib/mihari/services/destroyers.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class AlertDestroyer < Service
+      #
+      # @param [String] id
+      #
+      def call(id)
+        Models::Alert.find(id).destroy
+      end
+    end
+
+    class ArtifactDestroyer < Service
+      #
+      # @param [Integer] id
+      #
+      def call(id)
+        Models::Artifact.find(id).destroy
+      end
+    end
+
+    class RuleDestroyer < Service
+      #
+      # @param [String] id
+      #
+      def call(id)
+        Models::Rule.find(id).destroy
+      end
+    end
+
+    class TagDestroyer < Service
+      #
+      # @param [Integer] id
+      #
+      def call(id)
+        Models::Tag.find(id).destroy
+      end
+    end
+  end
+end

data/lib/mihari/services/enrichers.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class ArtifactEnricher < Service
+      #
+      # @param [String] id
+      #
+      def call(id)
+        artifact = Mihari::Models::Artifact.includes(
+          :autonomous_system,
+          :geolocation,
+          :whois_record,
+          :dns_records,
+          :reverse_dns_names,
+          :cpes,
+          :ports
+        ).find(id)
+
+        artifact.enrich_all
+        artifact.save
+      end
+    end
+  end
+end

data/lib/mihari/services/feed.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "csv"
+
+require "jr/cli/core_ext"
+
+module Mihari
+  module Services
+    #
+    # Feed reader
+    #
+    class FeedReader < Service
+      #
+      # @param [String] url
+      # @param [Hash, nil] options
+      # @param [String] method
+      # @param [Hash, nil] headers
+      # @param [Hash, nil] params
+      # @param [Hash, nil] json
+      # @param [form, nil] form
+      # @param [String] selector
+      # @param [Integer, nil] timeout
+      #
+      # @return [Array<Hash>]
+      #
+      def call(url, headers: {}, method: "GET", params: nil, json: nil, form: nil, timeout: nil)
+        url = Addressable::URI.parse(url)
+
+        return read_file(url.path) if url.scheme == "file"
+
+        http = HTTP::Factory.build(headers: headers, timeout: timeout)
+
+        res = http.get(url, params: params) if method == "GET"
+        res = http.post(url, params: params, json: json, form: form) if method == "POST"
+
+        body = res.body.to_s
+        content_type = res["Content-Type"].to_s
+        return convert_as_json(body) if content_type.include?("application/json")
+
+        convert_as_csv(body)
+      end
+
+      #
+      # Convert text as JSON
+      #
+      # @param [String] text
+      #
+      # @return [Hash, Array<Object>]
+      #
+      def convert_as_json(text)
+        JSON.parse(text).deep_symbolize_keys
+      end
+
+      #
+      # Convert text as CSV
+      #
+      # @param [String] text
+      #
+      # @return [Array<Object>]
+      #
+      def convert_as_csv(text)
+        text_without_comments = text.lines.reject { |line| line.start_with? "#" }.join("\n")
+
+        CSV.new(text_without_comments).to_a.reject(&:empty?)
+      end
+
+      #
+      # Read & convert a file
+      #
+      # @param [String] path
+      #
+      # @return [Array<Object>]
+      #
+      def read_file(path)
+        text = File.read(path)
+
+        return convert_as_json(text) if path.end_with?(".json")
+
+        convert_as_csv text
+      end
+    end
+
+    #
+    # Feed parser
+    #
+    class FeedParser < Service
+      # Parse data by selector
+      #
+      # @param [Hash, Array<Object>] input_enumerator
+      # @param [String] selector
+      #
+      # @return [Array<String>]
+      #
+      # @param [Object] read_data
+      def call(input_enumerator, selector)
+        parsed = proc do
+          $SAFE = 1
+          input_enumerator.instance_eval(selector)
+        end.call
+
+        raise TypeError unless parsed.all?(String)
+
+        parsed
+      end
+    end
+  end
+end

data/lib/mihari/services/getters.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class ArtifactGetter < Service
+      #
+      # @param [Integer] id
+      #
+      # @return [Mihari::Models::Artifact]
+      #
+      def call(id)
+        Mihari::Models::Artifact.eager_load(
+          :autonomous_system,
+          :geolocation,
+          :whois_record,
+          :dns_records,
+          :reverse_dns_names
+        ).find id
+      end
+    end
+
+    class AlertGetter < Service
+      #
+      # @param [Integer] id
+      #
+      # @return [Mihari::Models::Artifact]
+      #
+      def call(id)
+        Mihari::Models::Alert.eager_load(
+          :artifacts,
+          rule: :tags
+        ).find id
+      end
+    end
+
+    class RuleGetter < Service
+      #
+      # @params [String] id
+      #
+      # @return [Mihari::Models::Rule]
+      #
+      def call(id)
+        Mihari::Models::Rule.find id
+      end
+    end
+
+    class IPGetter < Service
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::Structs::MMDB::Response]
+      #
+      def call(ip)
+        Mihari::Enrichers::MMDB.new.call ip
+      end
+    end
+  end
+end

data/lib/mihari/services/initializers.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class RuleInitializer < Service
+      #
+      # @param [String] path
+      # @param [Dry::Files] files
+      #
+      def call(path, files = Dry::Files.new)
+        rule = Mihari::Rule.new(
+          id: SecureRandom.uuid,
+          title: "Title goes here",
+          description: "Description goes here",
+          created_on: Date.today,
+          queries: []
+        )
+        files.write(path, rule.yaml)
+      end
+    end
+  end
+end

data/lib/mihari/services/{alert_builder.rb → proxies.rb}

@@ -1,11 +1,5 @@
 # frozen_string_literal: true
 
-require "date"
-
-require "erb"
-require "pathname"
-require "yaml"
-
 module Mihari
   module Services
     #
@@ -36,9 +30,9 @@ module Mihari
       # @return [Boolean]
       #
       def errors?
-        return false if @errors.nil?
+        return false if errors.nil?
 
-        !@errors.empty?
+        !errors.empty?
       end
 
       def validate!
@@ -69,10 +63,18 @@ module Mihari
         @artifacts ||= data[:artifacts].map do |data|
           artifact = Models::Artifact.new(data: data)
           artifact.rule_id = rule_id
+          artifact.source = source
           artifact
         end.uniq(&:data).select(&:valid?)
       end
 
+      #
+      # @return [String, nil]
+      #
+      def source
+        @source ||= data[:source]
+      end
+
       #
       # @return [Mihari::Rule]
       #
@@ -83,37 +85,5 @@ module Mihari
         end.first
       end
     end
-
-    #
-    # Alert builder
-    #
-    class AlertBuilder < Service
-      # @return [String]
-      attr_reader :path
-
-      #
-      # @param [String] path
-      #
-      # @return [Hash]
-      #
-      def data
-        raise ArgumentError, "#{path} does not exist" unless Pathname(path).exist?
-
-        YAML.safe_load(
-          ERB.new(File.read(path)).result,
-          permitted_classes: [Date, Symbol]
-        )
-      end
-
-      #
-      # @param [String] path
-      #
-      # @return [Mihari::AlertProxy]
-      #
-      def call(path)
-        @path = path
-        AlertProxy.new(**data)
-      end
-    end
   end
 end