mihari 6.3.0 → 7.0.1

data/lib/mihari/models/tag.rb

@@ -7,7 +7,23 @@ module Mihari
     #
     class Tag < ActiveRecord::Base
       has_many :taggings, dependent: :destroy
-      has_many :tags, through: :taggings
+
+      include SearchCop
+      include Concerns::Searchable
+
+      search_scope :search do
+        attributes :id, :name
+      end
+
+      class << self
+        # @!method search_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Array<Mihari::Models::Tag>]
+
+        # @!method count_by_filter(filter)
+        #   @param [Mihari::Structs::Filters::Search] filter
+        #   @return [Integer]
+      end
     end
   end
 end

data/lib/mihari/models/tagging.rb

@@ -6,7 +6,7 @@ module Mihari
     # Tagging model
     #
     class Tagging < ActiveRecord::Base
-      belongs_to :alert
+      belongs_to :rule
       belongs_to :tag
     end
   end

data/lib/mihari/models/whois.rb

@@ -7,23 +7,6 @@ module Mihari
     #
     class WhoisRecord < ActiveRecord::Base
       belongs_to :artifact
-
-      @memo = {}
-
-      class << self
-        #
-        # Build whois record
-        #
-        # @param [String] domain
-        # @param [Mihari::Enrichers::Whois] enricher
-        #
-        # @return [WhoisRecord, nil]
-        #
-        def build_by_domain(domain, enricher: Enrichers::Whois.new)
-          result = enricher.result(domain)
-          result.value_or nil
-        end
-      end
     end
   end
 end

data/lib/mihari/rule.rb

@@ -2,7 +2,7 @@
 
 module Mihari
   class Rule < Service
-    include Mixins::FalsePositive
+    include Concerns::FalsePositiveValidatable
 
     # @return [Hash]
     attr_reader :data
@@ -84,10 +84,19 @@ module Mihari
     end
 
     #
-    # @return [Array<String>]
+    # @return [Array<Mihari::Models::Tag>]
     #
     def tags
-      data[:tags]
+      data[:tags].uniq.filter_map do |name|
+        Models::Tag.find_or_create_by(name: name)
+      end
+    end
+
+    #
+    # @return [Array<Mihari::Models::Tagging>]
+    #
+    def taggings
+      tags.map { |tag| Models::Tagging.find_or_create_by(tag_id: tag.id, rule_id: id) }
     end
 
     #
@@ -100,8 +109,8 @@ module Mihari
     #
     # @return [Integer, nil]
     #
-    def artifact_lifetime
-      data[:artifact_lifetime] || data[:artifact_ttl]
+    def artifact_ttl
+      data[:artifact_ttl]
     end
 
     #
@@ -111,15 +120,10 @@ module Mihari
     #
     def artifacts
       analyzer_results.flat_map do |result|
-        case result
-        when Success
-          artifacts = result.value!
-          artifacts.map do |artifact|
-            artifact.rule_id = id
-            artifact
-          end
-        else
-          raise result.failure unless analyzer.ignore_error?
+        artifacts = result.value!
+        artifacts.map do |artifact|
+          artifact.rule_id = id
+          artifact
         end
       end
     end
@@ -146,7 +150,7 @@ module Mihari
     #
     def unique_artifacts
       normalized_artifacts.select do |artifact|
-        artifact.unique?(base_time: base_time, artifact_lifetime: artifact_lifetime)
+        artifact.unique?(base_time: base_time, artifact_ttl: artifact_ttl)
       end
     end
 
@@ -206,19 +210,19 @@ module Mihari
     # @return [Mihari::Models::Rule]
     #
     def model
-      rule = Mihari::Models::Rule.find(id)
-
-      rule.title = title
-      rule.description = description
-      rule.data = data
-
-      rule
+      Mihari::Models::Rule.find(id).tap do |rule|
+        rule.title = title
+        rule.description = description
+        rule.data = data
+        rule.taggings = taggings
+      end
     rescue ActiveRecord::RecordNotFound
       Mihari::Models::Rule.new(
         id: id,
         title: title,
         description: description,
-        data: data
+        data: data,
+        taggings: taggings
       )
     end
 
@@ -232,6 +236,13 @@ module Mihari
       false
     end
 
+    #
+    # @return [Boolean]
+    #
+    def exists?
+      Mihari::Models::Rule.exists? id
+    end
+
     def update_or_create
       model.save
     end
@@ -309,7 +320,7 @@ module Mihari
 
     # @return [Array<Dry::Monads::Result::Success<Array<Mihari::Models::Artifact>>, Dry::Monads::Result::Failure>]
     def analyzer_results
-      parallel_results = Parallel.map(parallel_analyzers) { |analyzer| analyzer.result }
+      parallel_results = Parallel.map(parallel_analyzers, &:result)
       serial_results = serial_analyzers.map(&:result)
       parallel_results + serial_results
     end

data/lib/mihari/schemas/alert.rb

@@ -5,6 +5,7 @@ module Mihari
     Alert = Dry::Schema.Params do
       required(:rule_id).value(:string)
       required(:artifacts).value(array[:string])
+      optional(:source).value(:string)
     end
 
     #

data/lib/mihari/schemas/analyzer.rb

@@ -6,7 +6,7 @@ module Mihari
     # Analyzer schemas
     #
     module Analyzers
-      extend Schemas::Mixins
+      extend Concerns::Orrable
 
       # Analyzer with API key and pagination
       [
@@ -91,6 +91,7 @@ module Mihari
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Crtsh.class_keys))
         required(:query).value(:string)
         optional(:exclude_expired).value(:bool).default(true)
+        optional(:match).value(Types::String.enum("=", "ILIKE", "LIKE", "single", "any", "FTS")).default(nil)
         optional(:options).hash(AnalyzerOptions)
       end
 
@@ -110,7 +111,7 @@ module Mihari
         optional(:method).value(Types::HTTPRequestMethods).default("GET")
         optional(:headers).value(:hash).default({})
         optional(:params).value(:hash)
-        optional(:data).value(:hash)
+        optional(:form).value(:hash)
         optional(:json).value(:hash)
         optional(:options).hash(AnalyzerOptions)
       end

data/lib/mihari/schemas/concerns/orrable.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Schemas
+    module Concerns
+      #
+      # OR-rable concern
+      #
+      module Orrable
+        extend ActiveSupport::Concern
+
+        def get_or_composition
+          schemas = constants.map { |sym| const_get sym }
+          return schemas.first if schemas.length <= 1
+
+          base, *others = schemas
+          others.each { |other| base = base.or(other) }
+
+          base
+        end
+      end
+    end
+  end
+end

data/lib/mihari/schemas/emitter.rb

@@ -6,7 +6,7 @@ module Mihari
     # Emitter schemas
     #
     module Emitters
-      extend Schemas::Mixins
+      extend Concerns::Orrable
 
       Database = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::Database.class_keys))
@@ -24,7 +24,6 @@ module Mihari
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::TheHive.class_keys))
         optional(:url).value(:string)
         optional(:api_key).value(:string)
-        optional(:api_version).value(Types::String.enum("v4", "v5")).default("v4")
         optional(:options).hash(Options)
       end
 

data/lib/mihari/schemas/enricher.rb

@@ -6,11 +6,10 @@ module Mihari
     # Enricher schemas
     #
     module Enrichers
-      extend Schemas::Mixins
+      extend Concerns::Orrable
 
-      IPInfo = Dry::Schema.Params do
-        required(:enricher).value(Types::String.enum(*Mihari::Enrichers::IPInfo.class_keys))
-        optional(:api_key).value(:string)
+      MMDB = Dry::Schema.Params do
+        required(:enricher).value(Types::String.enum(*Mihari::Enrichers::MMDB.class_keys))
         optional(:options).hash(Options)
       end
 

data/lib/mihari/schemas/macros.rb

@@ -10,7 +10,7 @@ module Dry
       class DSL
         def default(value)
           schema_dsl.before(:rule_applier) do |result|
-            result.update(name => value) unless result[name]
+            result.update(name => value) if result.output && !result[name]
           end
         end
       end

data/lib/mihari/schemas/options.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "dry/schema"
-
 module Mihari
   module Schemas
     Options = Dry::Schema.Params do

data/lib/mihari/schemas/rule.rb

@@ -29,7 +29,6 @@ module Mihari
       optional(:data_types).value(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
       optional(:falsepositives).value(array[:string]).default([])
 
-      optional(:artifact_lifetime).value(:integer)
       optional(:artifact_ttl).value(:integer)
     end
 
@@ -37,7 +36,7 @@ module Mihari
     # Rule schema contract
     #
     class RuleContract < Dry::Validation::Contract
-      include Mihari::Mixins::FalsePositive
+      include Mihari::Concerns::FalsePositiveValidatable
 
       params(Rule)
 

data/lib/mihari/services/builders.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    #
+    # Rule builder
+    #
+    class RuleBuilder < Service
+      #
+      # @param [String] path_or_id
+      #
+      # @return [Mihari::Rule]
+      #
+      def call(path_or_id)
+        res = Try { Rule.from_model Mihari::Models::Rule.find(path_or_id) }
+        return res.value! if res.value?
+
+        raise ArgumentError, "#{path_or_id} not found" unless Pathname(path_or_id).exist?
+
+        Rule.from_yaml ERB.new(File.read(path_or_id)).result
+      end
+    end
+
+    #
+    # Autonomous system builder
+    #
+    class AutonomousSystemBuilder < Service
+      #
+      # @param [String] ip
+      # @param [Mihari::Enrichers::MMDB] enricher
+      #
+      # @return [Mihari::Models::AutonomousSystem, nil]
+      #
+      def call(ip, enricher: Enrichers::MMDB.new)
+        enricher.result(ip).fmap do |res|
+          Models::AutonomousSystem.new(asn: res.asn) if res.asn
+        end.value_or nil
+      end
+    end
+
+    #
+    # CPE builder
+    #
+    class CPEBuilder < Service
+      #
+      # Build CPEs
+      #
+      # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      # @return [Array<Mihari::Models::CPE>]
+      #
+      def call(ip, enricher: Enrichers::Shodan.new)
+        enricher.result(ip).fmap do |res|
+          (res&.cpes || []).map { |cpe| Models::CPE.new(cpe: cpe) }
+        end.value_or []
+      end
+    end
+
+    #
+    # DNS record builder
+    #
+    class DnsRecordBuilder < Service
+      #
+      # Build DNS records
+      #
+      # @param [String] domain
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      # @return [Array<Mihari::Models::DnsRecord>]
+      #
+      def call(domain, enricher: Enrichers::GooglePublicDNS.new)
+        enricher.result(domain).fmap do |res|
+          res.answers.map { |answer| Models::DnsRecord.new(resource: answer.resource_type, value: answer.data) }
+        end.value_or []
+      end
+    end
+
+    #
+    # Geolocation builder
+    #
+    class GeolocationBuilder < Service
+      #
+      # Build Geolocation
+      #
+      # @param [String] ip
+      # @param [Mihari::Enrichers::MMDB] enricher
+      #
+      # @return [Mihari::Models::Geolocation, nil]
+      #
+      def call(ip, enricher: Enrichers::MMDB.new)
+        enricher.result(ip).fmap do |res|
+          if res.country_code
+            Models::Geolocation.new(
+              country: NormalizeCountry(res.country_code, to: :short),
+              country_code: res.country_code
+            )
+          end
+        end.value_or nil
+      end
+    end
+
+    #
+    # Port builder
+    #
+    class PortBuilder < Service
+      #
+      # Build ports
+      #
+      # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      # @return [Array<Mihari::Models::Port>]
+      #
+      def call(ip, enricher: Enrichers::Shodan.new)
+        enricher.result(ip).fmap do |res|
+          (res&.ports || []).map { |port| Models::Port.new(port: port) }
+        end.value_or []
+      end
+    end
+
+    #
+    # Reverse DNS name builder
+    #
+    class ReverseDnsNameBuilder < Service
+      #
+      # Build reverse DNS names
+      #
+      # @param [String] ip
+      # @param [Mihari::Enrichers::Shodan] enricher
+      #
+      # @return [Array<Mihari::Models::ReverseDnsName>]
+      #
+      def call(ip, enricher: Enrichers::Shodan.new)
+        enricher.result(ip).fmap do |res|
+          (res&.hostnames || []).map { |name| Models::ReverseDnsName.new(name: name) }
+        end.value_or []
+      end
+    end
+
+    #
+    # Whois record builder
+    #
+    class WhoisRecordBuilder < Service
+      #
+      # Build whois record
+      #
+      # @param [String] domain
+      # @param [Mihari::Enrichers::Whois] enricher
+      #
+      # @return [Mihari::Models::WhoisRecord, nil]
+      #
+      def call(domain, enricher: Enrichers::Whois.new)
+        enricher.result(domain).value_or nil
+      end
+    end
+  end
+end

data/lib/mihari/services/creators.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    #
+    # Alert creator
+    #
+    class AlertCreator < Service
+      #
+      # @param [Hash] params
+      #
+      # @return [Mihari::Models::Alert]
+      #
+      def call(params)
+        normalized = params.to_snake_keys
+        proxy = Services::AlertProxy.new(**normalized)
+        emitter = Emitters::Database.new(rule: proxy.rule)
+        emitter.call proxy.artifacts
+      end
+    end
+  end
+end

data/lib/mihari/services/destroyers.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class AlertDestroyer < Service
+      #
+      # @param [String] id
+      #
+      def call(id)
+        Models::Alert.find(id).destroy
+      end
+    end
+
+    class ArtifactDestroyer < Service
+      #
+      # @param [Integer] id
+      #
+      def call(id)
+        Models::Artifact.find(id).destroy
+      end
+    end
+
+    class RuleDestroyer < Service
+      #
+      # @param [String] id
+      #
+      def call(id)
+        Models::Rule.find(id).destroy
+      end
+    end
+
+    class TagDestroyer < Service
+      #
+      # @param [Integer] id
+      #
+      def call(id)
+        Models::Tag.find(id).destroy
+      end
+    end
+  end
+end

data/lib/mihari/services/enrichers.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class ArtifactEnricher < Service
+      #
+      # @param [String] id
+      #
+      def call(id)
+        artifact = Mihari::Models::Artifact.includes(
+          :autonomous_system,
+          :geolocation,
+          :whois_record,
+          :dns_records,
+          :reverse_dns_names,
+          :cpes,
+          :ports
+        ).find(id)
+
+        artifact.enrich_all
+        artifact.save
+      end
+    end
+  end
+end

data/lib/mihari/services/feed.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "csv"
+
+require "jr/cli/core_ext"
+
+module Mihari
+  module Services
+    #
+    # Feed reader
+    #
+    class FeedReader < Service
+      #
+      # @param [String] url
+      # @param [Hash, nil] options
+      # @param [String] method
+      # @param [Hash, nil] headers
+      # @param [Hash, nil] params
+      # @param [Hash, nil] json
+      # @param [form, nil] form
+      # @param [String] selector
+      # @param [Integer, nil] timeout
+      #
+      # @return [Array<Hash>]
+      #
+      def call(url, headers: {}, method: "GET", params: nil, json: nil, form: nil, timeout: nil)
+        url = Addressable::URI.parse(url)
+
+        return read_file(url.path) if url.scheme == "file"
+
+        http = HTTP::Factory.build(headers: headers, timeout: timeout)
+
+        res = http.get(url, params: params) if method == "GET"
+        res = http.post(url, params: params, json: json, form: form) if method == "POST"
+
+        body = res.body.to_s
+        content_type = res["Content-Type"].to_s
+        return convert_as_json(body) if content_type.include?("application/json")
+
+        convert_as_csv(body)
+      end
+
+      #
+      # Convert text as JSON
+      #
+      # @param [String] text
+      #
+      # @return [Hash, Array<Object>]
+      #
+      def convert_as_json(text)
+        JSON.parse(text).deep_symbolize_keys
+      end
+
+      #
+      # Convert text as CSV
+      #
+      # @param [String] text
+      #
+      # @return [Array<Object>]
+      #
+      def convert_as_csv(text)
+        text_without_comments = text.lines.reject { |line| line.start_with? "#" }.join("\n")
+
+        CSV.new(text_without_comments).to_a.reject(&:empty?)
+      end
+
+      #
+      # Read & convert a file
+      #
+      # @param [String] path
+      #
+      # @return [Array<Object>]
+      #
+      def read_file(path)
+        text = File.read(path)
+
+        return convert_as_json(text) if path.end_with?(".json")
+
+        convert_as_csv text
+      end
+    end
+
+    #
+    # Feed parser
+    #
+    class FeedParser < Service
+      # Parse data by selector
+      #
+      # @param [Hash, Array<Object>] input_enumerator
+      # @param [String] selector
+      #
+      # @return [Array<String>]
+      #
+      # @param [Object] read_data
+      def call(input_enumerator, selector)
+        parsed = proc do
+          $SAFE = 1
+          input_enumerator.instance_eval(selector)
+        end.call
+
+        raise TypeError unless parsed.all?(String)
+
+        parsed
+      end
+    end
+  end
+end