mihari 6.3.0 → 7.0.0

data/lib/mihari/schemas/alert.rb

@@ -5,6 +5,7 @@ module Mihari
     Alert = Dry::Schema.Params do
       required(:rule_id).value(:string)
       required(:artifacts).value(array[:string])
+      optional(:source).value(:string)
     end
 
     #

data/lib/mihari/schemas/analyzer.rb

@@ -6,7 +6,7 @@ module Mihari
     # Analyzer schemas
     #
     module Analyzers
-      extend Schemas::Mixins
+      extend Concerns::Orrable
 
       # Analyzer with API key and pagination
       [
@@ -110,7 +110,7 @@ module Mihari
         optional(:method).value(Types::HTTPRequestMethods).default("GET")
         optional(:headers).value(:hash).default({})
         optional(:params).value(:hash)
-        optional(:data).value(:hash)
+        optional(:form).value(:hash)
         optional(:json).value(:hash)
         optional(:options).hash(AnalyzerOptions)
       end

data/lib/mihari/schemas/concerns/orrable.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Schemas
+    module Concerns
+      #
+      # OR-rable concern
+      #
+      module Orrable
+        extend ActiveSupport::Concern
+
+        def get_or_composition
+          schemas = constants.map { |sym| const_get sym }
+          return schemas.first if schemas.length <= 1
+
+          base, *others = schemas
+          others.each { |other| base = base.or(other) }
+
+          base
+        end
+      end
+    end
+  end
+end

data/lib/mihari/schemas/emitter.rb

@@ -6,7 +6,7 @@ module Mihari
     # Emitter schemas
     #
     module Emitters
-      extend Schemas::Mixins
+      extend Concerns::Orrable
 
       Database = Dry::Schema.Params do
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::Database.class_keys))
@@ -24,7 +24,6 @@ module Mihari
         required(:emitter).value(Types::String.enum(*Mihari::Emitters::TheHive.class_keys))
         optional(:url).value(:string)
         optional(:api_key).value(:string)
-        optional(:api_version).value(Types::String.enum("v4", "v5")).default("v4")
         optional(:options).hash(Options)
       end
 

data/lib/mihari/schemas/enricher.rb

@@ -6,11 +6,10 @@ module Mihari
     # Enricher schemas
     #
     module Enrichers
-      extend Schemas::Mixins
+      extend Concerns::Orrable
 
-      IPInfo = Dry::Schema.Params do
-        required(:enricher).value(Types::String.enum(*Mihari::Enrichers::IPInfo.class_keys))
-        optional(:api_key).value(:string)
+      MMDB = Dry::Schema.Params do
+        required(:enricher).value(Types::String.enum(*Mihari::Enrichers::MMDB.class_keys))
         optional(:options).hash(Options)
       end
 

data/lib/mihari/schemas/macros.rb

@@ -10,7 +10,7 @@ module Dry
       class DSL
         def default(value)
           schema_dsl.before(:rule_applier) do |result|
-            result.update(name => value) unless result[name]
+            result.update(name => value) if result.output && !result[name]
           end
         end
       end

data/lib/mihari/schemas/options.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "dry/schema"
-
 module Mihari
   module Schemas
     Options = Dry::Schema.Params do

data/lib/mihari/schemas/rule.rb

@@ -29,7 +29,6 @@ module Mihari
       optional(:data_types).value(array[Types::DataTypes]).default(Mihari::Types::DataTypes.values)
       optional(:falsepositives).value(array[:string]).default([])
 
-      optional(:artifact_lifetime).value(:integer)
       optional(:artifact_ttl).value(:integer)
     end
 
@@ -37,7 +36,7 @@ module Mihari
     # Rule schema contract
     #
     class RuleContract < Dry::Validation::Contract
-      include Mihari::Mixins::FalsePositive
+      include Mihari::Concerns::FalsePositiveValidatable
 
       params(Rule)
 

data/lib/mihari/services/{rule_builder.rb → builders.rb}

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require "date"
-require "erb"
-require "pathname"
-require "yaml"
-
 module Mihari
   module Services
     #
@@ -23,7 +18,7 @@ module Mihari
         result = Try { Mihari::Models::Rule.find path_or_id }.to_result
         return result.value! if result.success?
 
-        raise ArgumentError, "#{path_or_id} does not exist" unless Pathname(path_or_id).exist?
+        raise ArgumentError, "#{path_or_id} not found" unless Pathname(path_or_id).exist?
 
         YAML.safe_load(
           ERB.new(File.read(path_or_id)).result,

data/lib/mihari/services/creators.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    #
+    # Alert creator
+    #
+    class AlertCreator < Service
+      #
+      # @param [Hash] params
+      #
+      # @return [Mihari::Models::Alert]
+      #
+      def call(params)
+        normalized = params.to_snake_keys
+        proxy = Services::AlertProxy.new(**normalized)
+        emitter = Emitters::Database.new(rule: proxy.rule)
+        emitter.call proxy.artifacts
+      end
+    end
+  end
+end

data/lib/mihari/services/destroyers.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class AlertDestroyer < Service
+      #
+      # @param [String] id
+      #
+      def call(id)
+        Models::Alert.find(id).destroy
+      end
+    end
+
+    class ArtifactDestroyer < Service
+      #
+      # @param [Integer] id
+      #
+      def call(id)
+        Models::Artifact.find(id).destroy
+      end
+    end
+
+    class RuleDestroyer < Service
+      #
+      # @param [String] id
+      #
+      def call(id)
+        Models::Rule.find(id).destroy
+      end
+    end
+
+    class TagDestroyer < Service
+      #
+      # @param [Integer] id
+      #
+      def call(id)
+        Models::Tag.find(id).destroy
+      end
+    end
+  end
+end

data/lib/mihari/services/enrichers.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class ArtifactEnricher < Service
+      #
+      # @param [String] id
+      #
+      def call(id)
+        artifact = Mihari::Models::Artifact.includes(
+          :autonomous_system,
+          :geolocation,
+          :whois_record,
+          :dns_records,
+          :reverse_dns_names,
+          :cpes,
+          :ports
+        ).find(id)
+
+        artifact.enrich_all
+        artifact.save
+      end
+    end
+  end
+end

data/lib/mihari/services/feed.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "csv"
+
+require "jr/cli/core_ext"
+
+module Mihari
+  module Services
+    #
+    # Feed reader
+    #
+    class FeedReader < Service
+      #
+      # @param [String] url
+      # @param [Hash, nil] options
+      # @param [String] method
+      # @param [Hash, nil] headers
+      # @param [Hash, nil] params
+      # @param [Hash, nil] json
+      # @param [form, nil] form
+      # @param [String] selector
+      # @param [Integer, nil] timeout
+      #
+      # @return [Array<Hash>]
+      #
+      def call(url, headers: {}, method: "GET", params: nil, json: nil, form: nil, timeout: nil)
+        url = Addressable::URI.parse(url)
+
+        return read_file(url.path) if url.scheme == "file"
+
+        http = HTTP::Factory.build(headers: headers, timeout: timeout)
+
+        res = http.get(url, params: params) if method == "GET"
+        res = http.post(url, params: params, json: json, form: form) if method == "POST"
+
+        body = res.body.to_s
+        content_type = res["Content-Type"].to_s
+        return convert_as_json(body) if content_type.include?("application/json")
+
+        convert_as_csv(body)
+      end
+
+      #
+      # Convert text as JSON
+      #
+      # @param [String] text
+      #
+      # @return [Hash, Array<Object>]
+      #
+      def convert_as_json(text)
+        JSON.parse(text).deep_symbolize_keys
+      end
+
+      #
+      # Convert text as CSV
+      #
+      # @param [String] text
+      #
+      # @return [Array<Object>]
+      #
+      def convert_as_csv(text)
+        text_without_comments = text.lines.reject { |line| line.start_with? "#" }.join("\n")
+
+        CSV.new(text_without_comments).to_a.reject(&:empty?)
+      end
+
+      #
+      # Read & convert a file
+      #
+      # @param [String] path
+      #
+      # @return [Array<Object>]
+      #
+      def read_file(path)
+        text = File.read(path)
+
+        return convert_as_json(text) if path.end_with?(".json")
+
+        convert_as_csv text
+      end
+    end
+
+    #
+    # Feed parser
+    #
+    class FeedParser < Service
+      # Parse data by selector
+      #
+      # @param [Hash, Array<Object>] input_enumerator
+      # @param [String] selector
+      #
+      # @return [Array<String>]
+      #
+      # @param [Object] read_data
+      def call(input_enumerator, selector)
+        parsed = proc do
+          $SAFE = 1
+          input_enumerator.instance_eval(selector)
+        end.call
+
+        raise TypeError unless parsed.all?(String)
+
+        parsed
+      end
+    end
+  end
+end

data/lib/mihari/services/getters.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class ArtifactGetter < Service
+      #
+      # @param [Integer] id
+      #
+      # @return [Mihari::Models::Artifact]
+      #
+      def call(id)
+        Mihari::Models::Artifact.eager_load(
+          :autonomous_system,
+          :geolocation,
+          :whois_record,
+          :dns_records,
+          :reverse_dns_names
+        ).find id
+      end
+    end
+
+    class AlertGetter < Service
+      #
+      # @param [Integer] id
+      #
+      # @return [Mihari::Models::Artifact]
+      #
+      def call(id)
+        Mihari::Models::Alert.eager_load(
+          :artifacts,
+          rule: :tags
+        ).find id
+      end
+    end
+
+    class RuleGetter < Service
+      #
+      # @params [String] id
+      #
+      # @return [Mihari::Models::Rule]
+      #
+      def call(id)
+        Mihari::Models::Rule.find id
+      end
+    end
+
+    class IPGetter < Service
+      #
+      # @param [String] ip
+      #
+      # @return [Mihari::Structs::MMDB::Response]
+      #
+      def call(ip)
+        Mihari::Enrichers::MMDB.new.call ip
+      end
+    end
+  end
+end

data/lib/mihari/services/initializers.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class RuleInitializer < Service
+      #
+      # @param [String] path
+      # @param [Dry::Files] files
+      #
+      def call(path, files = Dry::Files.new)
+        rule = Mihari::Rule.new(
+          id: SecureRandom.uuid,
+          title: "Title goes here",
+          description: "Description goes here",
+          created_on: Date.today,
+          queries: []
+        )
+        files.write(path, rule.yaml)
+      end
+    end
+  end
+end

data/lib/mihari/services/{alert_builder.rb → proxies.rb}

@@ -1,11 +1,5 @@
 # frozen_string_literal: true
 
-require "date"
-
-require "erb"
-require "pathname"
-require "yaml"
-
 module Mihari
   module Services
     #
@@ -36,9 +30,9 @@ module Mihari
       # @return [Boolean]
       #
       def errors?
-        return false if @errors.nil?
+        return false if errors.nil?
 
-        !@errors.empty?
+        !errors.empty?
       end
 
       def validate!
@@ -69,10 +63,18 @@ module Mihari
         @artifacts ||= data[:artifacts].map do |data|
           artifact = Models::Artifact.new(data: data)
           artifact.rule_id = rule_id
+          artifact.source = source
           artifact
         end.uniq(&:data).select(&:valid?)
       end
 
+      #
+      # @return [String, nil]
+      #
+      def source
+        @source ||= data[:source]
+      end
+
       #
       # @return [Mihari::Rule]
       #
@@ -83,37 +85,5 @@ module Mihari
         end.first
       end
     end
-
-    #
-    # Alert builder
-    #
-    class AlertBuilder < Service
-      # @return [String]
-      attr_reader :path
-
-      #
-      # @param [String] path
-      #
-      # @return [Hash]
-      #
-      def data
-        raise ArgumentError, "#{path} does not exist" unless Pathname(path).exist?
-
-        YAML.safe_load(
-          ERB.new(File.read(path)).result,
-          permitted_classes: [Date, Symbol]
-        )
-      end
-
-      #
-      # @param [String] path
-      #
-      # @return [Mihari::AlertProxy]
-      #
-      def call(path)
-        @path = path
-        AlertProxy.new(**data)
-      end
-    end
   end
 end

data/lib/mihari/services/searchers.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Services
+    class ResultValue
+      # @return [Array<Object>]
+      attr_reader :results
+
+      # @return [Integer]
+      attr_reader :total
+
+      # @return [Mihari::Structs::Filters::Search]
+      attr_reader :filter
+
+      #
+      # @param [Array<Object>] results
+      # @param [Integer] total
+      # @param [Mihari::Structs::Filters::Search] filter
+      #
+      def initialize(results:, total:, filter:)
+        @results = results
+        @total = total
+        @filter = filter
+      end
+    end
+
+    class BaseSearcher < Service
+      #
+      # @param [Hash] params
+      #
+      # @return [ResultValue]
+      #
+      def call(params)
+        filter = build_filter(params)
+        ResultValue.new(
+          total: klass.count_by_filter(filter),
+          results: klass.search_by_filter(filter),
+          filter: filter
+        )
+      end
+
+      private
+
+      #
+      # @param [Hash] params
+      #
+      # @return [Mihari::Structs::Filters::Search]
+      #
+      def build_filter(params)
+        normalized = params.to_h.to_snake_keys.symbolize_keys
+        Structs::Filters::Search.new(**normalized)
+      end
+    end
+
+    class AlertSearcher < BaseSearcher
+      def klass
+        Models::Alert
+      end
+    end
+
+    class ArtifactSearcher < BaseSearcher
+      def klass
+        Models::Artifact
+      end
+    end
+
+    class RuleSearcher < BaseSearcher
+      def klass
+        Models::Rule
+      end
+    end
+
+    class TagSearcher < BaseSearcher
+      def klass
+        Models::Tag
+      end
+    end
+
+    class ConfigSearcher < Service
+      #
+      # @return [Array<Mihari::Structs::Config>]
+      #
+      def call
+        # NOTE: literally it does not search but list configs
+        (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).filter_map do |klass|
+          Mihari::Structs::Config.from_class klass
+        end
+      end
+    end
+  end
+end

data/lib/mihari/sidekiq/application.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+
+require "mihari/sidekiq/jobs"
+
+Sidekiq.configure_server do |config|
+  config.redis = { url: Mihari.config.sidekiq_redis_url.to_s }
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = { url: Mihari.config.sidekiq_redis_url.to_s }
+end

data/lib/mihari/sidekiq/jobs.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+
+module Mihari
+  module Jobs
+    class SearchJob
+      include Sidekiq::Job
+      include Concerns::DatabaseConnectable
+
+      #
+      # @param [String] path_or_id
+      #
+      def perform(path_or_id)
+        with_db_connection do
+          rule = Mihari::Rule.from_model(Mihari::Models::Rule.find(path_or_id))
+          rule.call
+        end
+      end
+    end
+
+    class ArtifactEnrichJob
+      include Sidekiq::Job
+      include Concerns::DatabaseConnectable
+
+      #
+      # @param [Integer] id
+      #
+      def perform(id)
+        with_db_connection do
+          Services::ArtifactEnricher.call id
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/censys.rb

@@ -4,7 +4,7 @@ module Mihari
   module Structs
     module Censys
       class AutonomousSystem < Dry::Struct
-        include Mixins::AutonomousSystem
+        include Concerns::AutonomousSystemNormalizable
 
         # @!attribute [r] asn
         #   @return [Integer]