mihari 5.6.2 → 5.7.1

data/lib/mihari/enrichers/whois.rb

@@ -4,6 +4,9 @@ require "whois-parser"
 
 module Mihari
   module Enrichers
+    #
+    # Whois enricher
+    #
     class Whois < Base
       # @return [Hash]
       attr_accessor :memo
@@ -20,11 +23,11 @@ module Mihari
       #
       # Query IAIA Whois API
       #
-      # @param [String] name
+      # @param [String] domain
       #
-      # @return [Mihari::WhoisRecord, nil]
+      # @return [Mihari::Models::WhoisRecord, nil]
       #
-      def query(domain)
+      def call(domain)
         domain = PublicSuffix.domain(domain)
 
         # check memo
@@ -71,7 +74,7 @@ module Mihari
       #
       # Get created_on
       #
-      # @param [::Whois::Parser:] parser
+      # @param [::Whois::Parser] parser
       #
       # @return [Date, nil]
       #
@@ -84,7 +87,7 @@ module Mihari
       #
       # Get updated_on
       #
-      # @param [::Whois::Parser:] parser
+      # @param [::Whois::Parser] parser
       #
       # @return [Date, nil]
       #
@@ -97,7 +100,7 @@ module Mihari
       #
       # Get expires_on
       #
-      # @param [::Whois::Parser:] parser
+      # @param [::Whois::Parser] parser
       #
       # @return [Date, nil]
       #
@@ -110,7 +113,7 @@ module Mihari
       #
       # Get registrar
       #
-      # @param [::Whois::Parser:] parser
+      # @param [::Whois::Parser] parser
       #
       # @return [Hash, nil]
       #
@@ -123,9 +126,9 @@ module Mihari
       #
       # Get contacts
       #
-      # @param [::Whois::Parser:] parser
+      # @param [::Whois::Parser] parser
       #
-      # @return [Array[Hash], nil]
+      # @return [Array<Hash>, nil]
       #
       def get_contacts(parser)
         parser.contacts.map(&:to_h)

data/lib/mihari/entities/tag.rb

@@ -3,6 +3,7 @@
 module Mihari
   module Entities
     class Tag < Grape::Entity
+      expose :id, documentation: { type: Integer, required: true }
       expose :name, documentation: { type: String, required: true }
     end
 

data/lib/mihari/errors.rb

@@ -20,6 +20,9 @@ module Mihari
 
   class TimeoutError < HTTPError; end
 
+  #
+  # HTTP status code error
+  #
   class StatusCodeError < HTTPError
     # @return [Integer]
     attr_reader :status_code
@@ -40,6 +43,9 @@ module Mihari
     end
   end
 
+  #
+  # (dry-schema) Schema validation error
+  #
   class ValidationError < Error
     attr_reader :errors
 

data/lib/mihari/feed/parser.rb

@@ -4,6 +4,9 @@ require "jr/cli/core_ext"
 
 module Mihari
   module Feed
+    #
+    # Feed parser
+    #
     class Parser
       # @return [Array<Hash>, Array<Array<String>>]
       attr_reader :data

data/lib/mihari/feed/reader.rb

@@ -4,6 +4,9 @@ require "csv"
 
 module Mihari
   module Feed
+    #
+    # Feed reader
+    #
     class Reader
       # @return [String]
       attr_reader :url

data/lib/mihari/http.rb

@@ -4,6 +4,9 @@ require "http"
 
 module Mihari
   module HTTP
+    #
+    # Better error handling feature
+    #
     class BetterError < ::HTTP::Feature
       def wrap_response(response)
         unless response.status.success?
@@ -24,6 +27,9 @@ module Mihari
       ::HTTP::Options.register_feature(:better_error, self)
     end
 
+    #
+    # HTTP client factory
+    #
     class Factory
       class << self
         #

data/lib/mihari/mixins/autonomous_system.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Mixins
+    #
+    # Autonomous System mixin
+    #
     module AutonomousSystem
       #
       # Normalize ASN value

data/lib/mihari/mixins/configurable.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Mixins
+    #
+    # Configurable mixin
+    #
     module Configurable
       #
       # Check whether there are configuration key-values or not

data/lib/mihari/mixins/error_notification.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Mixins
+    #
+    # Error notification mixin
+    #
     module ErrorNotification
       #
       # Send an exception notification if there is any error in a block

data/lib/mihari/mixins/falsepositive.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Mixins
+    #
+    # False positive mixins
+    #
     module FalsePositive
       include Memist::Memoizable
 

data/lib/mihari/mixins/refang.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Mixins
+    #
+    # Refang mixin
+    #
     module Refang
       #
       # Refang defanged indicator

data/lib/mihari/mixins/retriable.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Mixins
+    #
+    # Retriable mixin
+    #
     module Retriable
       DEFAULT_ON = [
         Errno::ECONNRESET,
@@ -11,8 +14,9 @@ module Mihari
         Timeout::Error,
         RetryableError,
         NetworkError,
-        TimeoutError
-      ]
+        TimeoutError,
+        StatusCodeError
+      ].freeze
 
       #
       # Retry on error

data/lib/mihari/models/alert.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Models
+    #
+    # Alert model
+    #
     class Alert < ActiveRecord::Base
       has_many :taggings, dependent: :destroy
       has_many :artifacts, dependent: :destroy
@@ -13,7 +16,7 @@ module Mihari
         #
         # Search alerts
         #
-        # @param [Structs::Filters::Alert::SearchFilterWithPagination] filter
+        # @param [Mihari::Structs::Filters::Alert::SearchFilterWithPagination] filter
         #
         # @return [Array<Alert>]
         #
@@ -35,7 +38,7 @@ module Mihari
         #
         # Count alerts
         #
-        # @param [String, nil] artifact_data
+        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
         #
         # @return [Integer]
         #
@@ -47,7 +50,7 @@ module Mihari
         private
 
         #
-        # @param [Structs::Filters::Alert::SearchFilter] filter
+        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
         #
         # @return [Array<Integer>]
         #
@@ -65,7 +68,7 @@ module Mihari
         end
 
         #
-        # @param [Structs::Filters::Alert::SearchFilter] filter
+        # @param [Mihari::Structs::Filters::Alert::SearchFilter] filter
         #
         # @return [Mihari::Models::Alert]
         #

data/lib/mihari/models/artifact.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Models
+    #
+    # Artifact validator
+    #
     class ArtifactValidator < ActiveModel::Validator
       def validate(record)
         return if record.data_type
@@ -10,6 +13,9 @@ module Mihari
       end
     end
 
+    #
+    # Artifact model
+    #
     class Artifact < ActiveRecord::Base
       belongs_to :alert
 

data/lib/mihari/models/autonomous_system.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Models
+    #
+    # AS model
+    #
     class AutonomousSystem < ActiveRecord::Base
       belongs_to :artifact
 
@@ -17,7 +20,7 @@ module Mihari
         # @return [Mihari::AutonomousSystem, nil]
         #
         def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
-          result = enricher.query_result(ip).bind do |res|
+          result = enricher.result(ip).bind do |res|
             value = res&.asn
             if value.nil?
               Success nil

data/lib/mihari/models/cpe.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Models
+    #
+    # CPE model
+    #
     class CPE < ActiveRecord::Base
       belongs_to :artifact
 
@@ -17,7 +20,7 @@ module Mihari
         # @return [Array<Mihari::CPE>]
         #
         def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.query_result(ip).bind do |res|
+          result = enricher.result(ip).bind do |res|
             if res.nil?
               Success []
             else

data/lib/mihari/models/dns.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Models
+    #
+    # DNS record model
+    #
     class DnsRecord < ActiveRecord::Base
       belongs_to :artifact
 
@@ -17,7 +20,7 @@ module Mihari
         # @return [Array<Mihari::Models::DnsRecord>]
         #
         def build_by_domain(domain, enricher: Enrichers::GooglePublicDNS.new)
-          result = enricher.query_result(domain).bind do |responses|
+          result = enricher.result(domain).bind do |responses|
             Success(
               responses.map do |res|
                 res.answers.map do |answer|

data/lib/mihari/models/geolocation.rb

@@ -4,6 +4,9 @@ require "normalize_country"
 
 module Mihari
   module Models
+    #
+    # Geolocation model
+    #
     class Geolocation < ActiveRecord::Base
       belongs_to :artifact
 
@@ -19,7 +22,7 @@ module Mihari
         # @return [Mihari::Geolocation, nil]
         #
         def build_by_ip(ip, enricher: Enrichers::IPInfo.new)
-          result = enricher.query_result(ip).bind do |res|
+          result = enricher.result(ip).bind do |res|
             value = res&.country_code
             if value.nil?
               Success nil

data/lib/mihari/models/port.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Models
+    #
+    # Port model
+    #
     class Port < ActiveRecord::Base
       belongs_to :artifact
 
@@ -17,7 +20,7 @@ module Mihari
         # @return [Array<Mihari::Port>]
         #
         def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.query_result(ip).bind do |res|
+          result = enricher.result(ip).bind do |res|
             if res.nil?
               Success []
             else

data/lib/mihari/models/reverse_dns.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Models
+    #
+    # Reverse DNS name model
+    #
     class ReverseDnsName < ActiveRecord::Base
       belongs_to :artifact
 
@@ -17,7 +20,7 @@ module Mihari
         # @return [Array<Mihari::Models::ReverseDnsName>]
         #
         def build_by_ip(ip, enricher: Enrichers::Shodan.new)
-          result = enricher.query_result(ip).bind do |res|
+          result = enricher.result(ip).bind do |res|
             if res.nil?
               Success []
             else

data/lib/mihari/models/rule.rb

@@ -4,6 +4,9 @@ require "yaml"
 
 module Mihari
   module Models
+    #
+    # Rule model
+    #
     class Rule < ActiveRecord::Base
       has_many :alerts, dependent: :destroy
 
@@ -23,7 +26,7 @@ module Mihari
         #
         # Search rules
         #
-        # @param [Structs::Filters::Rule::SearchFilterWithPagination] filter
+        # @param [Mihari::Structs::Filters::Rule::SearchFilterWithPagination] filter
         #
         # @return [Array<Rule>]
         #
@@ -46,7 +49,7 @@ module Mihari
         #
         # Count alerts
         #
-        # @param [Structs::Filters::Rule::SearchFilterWithPagination] filter
+        # @param [Mihari::Structs::Filters::Rule::SearchFilterWithPagination] filter
         #
         # @return [Integer]
         #
@@ -58,7 +61,7 @@ module Mihari
         private
 
         #
-        # @param [Structs::Filters::Rule::SearchFilter] filter
+        # @param [Mihari::Structs::Filters::Rule::SearchFilter] filter
         #
         # @return [Mihari::Models::Rule]
         #

data/lib/mihari/models/tag.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Models
+    #
+    # Tag model
+    #
     class Tag < ActiveRecord::Base
       has_many :taggings, dependent: :destroy
       has_many :tags, through: :taggings

data/lib/mihari/models/tagging.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Models
+    #
+    # Tagging model
+    #
     class Tagging < ActiveRecord::Base
       belongs_to :alert
       belongs_to :tag

data/lib/mihari/models/whois.rb

@@ -2,14 +2,15 @@
 
 module Mihari
   module Models
+    #
+    # Whois record model
+    #
     class WhoisRecord < ActiveRecord::Base
       belongs_to :artifact
 
       @memo = {}
 
       class << self
-        include Dry::Monads[:result]
-
         #
         # Build whois record
         #
@@ -19,7 +20,7 @@ module Mihari
         # @return [WhoisRecord, nil]
         #
         def build_by_domain(domain, enricher: Enrichers::Whois.new)
-          result = enricher.query_result(domain)
+          result = enricher.result(domain)
           result.value_or nil
         end
       end

data/lib/mihari/rule.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Mihari
-  class Rule
+  class Rule < Service
     include Mixins::FalsePositive
 
     # @return [Hash]
@@ -19,6 +19,8 @@ module Mihari
     # @param [Hash] data
     #
     def initialize(**data)
+      super()
+
       @data = data.deep_symbolize_keys
       @errors = nil
       @base_time = Time.now.utc
@@ -109,16 +111,19 @@ module Mihari
     #
     def artifacts
       analyzers.flat_map do |analyzer|
+        # @type [Dry::Monads::Result::Success<Array<Mihari::Models::Artifact>>, Dry::Monads::Result::Failure]
         result = analyzer.result
 
-        raise result.failure if result.failure? && !analyzer.ignore_error?
-
-        artifacts = result.value!
-        artifacts.map do |artifact|
-          artifact.rule_id = id
-          artifact
+        if result.failure?
+          raise result.failure unless analyzer.ignore_error?
+        else
+          artifacts = result.value!
+          artifacts.map do |artifact|
+            artifact.rule_id = id
+            artifact
+          end
         end
-      end
+      end.compact
     end
 
     #
@@ -169,7 +174,7 @@ module Mihari
 
       # NOTE: separate parallel execution and logging
       #       because the logger does not work along with Parallel
-      results = Parallel.map(emitters) { |emitter| emitter.emit_result(enriched_artifacts) }
+      results = Parallel.map(emitters) { |emitter| emitter.result enriched_artifacts }
       results.zip(emitters).map do |result_and_emitter|
         result, emitter = result_and_emitter
         Mihari.logger.info "Emission by #{emitter.class} is failed: #{result.failure}" if result.failure?
@@ -183,7 +188,7 @@ module Mihari
     #
     # @return [Mihari::Models::Alert, nil]
     #
-    def run
+    def call
       # Validate analyzers & emitters before using them
       analyzers
       emitters
@@ -213,13 +218,27 @@ module Mihari
       )
     end
 
+    #
+    # @return [Boolean]
+    #
+    def diff?
+      model = Mihari::Models::Rule.find(id)
+      model.data != data.deep_stringify_keys
+    rescue ActiveRecord::RecordNotFound
+      false
+    end
+
+    def update_or_create
+      model.save
+    end
+
     class << self
       #
       # Load rule from YAML string
       #
       # @param [String] yaml
       #
-      # @return [Mihari::Services::Rule]
+      # @return [Mihari::Rule]
       #
       def from_yaml(yaml)
         data = YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
@@ -229,7 +248,7 @@ module Mihari
       #
       # @param [Mihari::Models::Rule] model
       #
-      # @return [Mihari::Services::Rule]
+      # @return [Mihari::Rule]
       #
       def from_model(model)
         new(**model.data)

data/lib/mihari/schemas/alert.rb

@@ -7,6 +7,9 @@ module Mihari
       required(:artifacts).value(array[:string])
     end
 
+    #
+    # Alert schema contract
+    #
     class AlertContract < Dry::Validation::Contract
       params(Alert)
     end

data/lib/mihari/schemas/analyzer.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Schemas
+    #
+    # Analyzer schemas
+    #
     module Analyzers
       extend Schemas::Mixins
 
@@ -61,6 +64,14 @@ module Mihari
         optional(:options).hash(AnalyzerOptions)
       end
 
+      Fofa = Dry::Schema.Params do
+        required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::Fofa.class_keys))
+        required(:query).value(:string)
+        optional(:api_key).value(:string)
+        optional(:email).value(:string)
+        optional(:options).hash(AnalyzerPaginationOptions)
+      end
+
       PassiveTotal = Dry::Schema.Params do
         required(:analyzer).value(Types::String.enum(*Mihari::Analyzers::PassiveTotal.class_keys))
         required(:query).value(:string)

data/lib/mihari/schemas/emitter.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Schemas
+    #
+    # Emitter schemas
+    #
     module Emitters
       extend Schemas::Mixins
 

data/lib/mihari/schemas/enricher.rb

@@ -2,6 +2,9 @@
 
 module Mihari
   module Schemas
+    #
+    # Enricher schemas
+    #
     module Enrichers
       extend Schemas::Mixins
 

data/lib/mihari/schemas/macros.rb

@@ -3,6 +3,10 @@
 module Dry
   module Schema
     module Macros
+      #
+      # Macros DSL for options with default values
+      # (see https://github.com/dry-rb/dry-schema/issues/70)
+      #
       class DSL
         def default(value)
           schema_dsl.before(:rule_applier) do |result|

data/lib/mihari/schemas/mixins.rb

@@ -1,5 +1,10 @@
+# frozen_string_literal: true
+
 module Mihari
   module Schemas
+    #
+    # Mixins for schemas
+    #
     module Mixins
       def get_or_composition
         schemas = constants.map { |sym| const_get sym }

data/lib/mihari/schemas/rule.rb

@@ -33,6 +33,9 @@ module Mihari
       optional(:artifact_ttl).value(:integer)
     end
 
+    #
+    # Rule schema contract
+    #
     class RuleContract < Dry::Validation::Contract
       include Mihari::Mixins::FalsePositive