mihari 5.4.2 → 5.4.4

data/lib/mihari/models/whois.rb

@@ -7,6 +7,8 @@ module Mihari
     @memo = {}
 
     class << self
+      include Dry::Monads[:result]
+
       #
       # Build whois record
       #
@@ -15,7 +17,8 @@ module Mihari
       # @return [WhoisRecord, nil]
       #
       def build_by_domain(domain)
-        Enrichers::Whois.query domain
+        result = Enrichers::Whois.query_result(domain)
+        result.value_or nil
       end
     end
   end

data/lib/mihari/schemas/analyzer.rb

@@ -7,6 +7,7 @@ module Mihari
       optional(:pagination_limit).value(:integer).default(Mihari.config.pagination_limit)
       optional(:retry_times).value(:integer).default(Mihari.config.retry_times)
       optional(:retry_interval).value(:integer).default(Mihari.config.retry_interval)
+      optional(:ignore_error).value(:bool).default(Mihari.config.ignore_error)
     end
 
     AnalyzerWithoutAPIKey = Dry::Schema.Params do

data/lib/mihari/services/alert_builder.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "date"
+
+require "erb"
+require "pathname"
+require "yaml"
+
+module Mihari
+  module Services
+    class AlertBuilder
+      include Dry::Monads[:result, :try]
+
+      # @return [String]
+      attr_reader :path
+
+      #
+      # Initialize
+      #
+      # @param [String] path
+      #
+      def initialize(path)
+        @path = path
+      end
+
+      #
+      # @return [Hash]
+      #
+      def data
+        raise ArgumentError, "#{path} does not exist" unless Pathname(path).exist?
+
+        YAML.safe_load(
+          ERB.new(File.read(path)).result,
+          permitted_classes: [Date, Symbol]
+        )
+      end
+
+      def result
+        Try[StandardError] { AlertProxy.new(data) }.to_result
+      end
+    end
+  end
+end

data/lib/mihari/services/alert_proxy.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "json"
+
 module Mihari
   module Services
     class AlertProxy
@@ -16,10 +18,9 @@ module Mihari
       #
       def initialize(data)
         @data = data.deep_symbolize_keys
-
         @errors = nil
 
-        validate
+        validate!
       end
 
       #
@@ -31,21 +32,14 @@ module Mihari
         !@errors.empty?
       end
 
-      def validate
+      def validate!
         contract = Schemas::AlertContract.new
         result = contract.call(data)
 
         @data = result.to_h
         @errors = result.errors
-      end
-
-      def validate!
-        return unless errors?
 
-        Mihari.logger.error "Failed to parse the input as an alert:"
-        Mihari.logger.error JSON.pretty_generate(errors.to_h)
-
-        raise AlertValidationError, errors
+        raise ValidationError.new("Validation failed", errors) if errors?
       end
 
       def [](key)
@@ -74,7 +68,7 @@ module Mihari
       # @return [Mihari::Services::RuleProxy]
       #
       def rule
-        @rule ||= Services::RuleProxy.from_model(Mihari::Rule.find(rule_id))
+        @rule ||= Services::RuleProxy.new(Mihari::Rule.find(rule_id).data)
       end
 
       class << self
@@ -86,19 +80,7 @@ module Mihari
         # @return [Mihari::Services::Alert]
         #
         def from_yaml(yaml)
-          Services::AlertProxy.new YAML.safe_load(yaml, permitted_classes: [Date, Symbol])
-        rescue Psych::SyntaxError => e
-          raise YAMLSyntaxError, e.message
-        end
-
-        # @param [String] path
-        #
-        # @return [Mihari::Services::Alert, nil]
-        #
-        def from_path(path)
-          return nil unless Pathname(path).exist?
-
-          from_yaml File.read(path)
+          new YAML.safe_load(yaml, permitted_classes: [Date, Symbol])
         end
       end
     end

data/lib/mihari/services/alert_runner.rb

@@ -3,6 +3,8 @@
 module Mihari
   module Services
     class AlertRunner
+      include Dry::Monads[:result, :try]
+
       # @return [Mihari::Services::AlertProxy]
       attr_reader :alert
 
@@ -17,6 +19,13 @@ module Mihari
         emitter = Mihari::Emitters::Database.new(artifacts: alert.artifacts, rule: alert.rule)
         emitter.emit
       end
+
+      #
+      # @return [Dry::Monads::Result::Success<Mihari::Alert, nil>, Dry::Monads::Result::Failure]
+      #
+      def result
+        Try[StandardError] { run }.to_result
+      end
     end
   end
 end

data/lib/mihari/services/rule_builder.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "date"
+require "erb"
+require "pathname"
+require "yaml"
+
+module Mihari
+  module Services
+    class RuleBuilder
+      include Dry::Monads[:result, :try]
+
+      # @return [String]
+      attr_reader :path_or_id
+
+      #
+      # Initialize
+      #
+      # @param [String] path_or_id
+      #
+      def initialize(path_or_id)
+        @path_or_id = path_or_id
+      end
+
+      #
+      # @return [Hash]
+      #
+      def data
+        if Mihari::Rule.exists?(path_or_id)
+          rule = Mihari::Rule.find(path_or_id)
+          return rule.data
+        end
+
+        raise ArgumentError, "#{path_or_id} does not exist" unless Pathname(path_or_id).exist?
+
+        YAML.safe_load(
+          ERB.new(File.read(path_or_id)).result,
+          permitted_classes: [Date, Symbol]
+        )
+      end
+
+      def result
+        Try[StandardError] { RuleProxy.new(data) }.to_result
+      end
+    end
+  end
+end

data/lib/mihari/services/rule_proxy.rb

@@ -1,11 +1,6 @@
 # frozen_string_literal: true
 
-require "date"
-require "erb"
 require "json"
-require "pathname"
-require "securerandom"
-require "yaml"
 
 module Mihari
   module Services
@@ -29,10 +24,9 @@ module Mihari
       #
       def initialize(data)
         @data = data.deep_symbolize_keys
-
         @errors = nil
 
-        validate
+        validate!
       end
 
       #
@@ -44,21 +38,14 @@ module Mihari
         !@errors.empty?
       end
 
-      def validate
+      def validate!
         contract = Schemas::RuleContract.new
         result = contract.call(data)
 
         @data = result.to_h
         @errors = result.errors
-      end
 
-      def validate!
-        return unless errors?
-
-        Mihari.logger.error "Failed to parse the input as a rule:"
-        Mihari.logger.error JSON.pretty_generate(errors.to_h)
-
-        raise RuleValidationError, errors
+        raise ValidationError.new("Validation failed", errors) if errors?
       end
 
       def [](key)
@@ -178,9 +165,7 @@ module Mihari
         # @return [Mihari::Services::Rule]
         #
         def from_yaml(yaml)
-          Services::RuleProxy.new YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
-        rescue Psych::SyntaxError => e
-          raise YAMLSyntaxError, e.message
+          new YAML.safe_load(ERB.new(yaml).result, permitted_classes: [Date, Symbol])
         end
 
         #
@@ -189,48 +174,7 @@ module Mihari
         # @return [Mihari::Services::Rule]
         #
         def from_model(model)
-          Services::RuleProxy.new model.data
-        end
-
-        #
-        # Load a rule from path
-        #
-        # @param [String] path
-        #
-        # @return [Mihari::Services::Rule, nil]
-        #
-        def from_path(path)
-          return nil unless Pathname(path).exist?
-
-          from_yaml File.read(path)
-        end
-
-        #
-        # Load a rule from DB
-        #
-        # @param [String] id
-        #
-        # @return [Mihari::Services::Rule, nil]
-        #
-        def from_id(id)
-          return nil unless Mihari::Rule.exists?(id)
-
-          Services::RuleProxy.from_model Mihari::Rule.find(id)
-        end
-
-        #
-        # @param [String] path_or_id Path to YAML file or YAML string or ID of a rule in the database
-        #
-        # @return [Mihari::Services::Rule]
-        #
-        def from_path_or_id(path_or_id)
-          rule = from_path(path_or_id)
-          return rule unless rule.nil?
-
-          rule = from_id(path_or_id)
-          return rule unless rule.nil?
-
-          raise ArgumentError, "#{path_or_id} does not exist"
+          new model.data
         end
       end
     end

data/lib/mihari/services/rule_runner.rb

@@ -3,6 +3,8 @@
 module Mihari
   module Services
     class RuleRunner
+      include Dry::Monads[:result, :try]
+
       include Mixins::ErrorNotification
 
       # @return [Mihari::Services::RuleProxy]
@@ -38,11 +40,14 @@ module Mihari
       # @return [Mihari::Alert, nil]
       #
       def run
-        analyzer = rule.analyzer
+        rule.analyzer.run
+      end
 
-        with_error_notification do
-          analyzer.run
-        end
+      #
+      # @return [Dry::Monads::Result::Success<Mihari::Alert, nil>, Dry::Monads::Result::Failure]
+      #
+      def result
+        Try[StandardError] { run }.to_result
       end
     end
   end

data/lib/mihari/structs/binaryedge.rb

@@ -0,0 +1,89 @@
+module Mihari
+  module Structs
+    module BinaryEdge
+      class Target < Dry::Struct
+        attribute :ip, Types::String
+
+        #
+        # @return [String]
+        #
+        def ip
+          attributes[:ip]
+        end
+
+        class << self
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              ip: d.fetch("ip")
+            )
+          end
+        end
+      end
+
+      class Event < Dry::Struct
+        attribute :target, Target
+
+        #
+        # @return [Target]
+        #
+        def target
+          attributes[:target]
+        end
+
+        class << self
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              target: Target.from_dynamic!(d.fetch("target"))
+            )
+          end
+        end
+      end
+
+      class Response < Dry::Struct
+        # @!attribute [r] page
+        # @return [Integer]
+        attribute :page, Types::Integer
+
+        # @!attribute [r] pagesize
+        # @return [Integer]
+        attribute :pagesize, Types::Integer
+
+        # @!attribute [r] total
+        # @return [Integer]
+        attribute :total, Types::Integer
+
+        # @!attribute [r] events
+        # @return [Array<Event>]
+        attribute :events, Types.Array(Event)
+
+        #
+        # @return [Array<Event>]
+        #
+        def events
+          attributes[:events]
+        end
+
+        #
+        # @return [Array<Artifact>]
+        #
+        def artifacts
+          events.map { |event| Artifact.new(data: event.target.ip) }
+        end
+
+        class << self
+          def from_dynamic!(d)
+            d = Types::Hash[d]
+            new(
+              page: d.fetch("page"),
+              pagesize: d.fetch("pagesize"),
+              total: d.fetch("total"),
+              events: d.fetch("events").map { |x| Event.from_dynamic!(x) }
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/censys.rb

@@ -18,7 +18,7 @@ module Mihari
         #
         # @return [Mihari::AutonomousSystem]
         #
-        def to_as
+        def as
           Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
         end
 
@@ -58,7 +58,7 @@ module Mihari
         #
         # @return [Mihari::Geolocation] <description>
         #
-        def to_geolocation
+        def geolocation
           # sometimes Censys overlooks country
           # then set geolocation as nil
           return nil if country.nil?
@@ -98,7 +98,7 @@ module Mihari
         #
         # @return [Mihari::Port]
         #
-        def to_port
+        def _port
           Port.new(port: port)
         end
 
@@ -162,20 +162,20 @@ module Mihari
         #
         # @return [Array<Mihari::Port>]
         #
-        def to_ports
-          services.map(&:to_port)
+        def ports
+          services.map(&:_port)
         end
 
         #
         # @return [Mihari::Artifact]
         #
-        def to_artifact
+        def artifact
           Artifact.new(
             data: ip,
             metadata: metadata,
-            autonomous_system: autonomous_system.to_as,
-            geolocation: location.to_geolocation,
-            ports: to_ports
+            autonomous_system: autonomous_system.as,
+            geolocation: location.geolocation,
+            ports: ports
           )
         end
 
@@ -269,8 +269,8 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
-          hits.map(&:to_artifact)
+        def artifacts
+          hits.map(&:artifact)
         end
 
         class << self

data/lib/mihari/structs/greynoise.rb

@@ -34,14 +34,14 @@ module Mihari
         #
         # @return [Mihari::AutonomousSystem]
         #
-        def to_as
+        def as
           Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
         end
 
         #
         # @return [Mihari::Geolocation]
         #
-        def to_geolocation
+        def geolocation
           Mihari::Geolocation.new(
             country: country,
             country_code: country_code
@@ -94,12 +94,12 @@ module Mihari
         #
         # @return [Mihari::Artifact]
         #
-        def to_artifact
+        def artifact
           Mihari::Artifact.new(
             data: ip,
             metadata: metadata_,
-            autonomous_system: metadata.to_as,
-            geolocation: metadata.to_geolocation
+            autonomous_system: metadata.as,
+            geolocation: metadata.geolocation
           )
         end
 
@@ -126,6 +126,7 @@ module Mihari
         attribute :data, Types.Array(Datum)
         attribute :message, Types::String
         attribute :query, Types::String
+        attribute :scroll, Types::String.optional
 
         #
         # @return [Boolean]
@@ -162,11 +163,18 @@ module Mihari
           attributes[:query]
         end
 
+        #
+        # @return [String, nil]
+        #
+        def scroll
+          attributes[:scroll]
+        end
+
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
-          data.map { |datum| datum.to_artifact }
+        def artifacts
+          data.map(&:artifact)
         end
 
         class << self
@@ -182,7 +190,8 @@ module Mihari
               count: d.fetch("count"),
               data: d.fetch("data").map { |x| Datum.from_dynamic!(x) },
               message: d.fetch("message"),
-              query: d.fetch("query")
+              query: d.fetch("query"),
+              scroll: d["scroll"]
             )
           end
         end

data/lib/mihari/structs/onyphe.rb

@@ -42,19 +42,19 @@ module Mihari
         #
         # @return [Mihari::Artifact]
         #
-        def to_artifact
+        def artifact
           Mihari::Artifact.new(
             data: ip,
             metadata: metadata,
-            autonomous_system: to_as,
-            geolocation: to_geolocation
+            autonomous_system: as,
+            geolocation: geolocation
           )
         end
 
         #
         # @return [Mihari::Geolocation, nil]
         #
-        def to_geolocation
+        def geolocation
           return nil if country_code.nil?
 
           Mihari::Geolocation.new(
@@ -66,7 +66,7 @@ module Mihari
         #
         # @return [Mihari::AutonomousSystem]
         #
-        def to_as
+        def as
           Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
         end
 
@@ -150,8 +150,8 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
-          results.map(&:to_artifact)
+        def artifacts
+          results.map(&:artifact)
         end
 
         class << self

data/lib/mihari/structs/shodan.rb

@@ -24,7 +24,7 @@ module Mihari
         #
         # @return [Mihari::Geolocation, nil]
         #
-        def to_geolocation
+        def geolocation
           return nil if country_name.nil? && country_code.nil?
 
           Mihari::Geolocation.new(
@@ -105,7 +105,7 @@ module Mihari
         #
         # @return [Mihari::AutonomousSystem, nil]
         #
-        def to_asn
+        def _asn
           return nil if asn.nil?
 
           Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
@@ -140,7 +140,7 @@ module Mihari
         end
       end
 
-      class Result < Dry::Struct
+      class Response < Dry::Struct
         attribute :matches, Types.Array(Match)
         attribute :total, Types::Int
 
@@ -194,9 +194,10 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
+        def artifacts
           matches.map do |match|
             metadata = collect_metadata_by_ip(match.ip_str)
+
             ports = collect_ports_by_ip(match.ip_str).map do |port|
               Mihari::Port.new(port: port)
             end
@@ -207,8 +208,8 @@ module Mihari
             Mihari::Artifact.new(
               data: match.ip_str,
               metadata: metadata,
-              autonomous_system: match.to_asn,
-              geolocation: match.location.to_geolocation,
+              autonomous_system: match._asn,
+              geolocation: match.location.geolocation,
               ports: ports,
               reverse_dns_names: reverse_dns_names
             )

data/lib/mihari/structs/urlscan.rb

@@ -83,11 +83,9 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
+        def artifacts
           values = [page.url, page.domain, page.ip].compact
-          values.map do |value|
-            Mihari::Artifact.new(data: value, metadata: metadata)
-          end
+          values.map { |value| Mihari::Artifact.new(data: value, metadata: metadata) }
         end
 
         class << self
@@ -129,8 +127,8 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
-          results.map(&:to_artifacts).flatten
+        def artifacts
+          results.map(&:artifacts).flatten
         end
 
         class << self