mihari 4.5.0 → 4.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dfd4bafdc33911546dce3477184426c581b9e3a40a762f32d1944870589ffcc9
-  data.tar.gz: 8765422118bcc6fe914439b416e6362551c9de0492397ce90c2c5bf7487d982a
+  metadata.gz: 02c4e60250ffc8e40e4d0c08fd3721101ab943abdfeb1f73ec267cc345bc0dd2
+  data.tar.gz: 5e65063ca7b9cc8c5b9c6d137abe6820abb6b179c3fb8582d1381d583eb27280
 SHA512:
-  metadata.gz: a0788f87ab0ef4838243e25ca71496408187add826900d0428af6f4a8e7f81093e81944d809425b2c716c778c35b725f757b656cec046adc2294b4a07d764b4f
-  data.tar.gz: 3c0c001e91aaec0090daeb117a7595a84172a355d527af0dab7f81dfee557b271697a0fbcf75c8a44d5e33b6fb1ff1f8b82e0c057dff91e85b4fb4a0a30d69ca
+  metadata.gz: 142cc9ec86582d37e7e5bd472f1a4f085582441e2204d406fa6ff9d2e94331cbc27513a410c00174950750e066ff9c53b100bd4cfe23039e4cb2be7d8f69ed33
+  data.tar.gz: 49a02b02e298b94d8d847d74ea3389a3e873e72e17fdb93d1e5392992be05db73b37f6cea05ead35b04573400b6956e5b34d323008efb679fba60dcf53d6b372

data/lib/mihari/analyzers/censys.rb

@@ -88,12 +88,17 @@ module Mihari
           )
         end
 
+        ports = hit.services.map(&:port).map do |port|
+          Port.new(port: port)
+        end
+
         Artifact.new(
           data: hit.ip,
           source: source,
           metadata: hit.metadata,
           autonomous_system: as,
-          geolocation: geolocation
+          geolocation: geolocation,
+          ports: ports
         )
       end
 

data/lib/mihari/analyzers/rule.rb

@@ -127,9 +127,12 @@ module Mihari
       # @return [Boolean]
       #
       def disallowed_data_value?(value)
-        normalized_disallowed_data_values.any? do |disallowed_data_value|
-          return value == disallowed_data_value if disallowed_data_value.is_a?(String)
-          return disallowed_data_value.match?(value) if disallowed_data_value.is_a?(Regexp)
+        return true if normalized_disallowed_data_values.include?(value)
+
+        normalized_disallowed_data_values.select do |disallowed_data_value|
+          disallowed_data_value.is_a?(Regexp)
+        end.any? do |disallowed_data_value|
+          disallowed_data_value.match?(value)
         end
       end
 

data/lib/mihari/analyzers/shodan.rb

@@ -23,10 +23,10 @@ module Mihari
         return [] unless results || results.empty?
 
         results = results.map { |result| Structs::Shodan::Result.from_dynamic!(result) }
-        results.map do |result|
-          matches = result.matches || []
-          matches.map { |match| build_artifact(match, matches) }
-        end.flatten.uniq(&:data)
+        matches = results.map { |result| result.matches || [] }.flatten
+
+        uniq_matches = matches.uniq(&:ip_str)
+        uniq_matches.map { |match| build_artifact(match, matches) }
       end
 
       private
@@ -94,6 +94,30 @@ module Mihari
         matches.select { |match| match.ip_str == ip }.map(&:metadata)
       end
 
+      #
+      # Collect ports from matches
+      #
+      # @param [Array<Structs::Shodan::Match>] matches
+      # @param [String] ip
+      #
+      # @return [Array<String>]
+      #
+      def collect_ports_by_ip(matches, ip)
+        matches.select { |match| match.ip_str == ip }.map(&:port)
+      end
+
+      #
+      # Collect hostnames from matches
+      #
+      # @param [Array<Structs::Shodan::Match>] matches
+      # @param [String] ip
+      #
+      # @return [Array<String>]
+      #
+      def collect_hostnames_by_ip(matches, ip)
+        matches.select { |match| match.ip_str == ip }.map(&:hostnames).flatten.uniq
+      end
+
       #
       # Build an artifact from a Shodan search API response
       #
@@ -116,12 +140,22 @@ module Mihari
 
         metadata = collect_metadata_by_ip(matches, match.ip_str)
 
+        ports = collect_ports_by_ip(matches, match.ip_str).map do |port|
+          Port.new(port: port)
+        end
+
+        reverse_dns_names = collect_hostnames_by_ip(matches, match.ip_str).map do |name|
+          ReverseDnsName.new(name: name)
+        end
+
         Artifact.new(
           data: match.ip_str,
           source: source,
           metadata: metadata,
           autonomous_system: as,
-          geolocation: geolocation
+          geolocation: geolocation,
+          ports: ports,
+          reverse_dns_names: reverse_dns_names
         )
       end
     end

data/lib/mihari/enrichers/ipinfo.rb

@@ -39,7 +39,7 @@ module Mihari
             data = JSON.parse(res.body.to_s)
 
             Structs::IPInfo::Response.from_dynamic! data
-          rescue HttpError
+          rescue HTTPError
             nil
           end
         end

data/lib/mihari/enrichers/shodan.rb

@@ -26,7 +26,7 @@ module Mihari
           data = JSON.parse(res.body.to_s)
 
           Structs::Shodan::InternetDBResponse.from_dynamic! data
-        rescue HttpError
+        rescue HTTPError
           nil
         end
         memoize :query

data/lib/mihari/errors.rb

@@ -11,11 +11,19 @@ module Mihari
 
   class FileNotFoundError < Error; end
 
-  class HttpError < Error; end
-
   class FeedParseError < Error; end
 
   class RuleValidationError < Error; end
 
   class ConfigurationError < Error; end
+
+  class HTTPError < Error; end
+
+  class UnsuccessfulStatusCodeError < HTTPError; end
+
+  class NetworkError < HTTPError; end
+
+  class TimeoutError < HTTPError; end
+
+  class SSLError < HTTPError; end
 end

data/lib/mihari/http.rb

@@ -90,11 +90,17 @@ module Mihari
 
         unless res.is_a?(Net::HTTPSuccess)
           code = res.code.to_i
-          raise HttpError, "Unsuccessful response code returned: #{code}"
+          raise UnsuccessfulStatusCodeError, "Unsuccessful response code returned: #{code}"
         end
 
         res
       end
+    rescue Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError, SocketError, Net::ProtocolError => e
+      raise NetworkError, e
+    rescue Timeout::Error => e
+      raise TimeoutError, e
+    rescue OpenSSL::SSL::SSLError => e
+      raise SSLError, e
     end
   end
 end

data/lib/mihari/schemas/analyzer.rb

@@ -89,10 +89,10 @@ module Mihari
       required(:analyzer).value(Types::String.enum("feed"))
       required(:query).value(:string)
       required(:selector).value(:string)
-      optional(:http_request_method).value(Types::HttpRequestMethods).default("GET")
+      optional(:http_request_method).value(Types::HTTPRequestMethods).default("GET")
       optional(:http_request_headers).value(:hash).default({})
       optional(:http_request_payload).value(:hash).default({})
-      optional(:http_request_payload_type).value(Types::HttpRequestPayloadTypes)
+      optional(:http_request_payload_type).value(Types::HTTPRequestPayloadTypes)
       optional(:options).hash(AnalyzerOptions)
     end
   end

data/lib/mihari/schemas/emitter.rb

@@ -27,7 +27,7 @@ module Mihari
     HTTP = Dry::Schema.Params do
       required(:emitter).value(Types::String.enum("http"))
       required(:url).value(:string)
-      optional(:http_request_method).value(Types::HttpRequestMethods).default("POST")
+      optional(:http_request_method).value(Types::HTTPRequestMethods).default("POST")
       optional(:http_request_headers).value(:hash).default({})
       optional(:template).value(:string)
     end

data/lib/mihari/structs/censys.rb

@@ -27,11 +27,23 @@ module Mihari
         end
       end
 
+      class Service < Dry::Struct
+        attribute :port, Types::Integer
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            port: d.fetch("port")
+          )
+        end
+      end
+
       class Hit < Dry::Struct
         attribute :ip, Types::String
         attribute :location, Location
         attribute :autonomous_system, AutonomousSystem
         attribute :metadata, Types::Hash
+        attribute :services, Types.Array(Service)
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
@@ -39,7 +51,8 @@ module Mihari
             ip: d.fetch("ip"),
             location: Location.from_dynamic!(d.fetch("location")),
             autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system")),
-            metadata: d
+            metadata: d,
+            services: d.fetch("services", []).map { |x| Service.from_dynamic!(x) }
           )
         end
       end

data/lib/mihari/structs/rule.rb

@@ -36,6 +36,9 @@ module Mihari
         # @return [Array, nil]
         attr_reader :errors
 
+        # @return [String]
+        attr_writer :id
+
         def initialize(data, yaml)
           @data = data.deep_symbolize_keys
           @yaml = yaml

data/lib/mihari/structs/shodan.rb

@@ -22,6 +22,7 @@ module Mihari
         attribute :location, Location
         attribute :domains, Types.Array(Types::String)
         attribute :ip_str, Types::String
+        attribute :port, Types::Integer
         attribute :metadata, Types::Hash
 
         def self.from_dynamic!(d)
@@ -40,6 +41,7 @@ module Mihari
             location: Location.from_dynamic!(d.fetch("location")),
             domains: d.fetch("domains"),
             ip_str: d.fetch("ip_str"),
+            port: d.fetch("port"),
             metadata: d
           )
         end

data/lib/mihari/types.rb

@@ -14,10 +14,8 @@ module Mihari
 
     DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
 
-    UrlscanDataTypes = Types::String.enum("ip", "domain", "url")
-
-    HttpRequestMethods = Types::String.enum("GET", "POST")
-    HttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
+    HTTPRequestMethods = Types::String.enum("GET", "POST")
+    HTTPRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
 
     EmitterTypes = Types::String.enum(
       "database",

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "4.5.0"
+  VERSION = "4.5.3"
 end

data/lib/mihari/web/app.rb

@@ -42,9 +42,15 @@ module Mihari
         end.to_app
       end
 
-      def run!(port: 9292, host: "localhost", threads: "0:5", verbose: false)
+      def run!(port: 9292, host: "localhost", threads: "1:1", verbose: false)
         url = "http://#{host}:#{port}"
 
+        # set maximum number of threads to use as PARALLEL_PROCESSOR_COUNT (if it is not set)
+        # ref. https://github.com/grosser/parallel#tips
+        # TODO: is this the best way?
+        _min_thread, max_thread = threads.split(":")
+        ENV["PARALLEL_PROCESSOR_COUNT"] = max_thread if ENV["PARALLEL_PROCESSOR_COUNT"].nil?
+
         Rack::Handler::Puma.run(instance, Port: port, Host: host, Threads: threads, Verbose: verbose) do |_launcher|
           Launchy.open(url) if ENV["RACK_ENV"] != "development"
         end

data/lib/mihari/web/endpoints/rules.rb

@@ -145,6 +145,7 @@ module Mihari
           end
 
           rule = Structs::Rule::Rule.from_yaml(yaml)
+          rule.id = id
 
           begin
             rule.validate!
@@ -159,7 +160,7 @@ module Mihari
             model = rule.to_model
             model.save
           rescue ActiveRecord::RecordNotUnique
-            error!({ message: "ID:#{rule.id} is already registered" }, 400)
+            error!({ message: "ID:#{model.id} is already registered" }, 400)
           end
 
           status 201

data/lib/mihari/web/public/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="viewport" content="width=device-width,initial-scale=1"/><link rel="icon" href="/static/favicon.ico"/><title>Mihari</title><script defer="defer" type="module" src="/static/js/chunk-vendors.3bdbaffb.js"></script><script defer="defer" type="module" src="/static/js/app.afd5025f.js"></script><link href="/static/css/chunk-vendors.da2a7bfc.css" rel="stylesheet"><link href="/static/css/app.2a5d3d21.css" rel="stylesheet"><script defer="defer" src="/static/js/chunk-vendors-legacy.d6b76c57.js" nomodule></script><script defer="defer" src="/static/js/app-legacy.c3595dce.js" nomodule></script></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="viewport" content="width=device-width,initial-scale=1"/><link rel="icon" href="/static/favicon.ico"/><title>Mihari</title><script defer="defer" type="module" src="/static/js/chunk-vendors.dde2116c.js"></script><script defer="defer" type="module" src="/static/js/app.823b5af7.js"></script><link href="/static/css/chunk-vendors.06251949.css" rel="stylesheet"><link href="/static/css/app.2a5d3d21.css" rel="stylesheet"><script defer="defer" src="/static/js/chunk-vendors-legacy.b110c129.js" nomodule></script><script defer="defer" src="/static/js/app-legacy.9d5c9c3d.js" nomodule></script></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>