mihari 4.5.0 → 4.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dfd4bafdc33911546dce3477184426c581b9e3a40a762f32d1944870589ffcc9
-  data.tar.gz: 8765422118bcc6fe914439b416e6362551c9de0492397ce90c2c5bf7487d982a
+  metadata.gz: dd4057ff69b6e8ee676c7ebf7e672b6830d26a02e22ebe372d2786f5220ae8d3
+  data.tar.gz: 202c7bce234c823fb10fe40fc1f7af2c25d676f587871d0a49740fd8f3af4325
 SHA512:
-  metadata.gz: a0788f87ab0ef4838243e25ca71496408187add826900d0428af6f4a8e7f81093e81944d809425b2c716c778c35b725f757b656cec046adc2294b4a07d764b4f
-  data.tar.gz: 3c0c001e91aaec0090daeb117a7595a84172a355d527af0dab7f81dfee557b271697a0fbcf75c8a44d5e33b6fb1ff1f8b82e0c057dff91e85b4fb4a0a30d69ca
+  metadata.gz: ed3c42b58a305f20e7d981a1ae5e19d6efe493d7252645cf54f78c16fe38607305497e983f80d1d04ab8964f0166f19ace1ba62bece5ecdeadb0b6041d4be49f
+  data.tar.gz: 12966b0cc1143bcff8e9a44dd5904c7fa70c234f26613df35e5be162c5e18805e15adb1a8fd1ddd8c1ad84d221b528d411daa29b764e67eed4357a86f6bf0cc7

data/lib/mihari/analyzers/censys.rb

@@ -88,12 +88,17 @@ module Mihari
           )
         end
 
+        ports = hit.services.map(&:port).map do |port|
+          Port.new(port: port)
+        end
+
         Artifact.new(
           data: hit.ip,
           source: source,
           metadata: hit.metadata,
           autonomous_system: as,
-          geolocation: geolocation
+          geolocation: geolocation,
+          ports: ports
         )
       end
 

data/lib/mihari/analyzers/shodan.rb

@@ -23,10 +23,10 @@ module Mihari
         return [] unless results || results.empty?
 
         results = results.map { |result| Structs::Shodan::Result.from_dynamic!(result) }
-        results.map do |result|
-          matches = result.matches || []
-          matches.map { |match| build_artifact(match, matches) }
-        end.flatten.uniq(&:data)
+        matches = results.map { |result| result.matches || [] }.flatten
+
+        uniq_matches = matches.uniq(&:ip_str)
+        uniq_matches.map { |match| build_artifact(match, matches) }
       end
 
       private
@@ -94,6 +94,30 @@ module Mihari
         matches.select { |match| match.ip_str == ip }.map(&:metadata)
       end
 
+      #
+      # Collect ports from matches
+      #
+      # @param [Array<Structs::Shodan::Match>] matches
+      # @param [String] ip
+      #
+      # @return [Array<String>]
+      #
+      def collect_ports_by_ip(matches, ip)
+        matches.select { |match| match.ip_str == ip }.map(&:port)
+      end
+
+      #
+      # Collect hostnames from matches
+      #
+      # @param [Array<Structs::Shodan::Match>] matches
+      # @param [String] ip
+      #
+      # @return [Array<String>]
+      #
+      def collect_hostnames_by_ip(matches, ip)
+        matches.select { |match| match.ip_str == ip }.map(&:hostnames).flatten.uniq
+      end
+
       #
       # Build an artifact from a Shodan search API response
       #
@@ -116,12 +140,22 @@ module Mihari
 
         metadata = collect_metadata_by_ip(matches, match.ip_str)
 
+        ports = collect_ports_by_ip(matches, match.ip_str).map do |port|
+          Port.new(port: port)
+        end
+
+        reverse_dns_names = collect_hostnames_by_ip(matches, match.ip_str).map do |name|
+          ReverseDnsName.new(name: name)
+        end
+
         Artifact.new(
           data: match.ip_str,
           source: source,
           metadata: metadata,
           autonomous_system: as,
-          geolocation: geolocation
+          geolocation: geolocation,
+          ports: ports,
+          reverse_dns_names: reverse_dns_names
         )
       end
     end

data/lib/mihari/enrichers/ipinfo.rb

@@ -39,7 +39,7 @@ module Mihari
             data = JSON.parse(res.body.to_s)
 
             Structs::IPInfo::Response.from_dynamic! data
-          rescue HttpError
+          rescue HTTPError
             nil
           end
         end

data/lib/mihari/enrichers/shodan.rb

@@ -26,7 +26,7 @@ module Mihari
           data = JSON.parse(res.body.to_s)
 
           Structs::Shodan::InternetDBResponse.from_dynamic! data
-        rescue HttpError
+        rescue HTTPError
           nil
         end
         memoize :query

data/lib/mihari/errors.rb

@@ -11,11 +11,19 @@ module Mihari
 
   class FileNotFoundError < Error; end
 
-  class HttpError < Error; end
-
   class FeedParseError < Error; end
 
   class RuleValidationError < Error; end
 
   class ConfigurationError < Error; end
+
+  class HTTPError < Error; end
+
+  class UnsuccessfulStatusCodeError < HTTPError; end
+
+  class NetworkError < HTTPError; end
+
+  class TimeoutError < HTTPError; end
+
+  class SSLError < HTTPError; end
 end

data/lib/mihari/http.rb

@@ -90,11 +90,17 @@ module Mihari
 
         unless res.is_a?(Net::HTTPSuccess)
           code = res.code.to_i
-          raise HttpError, "Unsuccessful response code returned: #{code}"
+          raise UnsuccessfulStatusCodeError, "Unsuccessful response code returned: #{code}"
         end
 
         res
       end
+    rescue Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError, SocketError, Net::ProtocolError => e
+      raise NetworkError, e
+    rescue Timeout::Error => e
+      raise TimeoutError, e
+    rescue OpenSSL::SSL::SSLError => e
+      raise SSLError, e
     end
   end
 end

data/lib/mihari/schemas/analyzer.rb

@@ -89,10 +89,10 @@ module Mihari
       required(:analyzer).value(Types::String.enum("feed"))
       required(:query).value(:string)
       required(:selector).value(:string)
-      optional(:http_request_method).value(Types::HttpRequestMethods).default("GET")
+      optional(:http_request_method).value(Types::HTTPRequestMethods).default("GET")
       optional(:http_request_headers).value(:hash).default({})
       optional(:http_request_payload).value(:hash).default({})
-      optional(:http_request_payload_type).value(Types::HttpRequestPayloadTypes)
+      optional(:http_request_payload_type).value(Types::HTTPRequestPayloadTypes)
       optional(:options).hash(AnalyzerOptions)
     end
   end

data/lib/mihari/schemas/emitter.rb

@@ -27,7 +27,7 @@ module Mihari
     HTTP = Dry::Schema.Params do
       required(:emitter).value(Types::String.enum("http"))
       required(:url).value(:string)
-      optional(:http_request_method).value(Types::HttpRequestMethods).default("POST")
+      optional(:http_request_method).value(Types::HTTPRequestMethods).default("POST")
       optional(:http_request_headers).value(:hash).default({})
       optional(:template).value(:string)
     end

data/lib/mihari/structs/censys.rb

@@ -27,11 +27,23 @@ module Mihari
         end
       end
 
+      class Service < Dry::Struct
+        attribute :port, Types::Integer
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            port: d.fetch("port")
+          )
+        end
+      end
+
       class Hit < Dry::Struct
         attribute :ip, Types::String
         attribute :location, Location
         attribute :autonomous_system, AutonomousSystem
         attribute :metadata, Types::Hash
+        attribute :services, Types.Array(Service)
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
@@ -39,7 +51,8 @@ module Mihari
             ip: d.fetch("ip"),
             location: Location.from_dynamic!(d.fetch("location")),
             autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system")),
-            metadata: d
+            metadata: d,
+            services: d.fetch("services", []).map { |x| Service.from_dynamic!(x) }
           )
         end
       end

data/lib/mihari/structs/shodan.rb

@@ -22,6 +22,7 @@ module Mihari
         attribute :location, Location
         attribute :domains, Types.Array(Types::String)
         attribute :ip_str, Types::String
+        attribute :port, Types::Integer
         attribute :metadata, Types::Hash
 
         def self.from_dynamic!(d)
@@ -40,6 +41,7 @@ module Mihari
             location: Location.from_dynamic!(d.fetch("location")),
             domains: d.fetch("domains"),
             ip_str: d.fetch("ip_str"),
+            port: d.fetch("port"),
             metadata: d
           )
         end

data/lib/mihari/types.rb

@@ -14,10 +14,8 @@ module Mihari
 
     DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
 
-    UrlscanDataTypes = Types::String.enum("ip", "domain", "url")
-
-    HttpRequestMethods = Types::String.enum("GET", "POST")
-    HttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
+    HTTPRequestMethods = Types::String.enum("GET", "POST")
+    HTTPRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
 
     EmitterTypes = Types::String.enum(
       "database",

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "4.5.0"
+  VERSION = "4.5.1"
 end

data/lib/mihari/web/app.rb

@@ -42,9 +42,15 @@ module Mihari
         end.to_app
       end
 
-      def run!(port: 9292, host: "localhost", threads: "0:5", verbose: false)
+      def run!(port: 9292, host: "localhost", threads: "1:1", verbose: false)
         url = "http://#{host}:#{port}"
 
+        # set maximum number of threads to use as PARALLEL_PROCESSOR_COUNT (if it is not set)
+        # ref. https://github.com/grosser/parallel#tips
+        # TODO: is this the best way?
+        _min_thread, max_thread = threads.split(":")
+        ENV["PARALLEL_PROCESSOR_COUNT"] = max_thread if ENV["PARALLEL_PROCESSOR_COUNT"].nil?
+
         Rack::Handler::Puma.run(instance, Port: port, Host: host, Threads: threads, Verbose: verbose) do |_launcher|
           Launchy.open(url) if ENV["RACK_ENV"] != "development"
         end

data/mihari.gemspec

@@ -39,8 +39,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rerun", "~> 0.13"
   spec.add_development_dependency "rspec", "~> 3.11"
   spec.add_development_dependency "simplecov-lcov", "~> 0.8.0"
-  spec.add_development_dependency "standard", "~> 1.9"
-  spec.add_development_dependency "steep", "~> 0.51"
+  spec.add_development_dependency "standard", "~> 1.10"
+  spec.add_development_dependency "steep", "~> 0.52"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.1"
   spec.add_development_dependency "webmock", "~> 3.14"
@@ -54,7 +54,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dnpedia", "0.1.0"
   spec.add_dependency "dnstwister", "0.1.0"
   spec.add_dependency "dotenv", "2.7.6"
-  spec.add_dependency "dry-configurable", "0.14.0"
+  spec.add_dependency "dry-configurable", "0.15.0"
   spec.add_dependency "dry-container", "0.9.0"
   spec.add_dependency "dry-files", "0.1.0"
   spec.add_dependency "dry-initializer", "3.1.1"
@@ -81,7 +81,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "passive_circl", "0.1.0"
   spec.add_dependency "passivetotalx", "0.1.1"
   spec.add_dependency "plissken", "2.0.1"
-  spec.add_dependency "public_suffix", "4.0.6"
+  spec.add_dependency "public_suffix", "4.0.7"
   spec.add_dependency "pulsedive", "0.1.5"
   spec.add_dependency "puma", "5.6.4"
   spec.add_dependency "rack", "2.2.3"

data/sig/lib/mihari/structs/censys.rbs

@@ -14,10 +14,18 @@ module Mihari
         def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Censys::Location
       end
 
+      class Service
+        attr_reader port: Integer
+
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Censys::Service
+      end
+
       class Hit
         attr_reader ip: String
         attr_reader location: Mihari::Structs::Censys::Location
         attr_reader autonomous_system: Mihari::Structs::Censys::AutonomousSystem
+        attr_reader metadata: Hash[(String | Symbol), untyped]
+        attr_reader services: Array[Mihari::Structs::Censys::Service]
 
         def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Censys::Hit
       end

data/sig/lib/mihari/structs/shodan.rbs

@@ -14,6 +14,8 @@ module Mihari
         attr_reader location: Mihari::Structs::Shodan::Location
         attr_reader domains: Array[String]
         attr_reader ip_str: String
+        attr_reader port: Integer
+
         def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Shodan::Match
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 4.5.0
+  version: 4.5.1
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-04-14 00:00:00.000000000 Z
+date: 2022-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -184,28 +184,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: steep
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.51'
+        version: '0.52'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.51'
+        version: '0.52'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -380,14 +380,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 0.15.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 0.15.0
 - !ruby/object:Gem::Dependency
   name: dry-container
   requirement: !ruby/object:Gem::Requirement
@@ -758,14 +758,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.0.6
+        version: 4.0.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.0.6
+        version: 4.0.7
 - !ruby/object:Gem::Dependency
   name: pulsedive
   requirement: !ruby/object:Gem::Requirement