mihari 4.4.1 → 4.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15a224f944350480b79957d50a30742b9c4a0597c6dc8765e80269104175cab5
-  data.tar.gz: d873df7d4c4d30f39f62674a34bd1a28a4737d2183f92acc0f4243de2dd9eacf
+  metadata.gz: 266aaff37fdb3453ed52ba4391f25e2fc9d26c37996a4d00c462d2c9cb9a2771
+  data.tar.gz: 5d3397886e6c63199f49233668772a38ac508b9e32f3cd64f8925c3bf8d391f7
 SHA512:
-  metadata.gz: 9f538f70afd329f84c3962c8ec6c7693d8656fc105dcf0f9f30145ec5e74337c9414fdd5804255a7105105a14246e223bc5d8660980af2407db1c0595b4b40aa
-  data.tar.gz: ce2bec572a183d29c184b188f9314f714446a67c7599ba4855de566120351b2041c6b9f5a98b7b972aa4aa960a6ea233ac7c6509b1fdbbb081a5a08e22ab51c2
+  metadata.gz: 6df709c192e0a533d40604ee4fd076eb9d5f776d13165e60fe1687cf2fc2d008decbfd80c48cc809ab6667946e86b7a74490109e2bff9bf6c15d7c5767f490dc
+  data.tar.gz: 3350eeaa0d82da0322d463c30d69cfb2d09378449d37e681c34ec082d1d6505bb8bf1838c201779c3f3e618d716e029c25bbf6e0ac1329ad6c2291eef7c38a3a

data/lib/mihari/analyzers/censys.rb

@@ -88,12 +88,17 @@ module Mihari
           )
         end
 
+        ports = hit.services.map(&:port).map do |port|
+          Port.new(port: port)
+        end
+
         Artifact.new(
           data: hit.ip,
           source: source,
           metadata: hit.metadata,
           autonomous_system: as,
-          geolocation: geolocation
+          geolocation: geolocation,
+          ports: ports
         )
       end
 

data/lib/mihari/analyzers/shodan.rb

@@ -23,10 +23,10 @@ module Mihari
         return [] unless results || results.empty?
 
         results = results.map { |result| Structs::Shodan::Result.from_dynamic!(result) }
-        results.map do |result|
-          matches = result.matches || []
-          matches.map { |match| build_artifact(match, matches) }
-        end.flatten.uniq(&:data)
+        matches = results.map { |result| result.matches || [] }.flatten
+
+        uniq_matches = matches.uniq(&:ip_str)
+        uniq_matches.map { |match| build_artifact(match, matches) }
       end
 
       private
@@ -94,6 +94,30 @@ module Mihari
         matches.select { |match| match.ip_str == ip }.map(&:metadata)
       end
 
+      #
+      # Collect ports from matches
+      #
+      # @param [Array<Structs::Shodan::Match>] matches
+      # @param [String] ip
+      #
+      # @return [Array<String>]
+      #
+      def collect_ports_by_ip(matches, ip)
+        matches.select { |match| match.ip_str == ip }.map(&:port)
+      end
+
+      #
+      # Collect hostnames from matches
+      #
+      # @param [Array<Structs::Shodan::Match>] matches
+      # @param [String] ip
+      #
+      # @return [Array<String>]
+      #
+      def collect_hostnames_by_ip(matches, ip)
+        matches.select { |match| match.ip_str == ip }.map(&:hostnames).flatten.uniq
+      end
+
       #
       # Build an artifact from a Shodan search API response
       #
@@ -116,12 +140,22 @@ module Mihari
 
         metadata = collect_metadata_by_ip(matches, match.ip_str)
 
+        ports = collect_ports_by_ip(matches, match.ip_str).map do |port|
+          Port.new(port: port)
+        end
+
+        reverse_dns_names = collect_hostnames_by_ip(matches, match.ip_str).map do |name|
+          ReverseDnsName.new(name: name)
+        end
+
         Artifact.new(
           data: match.ip_str,
           source: source,
           metadata: metadata,
           autonomous_system: as,
-          geolocation: geolocation
+          geolocation: geolocation,
+          ports: ports,
+          reverse_dns_names: reverse_dns_names
         )
       end
     end

data/lib/mihari/commands/web.rb

@@ -8,7 +8,7 @@ module Mihari
           desc "web", "Launch the web app"
           method_option :port, type: :numeric, default: 9292, desc: "Hostname to listen on"
           method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
-          method_option :threads, type: :string, default: "0:16", desc: "min:max threads to use"
+          method_option :threads, type: :string, default: "1:1", desc: "min:max threads to use"
           method_option :verbose, type: :boolean, default: true, desc: "Report each request"
           def web
             port = options["port"]

data/lib/mihari/database.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
+# Make possible to use upper case acronyms in class names
+ActiveSupport::Inflector.inflections(:en) do |inflect|
+  inflect.acronym "CPE"
+end
+
 def env
   ENV["APP_ENV"] || ENV["RACK_ENV"]
 end
@@ -118,6 +123,20 @@ class AddYAMLToRulesSchema < ActiveRecord::Migration[7.0]
   end
 end
 
+class EnrichmentsV45Schema < ActiveRecord::Migration[7.0]
+  def change
+    create_table :cpes, if_not_exists: true do |t|
+      t.string :cpe, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+
+    create_table :ports, if_not_exists: true do |t|
+      t.integer :port, null: false
+      t.belongs_to :artifact, foreign_key: true
+    end
+  end
+end
+
 def adapter
   return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
   return "mysql2" if Mihari.config.database.start_with?("mysql2://")
@@ -147,7 +166,9 @@ module Mihari
           RuleSchema,
           AddeMetadataToArtifactSchema,
           # v4.4
-          AddYAMLToRulesSchema
+          AddYAMLToRulesSchema,
+          # v4.5
+          EnrichmentsV45Schema
         ].each { |schema| schema.migrate direction }
       end
       memoize :migrate unless test_env?

data/lib/mihari/enrichers/ipinfo.rb

@@ -39,7 +39,7 @@ module Mihari
             data = JSON.parse(res.body.to_s)
 
             Structs::IPInfo::Response.from_dynamic! data
-          rescue HttpError
+          rescue HTTPError
             nil
           end
         end

data/lib/mihari/enrichers/shodan.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "net/https"
+
+module Mihari
+  module Enrichers
+    class Shodan < Base
+      # @return [Boolean]
+      def valid?
+        true
+      end
+
+      class << self
+        include Memist::Memoizable
+
+        #
+        # Query Shodan Internet DB
+        #
+        # @param [String] ip
+        #
+        # @return [Mihari::Structs::Shodan::InternetDBResponse, nil]
+        #
+        def query(ip)
+          url = "https://internetdb.shodan.io/#{ip}"
+          res = HTTP.get(url)
+          data = JSON.parse(res.body.to_s)
+
+          Structs::Shodan::InternetDBResponse.from_dynamic! data
+        rescue HTTPError
+          nil
+        end
+        memoize :query
+      end
+    end
+  end
+end

data/lib/mihari/entities/artifact.rb

@@ -21,6 +21,12 @@ module Mihari
       expose :dns_records, using: Entities::DnsRecord, documentation: { type: Entities::DnsRecord, is_array: true, required: false }, as: :dnsRecords do |status, _options|
         status.dns_records.empty? ? nil : status.dns_records
       end
+      expose :ceps, using: Entities::CPE, documentation: { type: Entities::CPE, is_array: true, required: false }, as: :cpes do |status, _options|
+        status.cpes.empty? ? nil : status.cpes
+      end
+      expose :ports, using: Entities::Port, documentation: { type: Entities::Port, is_array: true, required: false }, as: :ports do |status, _options|
+        status.ports.empty? ? nil : status.ports
+      end
     end
   end
 end

data/lib/mihari/entities/cpe.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Entities
+    class CPE < Grape::Entity
+      expose :cpe, documentation: { type: String, required: true }
+    end
+  end
+end

data/lib/mihari/entities/port.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Entities
+    class Port < Grape::Entity
+      expose :port, documentation: { type: Integer, required: true }
+    end
+  end
+end

data/lib/mihari/errors.rb

@@ -11,11 +11,19 @@ module Mihari
 
   class FileNotFoundError < Error; end
 
-  class HttpError < Error; end
-
   class FeedParseError < Error; end
 
   class RuleValidationError < Error; end
 
   class ConfigurationError < Error; end
+
+  class HTTPError < Error; end
+
+  class UnsuccessfulStatusCodeError < HTTPError; end
+
+  class NetworkError < HTTPError; end
+
+  class TimeoutError < HTTPError; end
+
+  class SSLError < HTTPError; end
 end

data/lib/mihari/http.rb

@@ -90,11 +90,17 @@ module Mihari
 
         unless res.is_a?(Net::HTTPSuccess)
           code = res.code.to_i
-          raise HttpError, "Unsuccessful response code returned: #{code}"
+          raise UnsuccessfulStatusCodeError, "Unsuccessful response code returned: #{code}"
         end
 
         res
       end
+    rescue Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError, SocketError, Net::ProtocolError => e
+      raise NetworkError, e
+    rescue Timeout::Error => e
+      raise TimeoutError, e
+    rescue OpenSSL::SSL::SSLError => e
+      raise SSLError, e
     end
   end
 end

data/lib/mihari/models/alert.rb

@@ -35,7 +35,9 @@ module Mihari
               :geolocation,
               :whois_record,
               :dns_records,
-              :reverse_dns_names
+              :reverse_dns_names,
+              :cpes,
+              :ports
             ]
           },
           :tags

data/lib/mihari/models/artifact.rb

@@ -16,7 +16,9 @@ module Mihari
     has_one :geolocation, dependent: :destroy
     has_one :whois_record, dependent: :destroy
 
+    has_many :cpes, dependent: :destroy
     has_many :dns_records, dependent: :destroy
+    has_many :ports, dependent: :destroy
     has_many :reverse_dns_names, dependent: :destroy
 
     include ActiveModel::Validations
@@ -94,7 +96,7 @@ module Mihari
     end
 
     #
-    # Enrich(add) geolocation
+    # Enrich AS
     #
     def enrich_autonomous_system
       return unless can_enrich_autonomous_system?
@@ -102,6 +104,24 @@ module Mihari
       self.autonomous_system = AutonomousSystem.build_by_ip(data)
     end
 
+    #
+    # Enrich ports
+    #
+    def enrich_ports
+      return unless can_enrich_ports?
+
+      self.ports = Port.build_by_ip(data)
+    end
+
+    #
+    # Enrich CPEs
+    #
+    def enrich_cpes
+      return unless can_enrich_cpes?
+
+      self.cpes = CPE.build_by_ip(data)
+    end
+
     #
     # Enrich all the enrichable relationships of the artifact
     #
@@ -111,6 +131,8 @@ module Mihari
       enrich_geolocation
       enrich_reverse_dns
       enrich_whois
+      enrich_ports
+      enrich_cpes
     end
 
     private
@@ -140,5 +162,13 @@ module Mihari
     def can_enrich_autonomous_system?
       data_type == "ip" && autonomous_system.nil?
     end
+
+    def can_enrich_ports?
+      data_type == "ip" && ports.empty?
+    end
+
+    def can_enrich_cpes?
+      data_type == "ip" && cpes.empty?
+    end
   end
 end

data/lib/mihari/models/cpe.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Mihari
+  class CPE < ActiveRecord::Base
+    belongs_to :artifact
+
+    class << self
+      #
+      # Build CPEs
+      #
+      # @param [String] ip
+      #
+      # @return [Array<Mihari::CPE>]
+      #
+      def build_by_ip(ip)
+        res = Enrichers::Shodan.query(ip)
+        return if res.nil?
+
+        res.cpes.map { |cpe| new(cpe: cpe) }
+      end
+    end
+  end
+end

data/lib/mihari/models/port.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Mihari
+  class Port < ActiveRecord::Base
+    belongs_to :artifact
+
+    class << self
+      #
+      # Build ports
+      #
+      # @param [String] ip
+      #
+      # @return [Array<Mihari::Port>]
+      #
+      def build_by_ip(ip)
+        res = Enrichers::Shodan.query(ip)
+        return if res.nil?
+
+        res.ports.map { |port| new(port: port) }
+      end
+    end
+  end
+end

data/lib/mihari/models/reverse_dns.rb

@@ -13,10 +13,10 @@ module Mihari
       # @return [Array<Mihari::ReverseDnsName>]
       #
       def build_by_ip(ip)
-        names = Resolv.getnames(ip)
-        names.map { |name| new(name: name) }
-      rescue Resolv::ResolvError
-        []
+        res = Enrichers::Shodan.query(ip)
+        return if res.nil?
+
+        res.hostnames.map { |name| new(name: name) }
       end
     end
   end

data/lib/mihari/schemas/analyzer.rb

@@ -89,10 +89,10 @@ module Mihari
       required(:analyzer).value(Types::String.enum("feed"))
       required(:query).value(:string)
       required(:selector).value(:string)
-      optional(:http_request_method).value(Types::HttpRequestMethods).default("GET")
+      optional(:http_request_method).value(Types::HTTPRequestMethods).default("GET")
       optional(:http_request_headers).value(:hash).default({})
       optional(:http_request_payload).value(:hash).default({})
-      optional(:http_request_payload_type).value(Types::HttpRequestPayloadTypes)
+      optional(:http_request_payload_type).value(Types::HTTPRequestPayloadTypes)
       optional(:options).hash(AnalyzerOptions)
     end
   end

data/lib/mihari/schemas/emitter.rb

@@ -27,7 +27,7 @@ module Mihari
     HTTP = Dry::Schema.Params do
       required(:emitter).value(Types::String.enum("http"))
       required(:url).value(:string)
-      optional(:http_request_method).value(Types::HttpRequestMethods).default("POST")
+      optional(:http_request_method).value(Types::HTTPRequestMethods).default("POST")
       optional(:http_request_headers).value(:hash).default({})
       optional(:template).value(:string)
     end

data/lib/mihari/structs/censys.rb

@@ -27,11 +27,23 @@ module Mihari
         end
       end
 
+      class Service < Dry::Struct
+        attribute :port, Types::Integer
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            port: d.fetch("port")
+          )
+        end
+      end
+
       class Hit < Dry::Struct
         attribute :ip, Types::String
         attribute :location, Location
         attribute :autonomous_system, AutonomousSystem
         attribute :metadata, Types::Hash
+        attribute :services, Types.Array(Service)
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
@@ -39,7 +51,8 @@ module Mihari
             ip: d.fetch("ip"),
             location: Location.from_dynamic!(d.fetch("location")),
             autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system")),
-            metadata: d
+            metadata: d,
+            services: d.fetch("services", []).map { |x| Service.from_dynamic!(x) }
           )
         end
       end

data/lib/mihari/structs/rule.rb

@@ -36,6 +36,9 @@ module Mihari
         # @return [Array, nil]
         attr_reader :errors
 
+        # @return [String]
+        attr_writer :id
+
         def initialize(data, yaml)
           @data = data.deep_symbolize_keys
           @yaml = yaml

data/lib/mihari/structs/shodan.rb

@@ -22,6 +22,7 @@ module Mihari
         attribute :location, Location
         attribute :domains, Types.Array(Types::String)
         attribute :ip_str, Types::String
+        attribute :port, Types::Integer
         attribute :metadata, Types::Hash
 
         def self.from_dynamic!(d)
@@ -40,6 +41,7 @@ module Mihari
             location: Location.from_dynamic!(d.fetch("location")),
             domains: d.fetch("domains"),
             ip_str: d.fetch("ip_str"),
+            port: d.fetch("port"),
             metadata: d
           )
         end
@@ -57,6 +59,31 @@ module Mihari
           )
         end
       end
+
+      class InternetDBResponse < Dry::Struct
+        attribute :ip, Types::String
+        attribute :ports, Types.Array(Types::Int)
+        attribute :cpes, Types.Array(Types::String)
+        attribute :hostnames, Types.Array(Types::String)
+        attribute :tags, Types.Array(Types::String)
+        attribute :vulns, Types.Array(Types::String)
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            ip: d.fetch("ip"),
+            ports: d.fetch("ports"),
+            cpes: d.fetch("cpes"),
+            hostnames: d.fetch("hostnames"),
+            tags: d.fetch("tags"),
+            vulns: d.fetch("vulns")
+          )
+        end
+
+        def self.from_json!(json)
+          from_dynamic!(JSON.parse(json))
+        end
+      end
     end
   end
 end

data/lib/mihari/types.rb

@@ -14,10 +14,8 @@ module Mihari
 
     DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
 
-    UrlscanDataTypes = Types::String.enum("ip", "domain", "url")
-
-    HttpRequestMethods = Types::String.enum("GET", "POST")
-    HttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
+    HTTPRequestMethods = Types::String.enum("GET", "POST")
+    HTTPRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
 
     EmitterTypes = Types::String.enum(
       "database",

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "4.4.1"
+  VERSION = "4.5.2"
 end

data/lib/mihari/web/api.rb

@@ -3,7 +3,6 @@
 # Endpoints
 require "mihari/web/endpoints/alerts"
 require "mihari/web/endpoints/artifacts"
-require "mihari/web/endpoints/command"
 require "mihari/web/endpoints/configs"
 require "mihari/web/endpoints/ip_addresses"
 require "mihari/web/endpoints/rules"
@@ -17,7 +16,6 @@ module Mihari
 
     mount Endpoints::Alerts
     mount Endpoints::Artifacts
-    mount Endpoints::Command
     mount Endpoints::Configs
     mount Endpoints::IPAddresses
     mount Endpoints::Rules

data/lib/mihari/web/app.rb

@@ -42,9 +42,15 @@ module Mihari
         end.to_app
       end
 
-      def run!(port: 9292, host: "localhost", threads: "0:5", verbose: false)
+      def run!(port: 9292, host: "localhost", threads: "1:1", verbose: false)
         url = "http://#{host}:#{port}"
 
+        # set maximum number of threads to use as PARALLEL_PROCESSOR_COUNT (if it is not set)
+        # ref. https://github.com/grosser/parallel#tips
+        # TODO: is this the best way?
+        _min_thread, max_thread = threads.split(":")
+        ENV["PARALLEL_PROCESSOR_COUNT"] = max_thread if ENV["PARALLEL_PROCESSOR_COUNT"].nil?
+
         Rack::Handler::Puma.run(instance, Port: port, Host: host, Threads: threads, Verbose: verbose) do |_launcher|
           Launchy.open(url) if ENV["RACK_ENV"] != "development"
         end

data/lib/mihari/web/endpoints/artifacts.rb

@@ -54,7 +54,9 @@ module Mihari
               :geolocation,
               :whois_record,
               :dns_records,
-              :reverse_dns_names
+              :reverse_dns_names,
+              :cpes,
+              :ports
             ).find(id)
           rescue ActiveRecord::RecordNotFound
             error!({ message: "ID:#{id} is not found" }, 404)

data/lib/mihari/web/endpoints/rules.rb

@@ -145,6 +145,7 @@ module Mihari
           end
 
           rule = Structs::Rule::Rule.from_yaml(yaml)
+          rule.id = id
 
           begin
             rule.validate!
@@ -159,7 +160,7 @@ module Mihari
             model = rule.to_model
             model.save
           rescue ActiveRecord::RecordNotUnique
-            error!({ message: "ID:#{rule.id} is already registered" }, 400)
+            error!({ message: "ID:#{model.id} is already registered" }, 400)
           end
 
           status 201

data/lib/mihari/web/public/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="viewport" content="width=device-width,initial-scale=1"/><link rel="icon" href="/static/favicon.ico"/><title>Mihari</title><script defer="defer" type="module" src="/static/js/chunk-vendors.3bdbaffb.js"></script><script defer="defer" type="module" src="/static/js/app.40749592.js"></script><link href="/static/css/chunk-vendors.da2a7bfc.css" rel="stylesheet"><link href="/static/css/app.de5845d8.css" rel="stylesheet"><script defer="defer" src="/static/js/chunk-vendors-legacy.d6b76c57.js" nomodule></script><script defer="defer" src="/static/js/app-legacy.f550d6ae.js" nomodule></script></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="viewport" content="width=device-width,initial-scale=1"/><link rel="icon" href="/static/favicon.ico"/><title>Mihari</title><script defer="defer" type="module" src="/static/js/chunk-vendors.dde2116c.js"></script><script defer="defer" type="module" src="/static/js/app.823b5af7.js"></script><link href="/static/css/chunk-vendors.06251949.css" rel="stylesheet"><link href="/static/css/app.2a5d3d21.css" rel="stylesheet"><script defer="defer" src="/static/js/chunk-vendors-legacy.b110c129.js" nomodule></script><script defer="defer" src="/static/js/app-legacy.9d5c9c3d.js" nomodule></script></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>