mihari 5.1.1 → 5.1.2

data/lib/mihari/clients/virustotal.rb

@@ -5,8 +5,7 @@ module Mihari
     class VirusTotal < Base
       #
       # @param [String] base_url
-      # @param [String] id
-      # @param [String] secret
+      # @param [String, nil] api_key
       # @param [Hash] headers
       #
       def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {})
@@ -20,6 +19,8 @@ module Mihari
       #
       # @param [String] query
       #
+      # @return [Hash]
+      #
       def domain_search(query)
         _get("/api/v3/domains/#{query}/resolutions")
       end
@@ -27,6 +28,8 @@ module Mihari
       #
       # @param [String] query
       #
+      # @return [Hash]
+      #
       def ip_search(query)
         _get("/api/v3/ip_addresses/#{query}/resolutions")
       end
@@ -35,6 +38,8 @@ module Mihari
       # @param [String] query
       # @param [String, nil] cursor
       #
+      # @return [Hash]
+      #
       def intel_search(query, cursor: nil)
         params = { query: query, cursor: cursor }.compact
         _get("/api/v3/intelligence/search", params: params)
@@ -42,11 +47,12 @@ module Mihari
 
       private
 
-      #
       #
       # @param [String] path
       # @param [Hash] params
       #
+      # @return [Hash]
+      #
       def _get(path, params: {})
         res = get(path, params: params)
         JSON.parse(res.body.to_s)

data/lib/mihari/clients/zoomeye.rb

@@ -5,6 +5,11 @@ module Mihari
     class ZoomEye < Base
       attr_reader :api_key
 
+      #
+      # @param [String] base_url
+      # @param [String, nil] api_key
+      # @param [Hash] headers
+      #
       def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {})
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
@@ -52,11 +57,12 @@ module Mihari
 
       private
 
-      #
       #
       # @param [String] path
       # @param [Hash] params
       #
+      # @return [Hash, nil]
+      #
       def _get(path, params: {})
         res = get(path, params: params)
         JSON.parse(res.body.to_s)

data/lib/mihari/commands/database.rb

@@ -3,8 +3,6 @@
 module Mihari
   module Commands
     module Database
-      include Mixins::Database
-
       def self.included(thor)
         thor.class_eval do
           desc "migrate", "Migrate DB schemas"
@@ -12,14 +10,11 @@ module Mihari
           #
           # @param [String] direction
           #
-          #
           def migrate(direction = "up")
             verbose = options["verbose"]
             ActiveRecord::Migration.verbose = verbose
 
-            with_db_connection do
-              Mihari::Database.migrate(direction.to_sym)
-            end
+            Mihari::Database.with_db_connection { Mihari::Database.migrate(direction.to_sym) }
           end
         end
       end

data/lib/mihari/commands/searcher.rb

@@ -3,7 +3,6 @@
 module Mihari
   module Commands
     module Searcher
-      include Mixins::Database
       include Mixins::ErrorNotification
 
       def self.included(thor)
@@ -16,7 +15,7 @@ module Mihari
           # @param [String] path_or_id
           #
           def search(path_or_id)
-            with_db_connection do
+            Mihari::Database.with_db_connection do
               rule = Structs::Rule.from_path_or_id path_or_id
 
               # validate

data/lib/mihari/database.rb

@@ -164,6 +164,15 @@ module Mihari
 
         ActiveRecord::Base.clear_active_connections!
       end
+
+      def with_db_connection
+        Mihari::Database.connect
+        yield
+      rescue ActiveRecord::StatementInvalid
+        Mihari.logger.error("You haven't finished the DB migration! Please run 'mihari db migrate'.")
+      ensure
+        Mihari::Database.close
+      end
     end
   end
 end

data/lib/mihari/structs/censys.rb

@@ -4,8 +4,17 @@ module Mihari
   module Structs
     module Censys
       class AutonomousSystem < Dry::Struct
+        include Mixins::AutonomousSystem
+
         attribute :asn, Types::Int
 
+        #
+        # @return [Mihari::AutonomousSystem]
+        #
+        def to_as
+          Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -18,6 +27,20 @@ module Mihari
         attribute :country, Types::String.optional
         attribute :country_code, Types::String.optional
 
+        #
+        # @return [Mihari::Geolocation] <description>
+        #
+        def to_geolocation
+          # sometimes Censys overlooks country
+          # then set geolocation as nil
+          return nil if country.nil?
+
+          Mihari::Geolocation.new(
+            country: country,
+            country_code: country_code
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -30,6 +53,13 @@ module Mihari
       class Service < Dry::Struct
         attribute :port, Types::Integer
 
+        #
+        # @return [Mihari::Port]
+        #
+        def to_port
+          Port.new(port: port)
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -45,6 +75,29 @@ module Mihari
         attribute :metadata, Types::Hash
         attribute :services, Types.Array(Service)
 
+        #
+        # @return [Array<Mihari::Port>]
+        #
+        def to_ports
+          services.map(&:to_port)
+        end
+
+        #
+        # @param [String] source
+        #
+        # @return [Mihari::Artifact]
+        #
+        def to_artifact(source = "Censys")
+          Artifact.new(
+            data: ip,
+            source: source,
+            metadata: metadata,
+            autonomous_system: autonomous_system.to_as,
+            geolocation: location.to_geolocation,
+            ports: to_ports
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -76,6 +129,15 @@ module Mihari
         attribute :hits, Types.Array(Hit)
         attribute :links, Links
 
+        #
+        # @param [String] source
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def to_artifacts(source = "Censys")
+          hits.map { |hit| hit.to_artifact(source) }
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(

data/lib/mihari/structs/greynoise.rb

@@ -4,10 +4,29 @@ module Mihari
   module Structs
     module GreyNoise
       class Metadata < Dry::Struct
+        include Mixins::AutonomousSystem
+
         attribute :country, Types::String
         attribute :country_code, Types::String
         attribute :asn, Types::String
 
+        #
+        # @return [Mihari::AutonomousSystem]
+        #
+        def to_as
+          Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
+        end
+
+        #
+        # @return [Mihari::Geolocation]
+        #
+        def to_geolocation
+          Mihari::Geolocation.new(
+            country: country,
+            country_code: country_code
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -23,6 +42,21 @@ module Mihari
         attribute :metadata, Metadata
         attribute :metadata_, Types::Hash
 
+        #
+        # @param [String] source
+        #
+        # @return [Mihari::Artifact]
+        #
+        def to_artifact(source = "GreyNoise")
+          Mihari::Artifact.new(
+            data: ip,
+            source: source,
+            metadata: metadata_,
+            autonomous_system: metadata.to_as,
+            geolocation: metadata.to_geolocation
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -40,6 +74,15 @@ module Mihari
         attribute :message, Types::String
         attribute :query, Types::String
 
+        #
+        # @param [String] source
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def to_artifacts(source = "GreyNoise")
+          data.map { |datum| datum.to_artifact(source) }
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(

data/lib/mihari/structs/onyphe.rb

@@ -4,11 +4,47 @@ module Mihari
   module Structs
     module Onyphe
       class Result < Dry::Struct
+        include Mixins::AutonomousSystem
+
         attribute :asn, Types::String
         attribute :country_code, Types::String.optional
         attribute :ip, Types::String
         attribute :metadata, Types::Hash
 
+        #
+        # @param [String] source
+        #
+        # @return [Mihari::Artifact]
+        #
+        def to_artifact(source = "Onyphe")
+          Mihari::Artifact.new(
+            data: ip,
+            source: source,
+            metadata: metadata,
+            autonomous_system: to_as,
+            geolocation: to_geolocation
+          )
+        end
+
+        #
+        # @return [Mihari::Geolocation, nil]
+        #
+        def to_geolocation
+          return nil if country_code.nil?
+
+          Mihari::Geolocation.new(
+            country: NormalizeCountry(country_code, to: :short),
+            country_code: country_code
+          )
+        end
+
+        #
+        # @return [Mihari::AutonomousSystem]
+        #
+        def to_as
+          Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -30,6 +66,15 @@ module Mihari
         attribute :status, Types::String
         attribute :total, Types::Int
 
+        #
+        # @param [String] source
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def to_artifacts(source = "Onyphe")
+          results.map { |result| result.to_artifact(source) }
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(

data/lib/mihari/structs/shodan.rb

@@ -7,6 +7,18 @@ module Mihari
         attribute :country_code, Types::String.optional
         attribute :country_name, Types::String.optional
 
+        #
+        # @return [Mihari::Geolocation, nil]
+        #
+        def to_geolocation
+          return nil if country_name.nil? && country_code.nil?
+
+          Mihari::Geolocation.new(
+            country: country_name,
+            country_code: country_code
+          )
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
@@ -17,6 +29,8 @@ module Mihari
       end
 
       class Match < Dry::Struct
+        include Mixins::AutonomousSystem
+
         attribute :asn, Types::String.optional
         attribute :hostnames, Types.Array(Types::String)
         attribute :location, Location
@@ -25,6 +39,15 @@ module Mihari
         attribute :port, Types::Integer
         attribute :metadata, Types::Hash
 
+        #
+        # @return [Mihari::AutonomousSystem, nil]
+        #
+        def to_asn
+          return nil if asn.nil?
+
+          Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
 
@@ -51,6 +74,66 @@ module Mihari
         attribute :matches, Types.Array(Match)
         attribute :total, Types::Int
 
+        #
+        # Collect metadata from matches
+        #
+        # @param [String] ip
+        #
+        # @return [Array<Hash>]
+        #
+        def collect_metadata_by_ip(ip)
+          matches.select { |match| match.ip_str == ip }.map(&:metadata)
+        end
+
+        #
+        # Collect ports from matches
+        #
+        # @param [String] ip
+        #
+        # @return [Array<String>]
+        #
+        def collect_ports_by_ip(ip)
+          matches.select { |match| match.ip_str == ip }.map(&:port)
+        end
+
+        #
+        # Collect hostnames from matches
+        #
+        # @param [String] ip
+        #
+        # @return [Array<String>]
+        #
+        def collect_hostnames_by_ip(ip)
+          matches.select { |match| match.ip_str == ip }.map(&:hostnames).flatten.uniq
+        end
+
+        #
+        # @param [Source] source
+        #
+        # @return [Array<Mihari::Artifact>]
+        #
+        def to_artifacts(source = "Shodan")
+          matches.map do |match|
+            metadata = collect_metadata_by_ip(match.ip_str)
+            ports = collect_ports_by_ip(match.ip_str).map do |port|
+              Mihari::Port.new(port: port)
+            end
+            reverse_dns_names = collect_hostnames_by_ip(match.ip_str).map do |name|
+              Mihari::ReverseDnsName.new(name: name)
+            end
+
+            Mihari::Artifact.new(
+              data: match.ip_str,
+              source: source,
+              metadata: metadata,
+              autonomous_system: match.to_asn,
+              geolocation: match.location.to_geolocation,
+              ports: ports,
+              reverse_dns_names: reverse_dns_names
+            )
+          end
+        end
+
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "5.1.1"
+  VERSION = "5.1.2"
 end

data/lib/mihari/web/middleware/connection_adapter.rb

@@ -1,14 +1,12 @@
 module Mihari
   module Middleware
     class ConnectionAdapter
-      include Mixins::Database
-
       def initialize(app)
         @app = app
       end
 
       def call(env)
-        with_db_connection do
+        Mihari::Database.with_db_connection do
           status, headers, body = @app.call(env)
 
           [status, headers, body]