mihari 3.3.0 → 3.6.0

data/lib/mihari/serializers/artifact.rb

@@ -3,7 +3,16 @@
 require "active_model_serializers"
 
 module Mihari
-  class ArtifactSerializer < ActiveModel::Serializer
-    attributes :id, :data, :data_type, :source
+  module Serializers
+    class ArtifactSerializer < ActiveModel::Serializer
+      attributes :id, :data, :data_type, :source
+
+      has_one :autonomous_system, serializer: AutonomousSystemSerializer
+      has_one :geolocation, serializer: GeolocationSerializer
+      has_one :whois_record, serializer: WhoisRecordSerializer
+
+      has_many :dns_records, serializer: DnsRecordSerializer
+      has_many :reverse_dns_names, serializer: ReverseDnsNameSerializer
+    end
   end
 end

data/lib/mihari/serializers/autonomous_system.rb

@@ -0,0 +1,9 @@
+require "active_model_serializers"
+
+module Mihari
+  module Serializers
+    class AutonomousSystemSerializer < ActiveModel::Serializer
+      attributes :asn
+    end
+  end
+end

data/lib/mihari/serializers/dns.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "active_model_serializers"
+
+module Mihari
+  module Serializers
+    class DnsRecordSerializer < ActiveModel::Serializer
+      attributes :resource, :value
+    end
+  end
+end

data/lib/mihari/serializers/geolocation.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "active_model_serializers"
+
+module Mihari
+  module Serializers
+    class GeolocationSerializer < ActiveModel::Serializer
+      attributes :country, :country_code
+    end
+  end
+end

data/lib/mihari/serializers/reverse_dns.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "active_model_serializers"
+
+module Mihari
+  module Serializers
+    class ReverseDnsNameSerializer < ActiveModel::Serializer
+      attributes :name
+    end
+  end
+end

data/lib/mihari/serializers/tag.rb

@@ -3,7 +3,9 @@
 require "active_model_serializers"
 
 module Mihari
-  class TagSerializer < ActiveModel::Serializer
-    attributes :id, :name
+  module Serializers
+    class TagSerializer < ActiveModel::Serializer
+      attributes :id, :name
+    end
   end
 end

data/lib/mihari/serializers/whois.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "active_model_serializers"
+
+module Mihari
+  module Serializers
+    class WhoisRecordSerializer < ActiveModel::Serializer
+      attributes :domain, :created_on, :updated_on, :expires_on, :registrar, :contacts
+    end
+  end
+end

data/lib/mihari/structs/censys.rb

@@ -0,0 +1,92 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module Censys
+      class AutonomousSystem < Dry::Struct
+        attribute :asn, Types::Int
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            asn: d.fetch("asn")
+          )
+        end
+      end
+
+      class Location < Dry::Struct
+        attribute :country, Types::String.optional
+        attribute :country_code, Types::String.optional
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            country: d["country"],
+            country_code: d["country_code"]
+          )
+        end
+      end
+
+      class Hit < Dry::Struct
+        attribute :ip, Types::String
+        attribute :location, Location
+        attribute :autonomous_system, AutonomousSystem
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            ip: d.fetch("ip"),
+            location: Location.from_dynamic!(d.fetch("location")),
+            autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system"))
+          )
+        end
+      end
+
+      class Links < Dry::Struct
+        attribute :next, Types::String
+        attribute :prev, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            next: d.fetch("next"),
+            prev: d.fetch("prev")
+          )
+        end
+      end
+
+      class Result < Dry::Struct
+        attribute :query, Types::String
+        attribute :total, Types::Int
+        attribute :hits, Types.Array(Hit)
+        attribute :links, Links
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            query: d.fetch("query"),
+            total: d.fetch("total"),
+            hits: d.fetch("hits", []).map { |x| Hit.from_dynamic!(x) },
+            links: Links.from_dynamic!(d.fetch("links"))
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :code, Types::Int
+        attribute :status, Types::String
+        attribute :result, Result
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            code: d.fetch("code"),
+            status: d.fetch("status"),
+            result: Result.from_dynamic!(d.fetch("result"))
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/onyphe.rb

@@ -0,0 +1,47 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module Onyphe
+      class Result < Dry::Struct
+        attribute :asn, Types::String
+        attribute :country_code, Types::String
+        attribute :ip, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            asn: d.fetch("asn"),
+            ip: d.fetch("ip"),
+            # Onyphe's country = 2-letter country code
+            country_code: d.fetch("country")
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :count, Types::Int
+        attribute :error, Types::Int
+        attribute :max_page, Types::Int
+        attribute :page, Types::String
+        attribute :results, Types.Array(Result)
+        attribute :status, Types::String
+        attribute :total, Types::Int
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            count: d.fetch("count"),
+            error: d.fetch("error"),
+            max_page: d.fetch("max_page"),
+            page: d.fetch("page"),
+            results: d.fetch("results").map { |x| Result.from_dynamic!(x) },
+            status: d.fetch("status"),
+            total: d.fetch("total")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/shodan.rb

@@ -0,0 +1,53 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module Shodan
+      class Location < Dry::Struct
+        attribute :country_code, Types::String
+        attribute :country_name, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            country_code: d.fetch("country_code"),
+            country_name: d.fetch("country_name")
+          )
+        end
+      end
+
+      class Match < Dry::Struct
+        attribute :asn, Types::String
+        attribute :hostnames, Types.Array(Types::String)
+        attribute :location, Location
+        attribute :domains, Types.Array(Types::String)
+        attribute :ip_str, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            asn: d.fetch("asn"),
+            hostnames: d.fetch("hostnames"),
+            location: Location.from_dynamic!(d.fetch("location")),
+            domains: d.fetch("domains"),
+            ip_str: d.fetch("ip_str")
+          )
+        end
+      end
+
+      class Result < Dry::Struct
+        attribute :matches, Types.Array(Match)
+        attribute :total, Types::Int
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            matches: d.fetch("matches", []).map { |x| Match.from_dynamic!(x) },
+            total: d.fetch("total")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/templates/rule.yml.erb

@@ -15,6 +15,9 @@ allowed_data_types: # Array<String> (Optional, defaults to ["hash", "ip", "domai
   - mail
 disallowed_data_values: [] # Array<String> (Optional, defaults to [])
 
+ignore_old_artifacts: true # Whether to ignore old artifacts from checking or not (Optional, defaults to true)
+ignore_threshold: 0 # Number of days to define whether an artifact is old or not (Optional, defaults to 0)
+
 queries: # Array<Hash> (required)
   - analyzer: shodan # String (required)
     query: ... # String (required)

data/lib/mihari/types.rb

@@ -0,0 +1,21 @@
+require "dry/types"
+
+module Mihari
+  module Types
+    include Dry.Types()
+
+    Int = Strict::Integer
+    Nil = Strict::Nil
+    Hash = Strict::Hash
+    String = Strict::String
+    Double = Strict::Float | Strict::Integer
+
+    DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
+
+    AnalyzerTypes = Types::String.enum(
+      "binaryedge", "censys", "circl", "dnpedia", "dnstwister",
+      "onyphe", "otx", "passivetotal", "pulsedive", "securitytrails",
+      "shodan", "virustotal"
+    )
+  end
+end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.3.0"
+  VERSION = "3.6.0"
 end

data/lib/mihari/web/app.rb

@@ -14,6 +14,7 @@ require "mihari/web/controllers/analyzers_controller"
 require "mihari/web/controllers/artifacts_controller"
 require "mihari/web/controllers/command_controller"
 require "mihari/web/controllers/config_controller"
+require "mihari/web/controllers/ip_address_controller"
 require "mihari/web/controllers/sources_controller"
 require "mihari/web/controllers/tags_controller"
 
@@ -31,6 +32,7 @@ module Mihari
     use Mihari::Controllers::ArtifactsController
     use Mihari::Controllers::CommandController
     use Mihari::Controllers::ConfigController
+    use Mihari::Controllers::IPAddressController
     use Mihari::Controllers::SourcesController
     use Mihari::Controllers::TagsController
 

data/lib/mihari/web/controllers/alerts_controller.rb

@@ -26,9 +26,7 @@ module Mihari
         title = params["title"]
 
         from_at = params["from_at"] || params["fromAt"]
-        from_at = DateTime.parse(from_at) if from_at
         to_at = params["to_at"] || params["toAt"]
-        to_at = DateTime.parse(to_at) if to_at
 
         alerts = Mihari::Alert.search(
           artifact_data: artifact_data,
@@ -55,8 +53,9 @@ module Mihari
       end
 
       delete "/api/alerts/:id" do
-        id = params["id"]
-        id = id.to_i
+        param :id, Integer, required: true
+
+        id = params["id"].to_i
 
         begin
           alert = Mihari::Alert.find(id)

data/lib/mihari/web/controllers/artifacts_controller.rb

@@ -3,9 +3,53 @@
 module Mihari
   module Controllers
     class ArtifactsController < BaseController
+      get "/api/artifacts/:id" do
+        param :id, Integer, required: true
+
+        id = params["id"].to_i
+
+        begin
+          artifact = Mihari::Artifact.includes(
+            :autonomous_system,
+            :geolocation,
+            :whois_record,
+            :dns_records,
+            :reverse_dns_names
+          ).find(id)
+        rescue ActiveRecord::RecordNotFound
+          status 404
+
+          return json({ message: "ID:#{id} is not found" })
+        end
+
+        # TODO: improve queries
+        alert_ids = Mihari::Artifact.where(data: artifact.data).pluck(:alert_id)
+        tag_ids = Mihari::Tagging.where(alert_id: alert_ids).pluck(:tag_id)
+        tag_names = Mihari::Tag.where(id: tag_ids).distinct.pluck(:name)
+
+        artifact_json = Serializers::ArtifactSerializer.new(artifact).as_json
+
+        # convert reverse DNS names into an array of string
+        # also change it as nil if it is empty
+        reverse_dns_names = (artifact_json[:reverse_dns_names] || []).filter_map { |v| v[:name] }
+        reverse_dns_names = nil if reverse_dns_names.empty?
+        artifact_json[:reverse_dns_names] = reverse_dns_names
+
+        # change DNS records as nil if it is empty
+        dns_records = artifact_json[:dns_records] || []
+        dns_records = nil if dns_records.empty?
+        artifact_json[:dns_records] = dns_records
+
+        # set tags
+        artifact_json[:tags] = tag_names
+
+        json artifact_json
+      end
+
       delete "/api/artifacts/:id" do
-        id = params["id"]
-        id = id.to_i
+        param :id, Integer, required: true
+
+        id = params["id"].to_i
 
         begin
           alert = Mihari::Artifact.find(id)

data/lib/mihari/web/controllers/ip_address_controller.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "http"
+require "json"
+
+module Mihari
+  module Controllers
+    class IPAddressController < BaseController
+      def query(ip)
+        headers = {}
+        token = Mihari.config.ipinfo_api_key
+        unless token.nil?
+          headers[:authorization] = "Bearer #{token}"
+        end
+
+        res = HTTP.headers(headers).get("https://ipinfo.io/#{ip}/json")
+        JSON.parse res.to_s
+      end
+
+      get "/api/ip_addresses/:ip" do
+        param :ip, String, required: true
+
+        ip = params["ip"].to_s
+
+        begin
+          data = query(ip)
+          json data
+        rescue HTTP::Error
+          status 404
+
+          json({ message: "IP:#{ip} is not found" })
+        end
+      end
+    end
+  end
+end