mihari 3.7.1 → 3.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50093699c012400a3f870f4bc44e373d8b2bf897ba47026f68f91c0899eaa42a
-  data.tar.gz: d690e4c92aa71ed193dfce5576b305698a74d3cf5ca99e0e6b7e696fd502254f
+  metadata.gz: 47f7a6231abe8eee1a1a4e4bc6f905d5f08fba2b68b1435d1621ab85e9e583c0
+  data.tar.gz: 591a8c71130095d6ac963fb333179d087f00727f45ab545e4a674a3c42a2af46
 SHA512:
-  metadata.gz: 27d489befd3a96dd8da1a8c5117f148500cd76acd66124c2d9111d15aecbb92e52ac5432edb1f7af1a69d4241ed90698aba615a802a89f2d20f2caf17154752e
-  data.tar.gz: 1f1cf73cb2eabb643d6be55786aa2e4e598c19d2e6de0c6467ea49667c04596e4366a8c301efb189f125429e05ba2e96e8ba8c79baf775e474ed0869705d1048
+  metadata.gz: da1c99c406086a9d8d029f8421a3316a5a6976c5b8fbc839a910bd0d61f04a3297614582974dbda925722308d206db9d2889fe44ae213df2650f1482cc15e918
+  data.tar.gz: edf130cb5027bbb6abfea72b150266b8768494b627162da814c93b38d54fe2597a870d78cd9f8f33238512e5a3205cb84a67b9da0589b71393b67fef062ad9b1

data/README.md

@@ -46,7 +46,7 @@ Mihari supports the following services by default.
 - [Shodan](https://shodan.io)
 - [Spyse](https://spyse.com)
 - [urlscan.io](https://urlscan.io)
-- [VirusTotal](http://virustotal.com)
+- [VirusTotal](http://virustotal.com) & [VirusTotal Intelligence](https://www.virustotal.com/gui/intelligence-overview)
 - [ZoomEye](https://zoomeye.org)
 
 ## Docs
@@ -64,5 +64,3 @@ The gem is available as open source under the terms of the [MIT License](https:/
 ## Acknowledgement
 
 Mihari is proudly supported by [Tines.io](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki), The SOAR Platform for Enterprise Security Teams.
-
-$ bundle exec rbs -rpathname  --repo=gem_rbs/gems -ractivesupport -ractionpack -ractivejob -ractivemodel -ractionview -ractiverecord -rrailties -I sig validate

data/lib/mihari/analyzers/rule.rb

@@ -4,6 +4,30 @@ require "uuidtools"
 
 module Mihari
   module Analyzers
+    ANALYZER_TO_CLASS = {
+      "binaryedge" => BinaryEdge,
+      "censys" => Censys,
+      "circl" => CIRCL,
+      "crtsh" => Crtsh,
+      "dnpedia" => DNPedia,
+      "dnstwister" => DNSTwister,
+      "onyphe" => Onyphe,
+      "otx" => OTX,
+      "passivetotal" => PassiveTotal,
+      "pt" => PassiveTotal,
+      "pulsedive" => Pulsedive,
+      "securitytrails" => SecurityTrails,
+      "shodan" => Shodan,
+      "spyse" => Spyse,
+      "st" => SecurityTrails,
+      "urlscan" => Urlscan,
+      "virustotal_intelligence" => VirusTotalIntelligence,
+      "virustotal" => VirusTotal,
+      "vt_intel" => VirusTotalIntelligence,
+      "vt" => VirusTotal,
+      "zoomeye" => ZoomEye
+    }.freeze
+
     class Rule < Base
       include Mihari::Mixins::DisallowedDataValue
 
@@ -26,25 +50,6 @@ module Mihari
         validate_analyzer_configurations
       end
 
-      ANALYZER_TO_CLASS = {
-        "binaryedge" => BinaryEdge,
-        "censys" => Censys,
-        "circl" => CIRCL,
-        "crtsh" => Crtsh,
-        "dnpedia" => DNPedia,
-        "dnstwister" => DNSTwister,
-        "onyphe" => Onyphe,
-        "otx" => OTX,
-        "passivetotal" => PassiveTotal,
-        "pulsedive" => Pulsedive,
-        "securitytrails" => SecurityTrails,
-        "shodan" => Shodan,
-        "spyse" => Spyse,
-        "urlscan" => Urlscan,
-        "virustotal" => VirusTotal,
-        "zoomeye" => ZoomEye
-      }.freeze
-
       #
       # Returns a list of artifacts matched with queries
       #

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "virustotal"
+
+module Mihari
+  module Analyzers
+    class VirusTotalIntelligence < Base
+      param :query
+      option :title, default: proc { "VirusTotal Intelligence search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
+
+      def initialize(*args, **kwargs)
+        super
+
+        @query = query
+      end
+
+      def artifacts
+        responses = search_witgh_cursor
+        responses.map do |response|
+          response.data.map(&:value)
+        end.flatten.compact.uniq
+      end
+
+      private
+
+      def configuration_keys
+        %w[virustotal_api_key]
+      end
+
+      #
+      # VT API
+      #
+      # @return [::VirusTotal::API]
+      #
+      def api
+        @api = ::VirusTotal::API.new(key: Mihari.config.virustotal_api_key)
+      end
+
+      #
+      # Search with cursor
+      #
+      # @return [Array<Structs::VirusTotalIntelligence::Response>]
+      #
+      def search_witgh_cursor
+        cursor = nil
+        responses = []
+
+        loop do
+          response = Structs::VirusTotalIntelligence::Response.from_dynamic!(api.intelligence.search(query, cursor: cursor))
+          responses << response
+
+          break if response.meta.cursor.nil?
+
+          cursor = response.meta.cursor
+        end
+
+        responses
+      end
+    end
+  end
+end

data/lib/mihari/cli/analyzer.rb

@@ -14,6 +14,7 @@ require "mihari/commands/securitytrails"
 require "mihari/commands/shodan"
 require "mihari/commands/spyse"
 require "mihari/commands/urlscan"
+require "mihari/commands/virustotal_intelligence"
 require "mihari/commands/virustotal"
 require "mihari/commands/zoomeye"
 
@@ -42,6 +43,7 @@ module Mihari
       include Mihari::Commands::Spyse
       include Mihari::Commands::Urlscan
       include Mihari::Commands::VirusTotal
+      include Mihari::Commands::VirusTotalIntelligence
       include Mihari::Commands::ZoomEye
     end
   end

data/lib/mihari/commands/passivetotal.rb

@@ -14,6 +14,7 @@ module Mihari
               run_analyzer Analyzers::PassiveTotal, query: indicator, options: options
             end
           end
+          map "pt" => :passivetotal
         end
       end
     end

data/lib/mihari/commands/virustotal.rb

@@ -14,6 +14,7 @@ module Mihari
               run_analyzer Analyzers::VirusTotal, query: indiactor, options: options
             end
           end
+          map "vt" => :virustotal
         end
       end
     end

data/lib/mihari/commands/virustotal_intelligence.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module VirusTotalIntelligence
+      def self.included(thor)
+        thor.class_eval do
+          desc "virustotal_intelligence [QUERY]", "VirusTotal Intelligence search"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          def virustotal_intelligence(query)
+            with_error_handling do
+              run_analyzer Analyzers::VirusTotalIntelligence, query: query, options: options
+            end
+          end
+          map "vt_intel" => :virustotal_intelligence
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/web.rb

@@ -6,19 +6,23 @@ module Mihari
       def self.included(thor)
         thor.class_eval do
           desc "web", "Launch the web app"
-          method_option :port, type: :numeric, default: 9292
-          method_option :host, type: :string, default: "localhost"
+          method_option :port, type: :numeric, default: 9292, desc: "Hostname to listen on"
+          method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
+          method_option :threads, type: :string, default: "0:16", desc: "min:max threads to use"
+          method_option :verbose, type: :boolean, default: true, desc: "Report each request"
           method_option :config, type: :string, desc: "Path to the config file"
           def web
-            port = options["port"].to_i || 9292
-            host = options["host"] || "localhost"
+            port = options["port"]
+            host = options["host"]
+            threads = options["threads"]
+            verbose = options["verbose"]
 
             load_configuration
 
             # set rack env as production
             ENV["RACK_ENV"] ||= "production"
 
-            Mihari::App.run!(port: port, host: host)
+            Mihari::App.run!(port: port, host: host, threads: threads, verbose: verbose)
           end
         end
       end

data/lib/mihari/database.rb

@@ -106,7 +106,7 @@ module Mihari
           )
         end
 
-        # ActiveRecord::Base.logger = Logger.new STDOUT
+        ActiveRecord::Base.logger = Logger.new($stdout) if ENV["RACK_ENV"] == "development"
         ActiveRecord::Migration.verbose = false
 
         InitialSchema.migrate(:up)

data/lib/mihari/models/alert.rb

@@ -13,36 +13,20 @@ module Mihari
       #
       # Search alerts
       #
-      # @param [String, nil] artifact_data
-      # @param [String, nil] description
-      # @param [String, nil] source
-      # @param [String, nil] tag_name
-      # @param [String, nil] title
-      # @param [DateTime, nil] from_at
-      # @param [DateTime, nil] to_at
-      # @param [Integer, nil] limit
-      # @param [Integer, nil] page
+      # @param [Structs::Alert::SearchFilterWithPagination] filter
       #
       # @return [Array<Hash>]
       #
-      def search(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil, limit: 10, page: 1)
-        limit = limit.to_i
+      def search(filter)
+        limit = filter.limit.to_i
         raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
 
-        page = page.to_i
+        page = filter.page.to_i
         raise ArgumentError, "page should be bigger than zero" unless page.positive?
 
         offset = (page - 1) * limit
 
-        relation = build_relation(
-          artifact_data: artifact_data,
-          title: title,
-          description: description,
-          source: source,
-          tag_name: tag_name,
-          from_at: from_at,
-          to_at: to_at
-        )
+        relation = build_relation(filter.without_pagination)
 
         # TODO: improve queires
         alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
@@ -60,45 +44,43 @@ module Mihari
       # Count alerts
       #
       # @param [String, nil] artifact_data
-      # @param [String, nil] description
-      # @param [String, nil] source
-      # @param [String, nil] tag_name
-      # @param [String, nil] title
-      # @param [DateTime, nil] from_at
-      # @param [DateTime, nil] to_at
       #
       # @return [Integer]
       #
-      def count(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil)
-        relation = build_relation(
-          artifact_data: artifact_data,
-          title: title,
-          description: description,
-          source: source,
-          tag_name: tag_name,
-          from_at: from_at,
-          to_at: to_at
-        )
+      def count(filter)
+        relation = build_relation(filter)
         relation.distinct("alerts.id").count
       end
 
       private
 
-      def build_relation(artifact_data: nil, title: nil, description: nil, source: nil, tag_name: nil, from_at: nil, to_at: nil)
-        relation = self
+      def build_relation(filter)
+        artifact_ids = []
+        artifact = Artifact.includes(:autonomous_system, :dns_records, :reverse_dns_names)
+        artifact = artifact.where(data: filter.artifact_data) if filter.artifact_data
+        artifact = artifact.where(autonomous_system: { asn: filter.asn }) if filter.asn
+        artifact = artifact.where(dns_records: { value: filter.dns_record }) if filter.dns_record
+        artifact = artifact.where(reverse_dns_names: { name: filter.reverse_dns_name }) if filter.reverse_dns_name
+        # get artifact ids if there is any valid filter for artifact
+        if filter.has_valid_artifact_filters
+          artifact_ids = artifact.pluck(:id)
+          # set invalid ID if nothing is matched with the filters
+          artifact_ids = [-1] if artifact_ids.empty?
+        end
 
+        relation = self
         relation = relation.includes(:artifacts, :tags)
 
-        relation = relation.where(artifacts: { data: artifact_data }) if artifact_data
-        relation = relation.where(tags: { name: tag_name }) if tag_name
+        relation = relation.where(artifacts: { id: artifact_ids }) unless artifact_ids.empty?
+        relation = relation.where(tags: { name: filter.tag_name }) if filter.tag_name
 
-        relation = relation.where(source: source) if source
-        relation = relation.where(title: title) if title
+        relation = relation.where(source: filter.source) if filter.source
+        relation = relation.where(title: filter.title) if filter.title
 
-        relation = relation.filter(description: { like: "%#{description}%" }) if description
+        relation = relation.filter(description: { like: "%#{filter.description}%" }) if filter.description
 
-        relation = relation.filter(created_at: { gte: from_at }) if from_at
-        relation = relation.filter(created_at: { lte: to_at }) if to_at
+        relation = relation.filter(created_at: { gte: filter.from_at }) if filter.from_at
+        relation = relation.filter(created_at: { lte: filter.to_at }) if filter.to_at
 
         relation
       end

data/lib/mihari/structs/alert.rb

@@ -0,0 +1,45 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module Alert
+      class SearchFilter < Dry::Struct
+        attribute? :artifact_data, Types::String.optional
+        attribute? :description, Types::String.optional
+        attribute? :source, Types::String.optional
+        attribute? :tag_name, Types::String.optional
+        attribute? :title, Types::String.optional
+        attribute? :from_at, Types::DateTime.optional
+        attribute? :to_at, Types::DateTime.optional
+        attribute? :asn, Types::Int.optional
+        attribute? :dns_record, Types::String.optional
+        attribute? :reverse_dns_name, Types::String.optional
+
+        def has_valid_artifact_filters
+          !(artifact_data || asn || dns_record || reverse_dns_name).nil?
+        end
+      end
+
+      class SearchFilterWithPagination < SearchFilter
+        attribute? :page, Types::Int.default(1)
+        attribute? :limit, Types::Int.default(10)
+
+        def without_pagination
+          SearchFilter.new(
+            artifact_data: artifact_data,
+            description: description,
+            from_at: from_at,
+            source: source,
+            tag_name: tag_name,
+            title: title,
+            to_at: to_at,
+            asn: asn,
+            dns_record: dns_record,
+            reverse_dns_name: reverse_dns_name
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/virustotal_intelligence.rb

@@ -0,0 +1,75 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module VirusTotalIntelligence
+      class ContextAttributes < Dry::Struct
+        attribute :url, Types.Array(Types::String).optional
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            url: d["url"]
+          )
+        end
+      end
+
+      class Datum < Dry::Struct
+        attribute :type, Types::String
+        attribute :id, Types::String
+        attribute :context_attributes, ContextAttributes.optional
+
+        def value
+          case type
+          when "file"
+            id
+          when "url"
+            (context_attributes.url || []).first
+          when "domain"
+            id
+          when "ip_address"
+            id
+          end
+        end
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+
+          context_attributes = nil
+          context_attributes = ContextAttributes.from_dynamic!(d.fetch("context_attributes")) if d.key?("context_attributes")
+
+          new(
+            type: d.fetch("type"),
+            id: d.fetch("id"),
+            context_attributes: context_attributes
+          )
+        end
+      end
+
+      class Meta < Dry::Struct
+        attribute :cursor, Types::String.optional
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            cursor: d["cursor"]
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :meta, Meta
+        attribute :data, Types.Array(Datum)
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            meta: Meta.from_dynamic!(d.fetch("meta")),
+            data: d.fetch("data").map { |x| Datum.from_dynamic!(x) }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/types.rb

@@ -9,13 +9,28 @@ module Mihari
     Hash = Strict::Hash
     String = Strict::String
     Double = Strict::Float | Strict::Integer
+    DateTime = Strict::DateTime
 
     DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
 
     AnalyzerTypes = Types::String.enum(
-      "binaryedge", "censys", "circl", "dnpedia", "dnstwister",
-      "onyphe", "otx", "passivetotal", "pulsedive", "securitytrails",
-      "shodan", "virustotal"
+      "binaryedge",
+      "censys",
+      "circl",
+      "dnpedia",
+      "dnstwister",
+      "onyphe",
+      "otx",
+      "passivetotal",
+      "pt",
+      "pulsedive",
+      "securitytrails",
+      "shodan",
+      "st",
+      "virustotal_intelligence",
+      "virustotal",
+      "vt_intel",
+      "vt"
     )
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.7.1"
+  VERSION = "3.9.0"
 end

data/lib/mihari/web/app.rb

@@ -37,10 +37,10 @@ module Mihari
     use Mihari::Controllers::TagsController
 
     class << self
-      def run!(port: 9292, host: "localhost")
+      def run!(port: 9292, host: "localhost", threads: "0:16", verbose: false)
         url = "http://#{host}:#{port}"
 
-        Rack::Handler::Puma.run self, Port: port, Host: host do |server|
+        Rack::Handler::Puma.run(self, Port: port, Host: host, Threads: threads, Verbose: verbose) do |server|
           Launchy.open url
 
           [:INT, :TERM].each do |sig|

data/lib/mihari/web/controllers/alerts_controller.rb

@@ -15,39 +15,32 @@ module Mihari
         param :to_at, DateTime
         param :toAt, DateTime
 
+        param :asn, Integer
+        param :dns_record, String
+        param :dnsRecord, String
+        param :reverse_dns_name, String
+        param :reverseDnsName, String
+
+        # set page & limit
         page = params["page"] || 1
-        page = page.to_i
+        params["page"] = page.to_i
+
         limit = 10
+        params["limit"] = 10
 
-        artifact_data = params["artifact"]
-        description = params["description"]
-        source = params["source"]
-        tag_name = params["tag"]
-        title = params["title"]
+        # normalize keys
+        params["artifact_data"] = params["artifact"]
+        params["from_at"] = params["from_at"] || params["fromAt"]
+        params["to_at"] = params["to_at"] || params["toAt"]
+        params["dns_record"] = params["dns_record"] || params["dnsRecord"]
+        params["reverse_dns_name"] = params["reverse_dns_name"] || params["reverseDnsName"]
 
-        from_at = params["from_at"] || params["fromAt"]
-        to_at = params["to_at"] || params["toAt"]
+        # symbolize hash keys
+        filter = params.to_h.transform_keys(&:to_sym)
 
-        alerts = Mihari::Alert.search(
-          artifact_data: artifact_data,
-          description: description,
-          from_at: from_at,
-          limit: limit,
-          page: page,
-          source: source,
-          tag_name: tag_name,
-          title: title,
-          to_at: to_at
-        )
-        total = Mihari::Alert.count(
-          artifact_data: artifact_data,
-          description: description,
-          from_at: from_at,
-          source: source,
-          tag_name: tag_name,
-          title: title,
-          to_at: to_at
-        )
+        search_filter_with_pagenation = Structs::Alert::SearchFilterWithPagination.new(**filter)
+        alerts = Mihari::Alert.search(search_filter_with_pagenation)
+        total = Mihari::Alert.count(search_filter_with_pagenation.without_pagination)
 
         json({ alerts: alerts, total: total, current_page: page, page_size: limit })
       end

data/lib/mihari/web/public/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/static/favicon.ico"><title>Mihari</title><link href="/static/js/app.06d5cf1c.js" rel="preload" as="script"></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/static/js/app.06d5cf1c.js"></script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/static/favicon.ico"><title>Mihari</title><link href="/static/js/app.378da3dc.js" rel="preload" as="script"></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/static/js/app.378da3dc.js"></script></body></html>