mihari 3.8.0 → 3.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aac91d43689cb53dc0570bfed3cec57a07cbe88de0716530f2ea8bfac8f8d39d
-  data.tar.gz: 0f59bdc53cfa75e56884dd3497fa0492d3a41a3b7540cbdab1345ec5b301c69c
+  metadata.gz: c12a4ec4b0c1eee79deba3e5f3a511edcea4a13d7d72dc8142dbd85821095f55
+  data.tar.gz: 4a1e388e55efff4b715caf2261990e6122afd4eb26aca0874a33d3025a16ba5b
 SHA512:
-  metadata.gz: 30aef30fb14c7c1a50e75162141d1266b1ec5b847f6329935a221a90f59d37a2bed97c6a8aa4371962ab85b18c5ff23cf0417f08f9dc3320c737721ac1a07602
-  data.tar.gz: 1029878ec85cbdbe0a2c800b6b68cce99d45510818dde6b1d84a826022379cbe21fc57f490e3a77521136dbc22e8112b804d891892ea0a351e0d1db935b28b4b
+  metadata.gz: 8ead20235892bf77e222eff200f25539ffdd68cd21d14c5000d9196e518b9d2da2b461a2fe5d3dae4bc4bd899fb210f5f75b6330afad61826bc84dcfe40af23f
+  data.tar.gz: 4b5679794091a79adb426ef34c5396819c702d67ad84791dec1430eba0f007d1dcab5f7defc53b5dab06e8a0e174963c777fb05606f79a5f16c4377f07847f75

data/.github/workflows/test.yml

@@ -43,17 +43,16 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Ruby 2.7
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
 
       - name: Install dependencies
         run: |
           sudo apt-get -yqq install libpq-dev libmysqlclient-dev
-          gem install bundler
-          bundle install
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
 
       - name: Test with PostgreSQL
         env:

data/config.ru

@@ -4,4 +4,4 @@ require "./lib/mihari"
 # set rack env as development
 ENV["RACK_ENV"] ||= "development"
 
-run Mihari::App
+run Mihari::App.instance

data/lib/mihari/analyzers/rule.rb

@@ -4,6 +4,30 @@ require "uuidtools"
 
 module Mihari
   module Analyzers
+    ANALYZER_TO_CLASS = {
+      "binaryedge" => BinaryEdge,
+      "censys" => Censys,
+      "circl" => CIRCL,
+      "crtsh" => Crtsh,
+      "dnpedia" => DNPedia,
+      "dnstwister" => DNSTwister,
+      "onyphe" => Onyphe,
+      "otx" => OTX,
+      "passivetotal" => PassiveTotal,
+      "pt" => PassiveTotal,
+      "pulsedive" => Pulsedive,
+      "securitytrails" => SecurityTrails,
+      "shodan" => Shodan,
+      "spyse" => Spyse,
+      "st" => SecurityTrails,
+      "urlscan" => Urlscan,
+      "virustotal_intelligence" => VirusTotalIntelligence,
+      "virustotal" => VirusTotal,
+      "vt_intel" => VirusTotalIntelligence,
+      "vt" => VirusTotal,
+      "zoomeye" => ZoomEye
+    }.freeze
+
     class Rule < Base
       include Mihari::Mixins::DisallowedDataValue
 
@@ -26,26 +50,6 @@ module Mihari
         validate_analyzer_configurations
       end
 
-      ANALYZER_TO_CLASS = {
-        "binaryedge" => BinaryEdge,
-        "censys" => Censys,
-        "circl" => CIRCL,
-        "crtsh" => Crtsh,
-        "dnpedia" => DNPedia,
-        "dnstwister" => DNSTwister,
-        "onyphe" => Onyphe,
-        "otx" => OTX,
-        "passivetotal" => PassiveTotal,
-        "pulsedive" => Pulsedive,
-        "securitytrails" => SecurityTrails,
-        "shodan" => Shodan,
-        "spyse" => Spyse,
-        "urlscan" => Urlscan,
-        "virustotal" => VirusTotal,
-        "virustotal_intelligence" => VirusTotalIntelligence,
-        "zoomeye" => ZoomEye
-      }.freeze
-
       #
       # Returns a list of artifacts matched with queries
       #

data/lib/mihari/commands/search.rb

@@ -10,6 +10,9 @@ module Mihari
           desc "search [RULE]", "Search by a rule"
           method_option :config, type: :string, desc: "Path to the config file"
           def search_by_rule(rule)
+            # load configuration
+            load_configuration
+
             # convert str(YAML) to hash or str(path/YAML file) to hash
             rule = load_rule(rule)
 
@@ -77,8 +80,6 @@ module Mihari
       # @return [nil]
       #
       def run_rule_analyzer(analyzer, ignore_old_artifacts: false, ignore_threshold: 0)
-        load_configuration
-
         analyzer.ignore_old_artifacts = ignore_old_artifacts
         analyzer.ignore_threshold = ignore_threshold
 

data/lib/mihari/commands/web.rb

@@ -6,19 +6,23 @@ module Mihari
       def self.included(thor)
         thor.class_eval do
           desc "web", "Launch the web app"
-          method_option :port, type: :numeric, default: 9292
-          method_option :host, type: :string, default: "localhost"
+          method_option :port, type: :numeric, default: 9292, desc: "Hostname to listen on"
+          method_option :host, type: :string, default: "localhost", desc: "Port to listen on"
+          method_option :threads, type: :string, default: "0:16", desc: "min:max threads to use"
+          method_option :verbose, type: :boolean, default: true, desc: "Report each request"
           method_option :config, type: :string, desc: "Path to the config file"
           def web
-            port = options["port"].to_i || 9292
-            host = options["host"] || "localhost"
+            port = options["port"]
+            host = options["host"]
+            threads = options["threads"]
+            verbose = options["verbose"]
 
             load_configuration
 
             # set rack env as production
             ENV["RACK_ENV"] ||= "production"
 
-            Mihari::App.run!(port: port, host: host)
+            Mihari::App.run!(port: port, host: host, threads: threads, verbose: verbose)
           end
         end
       end

data/lib/mihari/database.rb

@@ -106,7 +106,7 @@ module Mihari
           )
         end
 
-        # ActiveRecord::Base.logger = Logger.new STDOUT
+        ActiveRecord::Base.logger = Logger.new($stdout) if ENV["RACK_ENV"] == "development"
         ActiveRecord::Migration.verbose = false
 
         InitialSchema.migrate(:up)

data/lib/mihari/errors.rb

@@ -6,4 +6,6 @@ module Mihari
   class InvalidInputError < Error; end
 
   class RetryableError < Error; end
+
+  class FileNotFoundError < Error; end
 end

data/lib/mihari/mixins/configuration.rb

@@ -80,10 +80,20 @@ module Mihari
         end
       end
 
+      #
+      # Load configuration file
+      #
+      # @param [String] path
+      #
+      # @return [Hash]
+      #
       def _load_config(path)
-        return YAML.safe_load(File.read(path), symbolize_names: true) if Pathname(path).exist?
+        unless Pathname(path).exist?
+          puts "#{path} does not exist".colorize(:red)
+          raise FileNotFoundError
+        end
 
-        YAML.safe_load(path, symbolize_names: true)
+        YAML.safe_load(File.read(path), symbolize_names: true)
       end
     end
   end

data/lib/mihari/models/alert.rb

@@ -13,92 +13,67 @@ module Mihari
       #
       # Search alerts
       #
-      # @param [String, nil] artifact_data
-      # @param [String, nil] description
-      # @param [String, nil] source
-      # @param [String, nil] tag_name
-      # @param [String, nil] title
-      # @param [DateTime, nil] from_at
-      # @param [DateTime, nil] to_at
-      # @param [Integer, nil] limit
-      # @param [Integer, nil] page
+      # @param [Structs::Alert::SearchFilterWithPagination] filter
       #
       # @return [Array<Hash>]
       #
-      def search(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil, limit: 10, page: 1)
-        limit = limit.to_i
+      def search(filter)
+        limit = filter.limit.to_i
         raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
 
-        page = page.to_i
+        page = filter.page.to_i
         raise ArgumentError, "page should be bigger than zero" unless page.positive?
 
         offset = (page - 1) * limit
 
-        relation = build_relation(
-          artifact_data: artifact_data,
-          title: title,
-          description: description,
-          source: source,
-          tag_name: tag_name,
-          from_at: from_at,
-          to_at: to_at
-        )
+        relation = build_relation(filter.without_pagination)
 
         # TODO: improve queires
         alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
-        alerts = includes(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
-
-        alerts.map do |alert|
-          json = Serializers::AlertSerializer.new(alert).as_json
-          json[:artifacts] = json[:artifacts] || []
-          json[:tags] = json[:tags] || []
-          json
-        end
+        includes(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
       end
 
       #
       # Count alerts
       #
       # @param [String, nil] artifact_data
-      # @param [String, nil] description
-      # @param [String, nil] source
-      # @param [String, nil] tag_name
-      # @param [String, nil] title
-      # @param [DateTime, nil] from_at
-      # @param [DateTime, nil] to_at
       #
       # @return [Integer]
       #
-      def count(artifact_data: nil, description: nil, source: nil, tag_name: nil, title: nil, from_at: nil, to_at: nil)
-        relation = build_relation(
-          artifact_data: artifact_data,
-          title: title,
-          description: description,
-          source: source,
-          tag_name: tag_name,
-          from_at: from_at,
-          to_at: to_at
-        )
+      def count(filter)
+        relation = build_relation(filter)
         relation.distinct("alerts.id").count
       end
 
       private
 
-      def build_relation(artifact_data: nil, title: nil, description: nil, source: nil, tag_name: nil, from_at: nil, to_at: nil)
-        relation = self
+      def build_relation(filter)
+        artifact_ids = []
+        artifact = Artifact.includes(:autonomous_system, :dns_records, :reverse_dns_names)
+        artifact = artifact.where(data: filter.artifact_data) if filter.artifact_data
+        artifact = artifact.where(autonomous_system: { asn: filter.asn }) if filter.asn
+        artifact = artifact.where(dns_records: { value: filter.dns_record }) if filter.dns_record
+        artifact = artifact.where(reverse_dns_names: { name: filter.reverse_dns_name }) if filter.reverse_dns_name
+        # get artifact ids if there is any valid filter for artifact
+        if filter.has_valid_artifact_filters
+          artifact_ids = artifact.pluck(:id)
+          # set invalid ID if nothing is matched with the filters
+          artifact_ids = [-1] if artifact_ids.empty?
+        end
 
+        relation = self
         relation = relation.includes(:artifacts, :tags)
 
-        relation = relation.where(artifacts: { data: artifact_data }) if artifact_data
-        relation = relation.where(tags: { name: tag_name }) if tag_name
+        relation = relation.where(artifacts: { id: artifact_ids }) unless artifact_ids.empty?
+        relation = relation.where(tags: { name: filter.tag_name }) if filter.tag_name
 
-        relation = relation.where(source: source) if source
-        relation = relation.where(title: title) if title
+        relation = relation.where(source: filter.source) if filter.source
+        relation = relation.where(title: filter.title) if filter.title
 
-        relation = relation.filter(description: { like: "%#{description}%" }) if description
+        relation = relation.filter(description: { like: "%#{filter.description}%" }) if filter.description
 
-        relation = relation.filter(created_at: { gte: from_at }) if from_at
-        relation = relation.filter(created_at: { lte: to_at }) if to_at
+        relation = relation.filter(created_at: { gte: filter.from_at }) if filter.from_at
+        relation = relation.filter(created_at: { lte: filter.to_at }) if filter.to_at
 
         relation
       end

data/lib/mihari/models/artifact.rb

@@ -29,10 +29,13 @@ module Mihari
 
     validates_with ArtifactValidator
 
+    attr_accessor :tags
+
     def initialize(attributes)
       super
 
       self.data_type = TypeChecker.type(data)
+      self.tags = []
     end
 
     #

data/lib/mihari/structs/alert.rb

@@ -0,0 +1,45 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module Alert
+      class SearchFilter < Dry::Struct
+        attribute? :artifact_data, Types::String.optional
+        attribute? :description, Types::String.optional
+        attribute? :source, Types::String.optional
+        attribute? :tag_name, Types::String.optional
+        attribute? :title, Types::String.optional
+        attribute? :from_at, Types::DateTime.optional
+        attribute? :to_at, Types::DateTime.optional
+        attribute? :asn, Types::Int.optional
+        attribute? :dns_record, Types::String.optional
+        attribute? :reverse_dns_name, Types::String.optional
+
+        def has_valid_artifact_filters
+          !(artifact_data || asn || dns_record || reverse_dns_name).nil?
+        end
+      end
+
+      class SearchFilterWithPagination < SearchFilter
+        attribute? :page, Types::Int.default(1)
+        attribute? :limit, Types::Int.default(10)
+
+        def without_pagination
+          SearchFilter.new(
+            artifact_data: artifact_data,
+            description: description,
+            from_at: from_at,
+            source: source,
+            tag_name: tag_name,
+            title: title,
+            to_at: to_at,
+            asn: asn,
+            dns_record: dns_record,
+            reverse_dns_name: reverse_dns_name
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/ipinfo.rb

@@ -18,10 +18,9 @@ module Mihari
             d = Types::Hash[d]
 
             asn = nil
-            org = d["org"]
-            unless org.nil?
-              asn = org.split.first
-              asn = normalize_asn(asn)
+            asn_ = d.dig("asn", "asn")
+            unless asn_.nil?
+              asn = normalize_asn(asn_)
             end
 
             new(

data/lib/mihari/types.rb

@@ -9,6 +9,7 @@ module Mihari
     Hash = Strict::Hash
     String = Strict::String
     Double = Strict::Float | Strict::Integer
+    DateTime = Strict::DateTime
 
     DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
 
@@ -21,11 +22,15 @@ module Mihari
       "onyphe",
       "otx",
       "passivetotal",
+      "pt",
       "pulsedive",
       "securitytrails",
       "shodan",
+      "st",
       "virustotal_intelligence",
-      "virustotal"
+      "virustotal",
+      "vt_intel",
+      "vt"
     )
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.8.0"
+  VERSION = "3.9.2"
 end

data/lib/mihari/web/api.rb

@@ -0,0 +1,43 @@
+# Entities
+require "mihari/web/entities/message"
+
+require "mihari/web/entities/autonomous_system"
+require "mihari/web/entities/command"
+require "mihari/web/entities/config"
+require "mihari/web/entities/dns"
+require "mihari/web/entities/geolocation"
+require "mihari/web/entities/ip_address"
+require "mihari/web/entities/reverse_dns"
+require "mihari/web/entities/source"
+require "mihari/web/entities/tag"
+require "mihari/web/entities/whois"
+
+require "mihari/web/entities/artifact"
+
+require "mihari/web/entities/alert"
+
+# Endpoints
+require "mihari/web/endpoints/alerts"
+require "mihari/web/endpoints/artifacts"
+require "mihari/web/endpoints/command"
+require "mihari/web/endpoints/configs"
+require "mihari/web/endpoints/ip_addresses"
+require "mihari/web/endpoints/sources"
+require "mihari/web/endpoints/tags"
+
+module Mihari
+  class API < Grape::API
+    prefix "api"
+    format :json
+
+    mount Endpoints::Alerts
+    mount Endpoints::Artifacts
+    mount Endpoints::Command
+    mount Endpoints::Configs
+    mount Endpoints::IPAddresses
+    mount Endpoints::Sources
+    mount Endpoints::Tags
+
+    add_swagger_documentation(api_version: "v1", info: { title: "Mihari API" })
+  end
+end

data/lib/mihari/web/app.rb

@@ -2,46 +2,47 @@
 
 require "launchy"
 require "rack"
+require "rack/contrib"
 require "rack/handler/puma"
-require "sinatra"
+require "rack/cors"
 
-require "mihari/web/helpers/json"
+require "grape"
+require "grape-entity"
+require "grape-swagger"
+require "grape-swagger-entity"
 
-require "mihari/web/controllers/base_controller"
-
-require "mihari/web/controllers/alerts_controller"
-require "mihari/web/controllers/analyzers_controller"
-require "mihari/web/controllers/artifacts_controller"
-require "mihari/web/controllers/command_controller"
-require "mihari/web/controllers/config_controller"
-require "mihari/web/controllers/ip_address_controller"
-require "mihari/web/controllers/sources_controller"
-require "mihari/web/controllers/tags_controller"
+require "mihari/web/api"
 
 module Mihari
-  class App < Sinatra::Base
-    set :root, File.dirname(__FILE__)
-    set :public_folder, File.join(root, "public")
-
-    get "/" do
-      send_file File.join(settings.public_folder, "index.html")
+  class App
+    def initialize
+      @filenames = ["", ".html", "index.html", "/index.html"]
+      @rack_static = ::Rack::Static.new(
+        -> { [404, {}, []] },
+        root: File.expand_path("./public", __dir__),
+        urls: ["/"]
+      )
     end
 
-    use Mihari::Controllers::AlertsController
-    use Mihari::Controllers::AnalyzersController
-    use Mihari::Controllers::ArtifactsController
-    use Mihari::Controllers::CommandController
-    use Mihari::Controllers::ConfigController
-    use Mihari::Controllers::IPAddressController
-    use Mihari::Controllers::SourcesController
-    use Mihari::Controllers::TagsController
-
     class << self
-      def run!(port: 9292, host: "localhost")
+      def instance
+        @instance ||= Rack::Builder.new do
+          use Rack::Cors do
+            allow do
+              origins "*"
+              resource "*", headers: :any, methods: [:get, :post, :put, :delete, :options]
+            end
+          end
+
+          run App.new
+        end.to_app
+      end
+
+      def run!(port: 9292, host: "localhost", threads: "0:16", verbose: false)
         url = "http://#{host}:#{port}"
 
-        Rack::Handler::Puma.run self, Port: port, Host: host do |server|
-          Launchy.open url
+        Rack::Handler::Puma.run(instance, Port: port, Host: host, Threads: threads, Verbose: verbose) do |server|
+          Launchy.open(url) if ENV["RACK_ENV"] != "development"
 
           [:INT, :TERM].each do |sig|
             trap(sig) do
@@ -51,5 +52,22 @@ module Mihari
         end
       end
     end
+
+    def call(env)
+      # api
+      api_response = API.call(env)
+
+      # Check if the App wants us to pass the response along to others
+      if api_response[1]["X-Cascade"] == "pass"
+        # static files
+        request_path = env["PATH_INFO"]
+        @filenames.each do |path|
+          response = @rack_static.call(env.merge("PATH_INFO" => request_path + path))
+          return response if response[0] != 404
+        end
+      end
+
+      api_response
+    end
   end
 end

data/lib/mihari/web/endpoints/alerts.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Endpoints
+    class Alerts < Grape::API
+      namespace :alerts do
+        desc "Search alerts", {
+          is_array: true,
+          success: Entities::Alert,
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+        }
+        params do
+          optional :page, type: Integer
+          optional :artifact, type: String
+          optional :description, type: String
+          optional :source, type: String
+          optional :tag, type: String
+
+          optional :fromAt, type: DateTime
+          optional :toAt, type: DateTime
+
+          optional :asn, type: Integer
+          optional :dnsRecord, type: String
+          optional :reverseDnsName, type: String
+        end
+        get "/" do
+          filter = params.to_h.to_snake_keys
+
+          # set page & limit
+          page = filter["page"] || 1
+          filter["page"] = page.to_i
+
+          limit = 10
+          filter["limit"] = 10
+
+          # normalize keys
+          filter["artifact_data"] = filter["artifact"]
+          filter["tag_name"] = filter["tag"]
+
+          # symbolize hash keys
+          filter = filter.to_h.transform_keys(&:to_sym)
+
+          search_filter_with_pagenation = Structs::Alert::SearchFilterWithPagination.new(**filter)
+          alerts = Mihari::Alert.search(search_filter_with_pagenation)
+          total = Mihari::Alert.count(search_filter_with_pagenation.without_pagination)
+
+          present({ alerts: alerts, total: total, current_page: page, page_size: limit }, with: Entities::AlertsWithPagination)
+        end
+
+        desc "Delete an alert", {
+          success: Entities::Message,
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+        }
+        params do
+          requires :id, type: Integer
+        end
+        delete "/:id" do
+          id = params["id"].to_i
+
+          begin
+            alert = Mihari::Alert.find(id)
+          rescue ActiveRecord::RecordNotFound
+            error!({ message: "ID:#{id} is not found" }, 404)
+          end
+
+          alert.destroy
+
+          status 204
+          present({ message: "" }, with: Entities::Message)
+        end
+      end
+    end
+  end
+end