mihari 2.0.0 → 2.3.1

data/lib/mihari/commands/securitytrails_domain_feed.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module SecurityTrailsDomainFeed
+      def self.included(thor)
+        thor.class_eval do
+          desc "securitytrails_domain_feed [REGEXP]", "SecurityTrails new domain feed search by a regexp"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          method_option :type, type: :string, default: "registered", desc: "A type of domain feed ('all', 'new' or 'registered')"
+          def securitytrails_domain_feed(regexp)
+            with_error_handling do
+              run_analyzer Analyzers::SecurityTrailsDomainFeed, query: regexp, options: options
+            end
+          end
+          map "st_domain_feed" => :securitytrails_domain_feedd
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/shodan.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Shodan
+      def self.included(thor)
+        thor.class_eval do
+          desc "shodan [QUERY]", "Shodan host search by a query"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          def shodan(query)
+            with_error_handling do
+              run_analyzer Analyzers::Shodan, query: query, options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/spyse.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Spyse
+      def self.included(thor)
+        thor.class_eval do
+          desc "spyse [QUERY]", "Spyse search by a query"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          method_option :type, type: :string, desc: "type to search (ip or domain)", default: "doamin"
+          def spyse(query)
+            with_error_handling do
+              run_analyzer Analyzers::Spyse, query: query, options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/ssh_fingerprint.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module SSHFingerprint
+      def self.included(thor)
+        thor.class_eval do
+          desc "ssh_fingerprint [FINGERPRINT]", "Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          def ssh_fingerprint(fingerprint)
+            with_error_handling do
+              run_analyzer Analyzers::SSHFingerprint, query: fingerprint, options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/urlscan.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Urlscan
+      def self.included(thor)
+        thor.class_eval do
+          desc "urlscan [QUERY]", "urlscan search by a given query"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
+          method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
+          def urlscan(query)
+            with_error_handling do
+              run_analyzer Analyzers::Urlscan, query: query, options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/virustotal.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module VirusTotal
+      def self.included(thor)
+        thor.class_eval do
+          desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions lookup by an ip or domain"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          def virustotal(indiactor)
+            with_error_handling do
+              run_analyzer Analyzers::VirusTotal, query: refang(indiactor), options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/web.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Web
+      def self.included(thor)
+        thor.class_eval do
+          desc "web", "Launch the web app"
+          method_option :port, type: :numeric, default: 9292
+          method_option :host, type: :string, default: "localhost"
+          def web
+            port = options["port"].to_i || 9292
+            host = options["host"] || "localhost"
+
+            load_configuration
+            Mihari::App.run!(port: port, host: host)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/zoomeye.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module ZoomEye
+      def self.included(thor)
+        thor.class_eval do
+          desc "zoomeye [QUERY]", "ZoomEye search by a query"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          method_option :type, type: :string, desc: "type to search(host / web)", default: "host"
+          def zoomeye(query)
+            with_error_handling do
+              run_analyzer Analyzers::ZoomEye, query: query, options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/config.rb

@@ -4,7 +4,7 @@ require "yaml"
 
 module Mihari
   class Config
-    attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_password, :zoomeye_username, :database
+    attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_api_key, :database
 
     def initialize
       load_from_env
@@ -32,8 +32,7 @@ module Mihari
       @thehive_api_key = ENV["THEHIVE_API_KEY"]
       @urlscan_api_key = ENV["URLSCAN_API_KEY"]
       @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
-      @zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
-      @zoomeye_username = ENV["ZOOMEYE_USERNAME"]
+      @zoomeye_api_key = ENV["ZOOMEYE_API_KEY"]
 
       @database = ENV["DATABASE"] || "mihari.db"
     end
@@ -55,6 +54,18 @@ module Mihari
           end
         end
       end
+
+      def initialize_yaml(filename)
+        keys = new.instance_variables.map do |key|
+          key.to_s[1..]
+        end
+
+        config = keys.map do |key|
+          [key, nil]
+        end.to_h
+
+        YAML.dump(config, File.open(filename, "w"))
+      end
     end
   end
 

data/lib/mihari/configurable.rb

@@ -10,7 +10,7 @@ module Mihari
       return nil if config_keys.empty?
 
       config_keys.map do |key|
-        {key: key.upcase, value: Mihari.config.send(key)}
+        { key: key.upcase, value: Mihari.config.send(key) }
       end
     end
 

data/lib/mihari/emitters/slack.rb

@@ -25,25 +25,25 @@ module Mihari
       def vt_link
         return nil unless _vt_link
 
-        {type: "button", text: "VirusTotal", url: _vt_link}
+        { type: "button", text: "VirusTotal", url: _vt_link }
       end
 
       def urlscan_link
         return nil unless _urlscan_link
 
-        {type: "button", text: "urlscan.io", url: _urlscan_link}
+        { type: "button", text: "urlscan.io", url: _urlscan_link }
       end
 
       def censys_link
         return nil unless _censys_link
 
-        {type: "button", text: "Censys", url: _censys_link}
+        { type: "button", text: "Censys", url: _censys_link }
       end
 
       def shodan_link
         return nil unless _shodan_link
 
-        {type: "button", text: "Shodan", url: _shodan_link}
+        { type: "button", text: "Shodan", url: _shodan_link }
       end
 
       # @return [Array]

data/lib/mihari/emitters/the_hive.rb

@@ -17,7 +17,7 @@ module Mihari
         api.alert.create(
           title: title,
           description: description,
-          artifacts: artifacts.map { |artifact| {data: artifact.data, data_type: artifact.data_type, message: description} },
+          artifacts: artifacts.map { |artifact| { data: artifact.data, data_type: artifact.data_type, message: description } },
           tags: tags,
           type: "external",
           source: "mihari"

data/lib/mihari/models/alert.rb

@@ -44,16 +44,16 @@ module Mihari
         relation = joins(:tags) if tag_name
         relation = joins(:artifacts) if artifact_data
 
-        relation = relation.where(artifacts: {data: artifact_data}) if artifact_data
-        relation = relation.where(tags: {name: tag_name}) if tag_name
+        relation = relation.where(artifacts: { data: artifact_data }) if artifact_data
+        relation = relation.where(tags: { name: tag_name }) if tag_name
 
         relation = relation.where(source: source) if source
         relation = relation.where(title: title) if title
 
-        relation = relation.filter(description: {like: "%#{description}%"}) if description
+        relation = relation.filter(description: { like: "%#{description}%" }) if description
 
-        relation = relation.filter(created_at: {gte: from_at}) if from_at
-        relation = relation.filter(created_at: {lte: to_at}) if to_at
+        relation = relation.filter(created_at: { gte: from_at }) if from_at
+        relation = relation.filter(created_at: { lte: to_at }) if to_at
 
         relation
       end

data/lib/mihari/models/artifact.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require "active_record"
+require "active_record/filter"
+require "active_support/core_ext/integer/time"
+require "active_support/core_ext/numeric/time"
 
 class ArtifactValidator < ActiveModel::Validator
   def validate(record)
@@ -20,8 +23,16 @@ module Mihari
       self.data_type = TypeChecker.type(data)
     end
 
-    def unique?
-      self.class.find_by(data: data).nil?
+    def unique?(ignore_old_artifacts: false, ignore_threshold: 0)
+      artifact = self.class.where(data: data).order(created_at: :desc).first
+      return true if artifact.nil?
+
+      return false unless ignore_old_artifacts
+
+      days_before = (-ignore_threshold).days.from_now
+      # if an artifact is created before {ignore_threshold} days, ignore it
+      #                           within {ignore_threshold} days, do not ignore it
+      artifact.created_at < days_before
     end
   end
 end

data/lib/mihari/notifiers/exception_notifier.rb

@@ -51,20 +51,20 @@ module Mihari
 
       def to_fields(clean_message, backtrace)
         fields = [
-          {title: "Exception", value: clean_message},
-          {title: "Hostname", value: hostname}
+          { title: "Exception", value: clean_message },
+          { title: "Hostname", value: hostname }
         ]
 
         if backtrace
           formatted_backtrace = format_backtrace(backtrace)
-          fields << {title: "Backtrace", value: formatted_backtrace}
+          fields << { title: "Backtrace", value: formatted_backtrace }
         end
         fields
       end
 
       def hostname
         Socket.gethostname
-      rescue => _e
+      rescue StandardError => _e
         "N/A"
       end
 

data/lib/mihari/status.rb

@@ -3,7 +3,7 @@
 module Mihari
   class Status
     def check
-      statuses.transform_values { |value| convert(**value) }
+      statuses
     end
 
     def self.check
@@ -12,14 +12,6 @@ module Mihari
 
     private
 
-    def convert(is_configured:, values:, type:)
-      {
-        is_configured: is_configured,
-        values: values,
-        type: type
-      }
-    end
-
     def statuses
       (Mihari.analyzers + Mihari.emitters).map do |klass|
         name = klass.to_s.split("::").last.to_s
@@ -36,7 +28,7 @@ module Mihari
       values = instance.configuration_values
       type = is_analyzer ? "Analyzer" : "Emitter"
 
-      values ? {is_configured: is_configured, values: values, type: type} : nil
+      values ? { is_configured: is_configured, values: values, type: type } : nil
     rescue ArgumentError => _e
       nil
     end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "2.0.0"
+  VERSION = "2.3.1"
 end

data/lib/mihari/web/app.rb

@@ -1,119 +1,44 @@
-require "awrence"
+# frozen_string_literal: true
+
 require "launchy"
 require "rack"
+require "rack/handler/puma"
 require "sinatra"
-require "sinatra/json"
-require "sinatra/reloader"
+
+require "mihari/web/helpers/json"
+
+require "mihari/web/controllers/base_controller"
+
+require "mihari/web/controllers/alerts_controller"
+require "mihari/web/controllers/artifacts_controller"
+require "mihari/web/controllers/command_controller"
+require "mihari/web/controllers/config_controller"
+require "mihari/web/controllers/sources_controller"
+require "mihari/web/controllers/tags_controller"
 
 module Mihari
   class App < Sinatra::Base
-    register Sinatra::Reloader
-
     set :root, File.dirname(__FILE__)
-    set :public_folder, proc { File.join(root, "public") }
+    set :public_folder, File.join(root, "public")
 
     get "/" do
       send_file File.join(settings.public_folder, "index.html")
     end
 
-    get "/api/alerts" do
-      page = params["page"] || 1
-      page = page.to_i
-      limit = 10
-
-      artifact_data = params["artifact"]
-      description = params["description"]
-      source = params["source"]
-      tag_name = params["tag"]
-      title = params["title"]
-
-      from_at = params["from_at"] || params["fromAt"]
-      from_at = DateTime.parse(from_at) if from_at
-      to_at = params["to_at"] || params["toAt"]
-      to_at = DateTime.parse(to_at) if to_at
-
-      alerts = Mihari::Alert.search(
-        artifact_data: artifact_data,
-        description: description,
-        from_at: from_at,
-        limit: limit,
-        page: page,
-        source: source,
-        tag_name: tag_name,
-        title: title,
-        to_at: to_at
-      )
-      total = Mihari::Alert.count(
-        artifact_data: artifact_data,
-        description: description,
-        from_at: from_at,
-        source: source,
-        tag_name: tag_name,
-        title: title,
-        to_at: to_at
-      )
-
-      json = {alerts: alerts, total: total, current_page: page, page_size: limit}
-      json json.to_camelback_keys
-    end
-
-    delete "/api/alerts/:id" do
-      id = params["id"]
-      id = id.to_i
-
-      begin
-        alert = Mihari::Alert.find(id)
-        alert.destroy
-
-        status 204
-        body ""
-      rescue ActiveRecord::RecordNotFound
-        status 404
-
-        message = {message: "ID:#{id} is not found"}
-        json message
-      end
-    end
-
-    delete "/api/artifacts/:id" do
-      id = params["id"]
-      id = id.to_i
-
-      begin
-        alert = Mihari::Artifact.find(id)
-        alert.delete
-
-        status 204
-        body ""
-      rescue ActiveRecord::RecordNotFound
-        status 404
-
-        message = {message: "ID:#{id} is not found"}
-        json message
-      end
-    end
-
-    get "/api/sources" do
-      tags = Mihari::Alert.distinct.pluck(:source)
-      json tags
-    end
-
-    get "/api/tags" do
-      tags = Mihari::Tag.distinct.pluck(:name)
-      json tags
-    end
-
-    get "/api/config" do
-      report = Status.check
-
-      json report.to_camelback_keys
-    end
+    use Mihari::Controllers::AlertsController
+    use Mihari::Controllers::ArtifactsController
+    use Mihari::Controllers::CommandController
+    use Mihari::Controllers::ConfigController
+    use Mihari::Controllers::SourcesController
+    use Mihari::Controllers::TagsController
 
     class << self
       def run!(port: 9292, host: "localhost")
-        Launchy.open "http://#{host}:#{port}"
+        url = "http://#{host}:#{port}"
+
+        Rack::Handler::Puma.run self, Port: port, Host: host do |server|
+          Launchy.open url
 
-        Rack::Handler::WEBrick.run self, Port: port, Host: host do |server|
           [:INT, :TERM].each do |sig|
             trap(sig) do
               server.shutdown