mihari 3.9.0 → 3.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47f7a6231abe8eee1a1a4e4bc6f905d5f08fba2b68b1435d1621ab85e9e583c0
-  data.tar.gz: 591a8c71130095d6ac963fb333179d087f00727f45ab545e4a674a3c42a2af46
+  metadata.gz: 0fcea779559f641125a388f63f0aa2d39979842f9b66f116194c36fe6410c56c
+  data.tar.gz: ceefe0fa86e5d3c8066bab202f7dc1dfb2b21017554a2b63c5a18f567e45aadb
 SHA512:
-  metadata.gz: da1c99c406086a9d8d029f8421a3316a5a6976c5b8fbc839a910bd0d61f04a3297614582974dbda925722308d206db9d2889fe44ae213df2650f1482cc15e918
-  data.tar.gz: edf130cb5027bbb6abfea72b150266b8768494b627162da814c93b38d54fe2597a870d78cd9f8f33238512e5a3205cb84a67b9da0589b71393b67fef062ad9b1
+  metadata.gz: 834c4c815ddf9c14b9047c9a8d9ed8ce56e3f115c16b797c4ebf40205529f7c94fe7ddda8f9c7f5cc0675c65123ac0957d28a0bfe51108d2492efc230f509dd8
+  data.tar.gz: 03ed05a672e17bfce72706de94aa2a339053f9b6545907c1e71cf60987be047e1af6bc6d4e95a16f6b6bb371de2ff3b65602ae4c3c9eb2eab3f70c2ace0735cf

data/.github/workflows/test.yml

@@ -43,17 +43,16 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Ruby 2.7
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
 
       - name: Install dependencies
         run: |
           sudo apt-get -yqq install libpq-dev libmysqlclient-dev
-          gem install bundler
-          bundle install
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
 
       - name: Test with PostgreSQL
         env:

data/README.md

@@ -38,6 +38,7 @@ Mihari supports the following services by default.
 - [crt.sh](https://crt.sh/)
 - [DN Pedia](https://dnpedia.com/)
 - [dnstwister](https://dnstwister.report/)
+- [GreyNoise](https://www.greynoise.io/)
 - [Onyphe](https://onyphe.io)
 - [OTX](https://otx.alienvault.com/)
 - [PassiveTotal](https://community.riskiq.com/)

data/config.ru

@@ -4,4 +4,4 @@ require "./lib/mihari"
 # set rack env as development
 ENV["RACK_ENV"] ||= "development"
 
-run Mihari::App
+run Mihari::App.instance

data/lib/mihari/analyzers/greynoise.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "greynoise"
+
+module Mihari
+  module Analyzers
+    class GreyNoise < Base
+      param :query
+      option :title, default: proc { "GreyNoise search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
+
+      def artifacts
+        res = Structs::GreyNoise::Response.from_dynamic!(search)
+        res.data.map do |datum|
+          build_artifact datum
+        end
+      end
+
+      private
+
+      PAGE_SIZE = 10_000
+
+      def configuration_keys
+        %w[greynoise_api_key]
+      end
+
+      def api
+        @api ||= ::GreyNoise::API.new(key: Mihari.config.greynoise_api_key)
+      end
+
+      #
+      # Search
+      #
+      # @return [Hash]
+      #
+      def search
+        api.experimental.gnql(query, size: PAGE_SIZE)
+      end
+
+      #
+      # Build an artifact from a GreyNoise search API response
+      #
+      # @param [Structs::GreyNoise::Datum] datum
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(datum)
+        as = AutonomousSystem.new(asn: normalize_asn(datum.metadata.asn))
+
+        geolocation = Geolocation.new(
+          country: datum.metadata.country,
+          country_code: datum.metadata.country_code
+        )
+
+        Artifact.new(
+          data: datum.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/rule.rb

@@ -11,6 +11,7 @@ module Mihari
       "crtsh" => Crtsh,
       "dnpedia" => DNPedia,
       "dnstwister" => DNSTwister,
+      "greynoise" => GreyNoise,
       "onyphe" => Onyphe,
       "otx" => OTX,
       "passivetotal" => PassiveTotal,

data/lib/mihari/analyzers/shodan.rb

@@ -58,6 +58,7 @@ module Mihari
         responses = []
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
+
           break unless res
 
           responses << res
@@ -78,11 +79,16 @@ module Mihari
       # @return [Artifact]
       #
       def build_artifact(match)
-        as = AutonomousSystem.new(asn: normalize_asn(match.asn))
-        geolocation = Geolocation.new(
-          country: match.location.country_name,
-          country_code: match.location.country_code
-        )
+        as = nil
+        as = AutonomousSystem.new(asn: normalize_asn(match.asn)) unless match.asn.nil?
+
+        geolocation = nil
+        if !match.location.country_name.nil? && !match.location.country_code.nil?
+          geolocation = Geolocation.new(
+            country: match.location.country_name,
+            country_code: match.location.country_code
+          )
+        end
 
         Artifact.new(
           data: match.ip_str,

data/lib/mihari/cli/analyzer.rb

@@ -6,6 +6,7 @@ require "mihari/commands/circl"
 require "mihari/commands/crtsh"
 require "mihari/commands/dnpedia"
 require "mihari/commands/dnstwister"
+require "mihari/commands/greynoise"
 require "mihari/commands/onyphe"
 require "mihari/commands/otx"
 require "mihari/commands/passivetotal"
@@ -33,6 +34,7 @@ module Mihari
       include Mihari::Commands::Crtsh
       include Mihari::Commands::DNPedia
       include Mihari::Commands::DNSTwister
+      include Mihari::Commands::GreyNoise
       include Mihari::Commands::JSON
       include Mihari::Commands::Onyphe
       include Mihari::Commands::OTX

data/lib/mihari/commands/greynoise.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module GreyNoise
+      def self.included(thor)
+        thor.class_eval do
+          desc "greynoise [QUERY]", "GreyNoise search"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          def greynoise(query)
+            with_error_handling do
+              run_analyzer Analyzers::GreyNoise, query: query, options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/search.rb

@@ -10,6 +10,9 @@ module Mihari
           desc "search [RULE]", "Search by a rule"
           method_option :config, type: :string, desc: "Path to the config file"
           def search_by_rule(rule)
+            # load configuration
+            load_configuration
+
             # convert str(YAML) to hash or str(path/YAML file) to hash
             rule = load_rule(rule)
 
@@ -77,8 +80,6 @@ module Mihari
       # @return [nil]
       #
       def run_rule_analyzer(analyzer, ignore_old_artifacts: false, ignore_threshold: 0)
-        load_configuration
-
         analyzer.ignore_old_artifacts = ignore_old_artifacts
         analyzer.ignore_threshold = ignore_threshold
 

data/lib/mihari/errors.rb

@@ -6,4 +6,6 @@ module Mihari
   class InvalidInputError < Error; end
 
   class RetryableError < Error; end
+
+  class FileNotFoundError < Error; end
 end

data/lib/mihari/mixins/configuration.rb

@@ -80,10 +80,20 @@ module Mihari
         end
       end
 
+      #
+      # Load configuration file
+      #
+      # @param [String] path
+      #
+      # @return [Hash]
+      #
       def _load_config(path)
-        return YAML.safe_load(File.read(path), symbolize_names: true) if Pathname(path).exist?
+        unless Pathname(path).exist?
+          puts "#{path} does not exist".colorize(:red)
+          raise FileNotFoundError
+        end
 
-        YAML.safe_load(path, symbolize_names: true)
+        YAML.safe_load(File.read(path), symbolize_names: true)
       end
     end
   end

data/lib/mihari/models/alert.rb

@@ -30,14 +30,7 @@ module Mihari
 
         # TODO: improve queires
         alert_ids = relation.limit(limit).offset(offset).order(id: :desc).pluck(:id).uniq
-        alerts = includes(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
-
-        alerts.map do |alert|
-          json = Serializers::AlertSerializer.new(alert).as_json
-          json[:artifacts] = json[:artifacts] || []
-          json[:tags] = json[:tags] || []
-          json
-        end
+        includes(:artifacts, :tags).where(id: [alert_ids]).order(id: :desc)
       end
 
       #

data/lib/mihari/models/artifact.rb

@@ -29,10 +29,13 @@ module Mihari
 
     validates_with ArtifactValidator
 
+    attr_accessor :tags
+
     def initialize(attributes)
       super
 
       self.data_type = TypeChecker.type(data)
+      self.tags = []
     end
 
     #

data/lib/mihari/schemas/configuration.rb

@@ -13,6 +13,8 @@ module Mihari
       optional(:censys_secret).value(:string)
       optional(:circl_passive_password).value(:string)
       optional(:circl_passive_username).value(:string)
+      optional(:database).value(:string)
+      optional(:greynoise_api_key).value(:string)
       optional(:ipinfo_api_key).value(:string)
       optional(:misp_api_endpoint).value(:string)
       optional(:misp_api_key).value(:string)
@@ -30,10 +32,9 @@ module Mihari
       optional(:thehive_api_key).value(:string)
       optional(:urlscan_api_key).value(:string)
       optional(:virustotal_api_key).value(:string)
-      optional(:zoomeye_api_key).value(:string)
       optional(:webhook_url).value(:string)
       optional(:webhook_use_json_body).value(:bool)
-      optional(:database).value(:string)
+      optional(:zoomeye_api_key).value(:string)
     end
 
     class ConfigurationContract < Dry::Validation::Contract

data/lib/mihari/structs/greynoise.rb

@@ -0,0 +1,55 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module GreyNoise
+      class Metadata < Dry::Struct
+        attribute :country, Types::String
+        attribute :country_code, Types::String
+        attribute :asn, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            country: d.fetch("country"),
+            country_code: d.fetch("country_code"),
+            asn: d.fetch("asn")
+          )
+        end
+      end
+
+      class Datum < Dry::Struct
+        attribute :ip, Types::String
+        attribute :metadata, Metadata
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            ip: d.fetch("ip"),
+            metadata: Metadata.from_dynamic!(d.fetch("metadata"))
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :complete, Types::Bool
+        attribute :count, Types::Int
+        attribute :data, Types.Array(Datum)
+        attribute :message, Types::String
+        attribute :query, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            complete: d.fetch("complete"),
+            count: d.fetch("count"),
+            data: d.fetch("data").map { |x| Datum.from_dynamic!(x) },
+            message: d.fetch("message"),
+            query: d.fetch("query")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/ipinfo.rb

@@ -18,10 +18,9 @@ module Mihari
             d = Types::Hash[d]
 
             asn = nil
-            org = d["org"]
-            unless org.nil?
-              asn = org.split.first
-              asn = normalize_asn(asn)
+            asn_ = d.dig("asn", "asn")
+            unless asn_.nil?
+              asn = normalize_asn(asn_)
             end
 
             new(

data/lib/mihari/structs/shodan.rb

@@ -5,20 +5,20 @@ module Mihari
   module Structs
     module Shodan
       class Location < Dry::Struct
-        attribute :country_code, Types::String
-        attribute :country_name, Types::String
+        attribute :country_code, Types::String.optional
+        attribute :country_name, Types::String.optional
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
-            country_code: d.fetch("country_code"),
-            country_name: d.fetch("country_name")
+            country_code: d["country_code"],
+            country_name: d["country_name"]
           )
         end
       end
 
       class Match < Dry::Struct
-        attribute :asn, Types::String
+        attribute :asn, Types::String.optional
         attribute :hostnames, Types.Array(Types::String)
         attribute :location, Location
         attribute :domains, Types.Array(Types::String)
@@ -27,7 +27,7 @@ module Mihari
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
-            asn: d.fetch("asn"),
+            asn: d["asn"],
             hostnames: d.fetch("hostnames"),
             location: Location.from_dynamic!(d.fetch("location")),
             domains: d.fetch("domains"),

data/lib/mihari/types.rb

@@ -19,6 +19,7 @@ module Mihari
       "circl",
       "dnpedia",
       "dnstwister",
+      "greynoise",
       "onyphe",
       "otx",
       "passivetotal",

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.9.0"
+  VERSION = "3.10.1"
 end

data/lib/mihari/web/api.rb

@@ -0,0 +1,43 @@
+# Entities
+require "mihari/web/entities/message"
+
+require "mihari/web/entities/autonomous_system"
+require "mihari/web/entities/command"
+require "mihari/web/entities/config"
+require "mihari/web/entities/dns"
+require "mihari/web/entities/geolocation"
+require "mihari/web/entities/ip_address"
+require "mihari/web/entities/reverse_dns"
+require "mihari/web/entities/source"
+require "mihari/web/entities/tag"
+require "mihari/web/entities/whois"
+
+require "mihari/web/entities/artifact"
+
+require "mihari/web/entities/alert"
+
+# Endpoints
+require "mihari/web/endpoints/alerts"
+require "mihari/web/endpoints/artifacts"
+require "mihari/web/endpoints/command"
+require "mihari/web/endpoints/configs"
+require "mihari/web/endpoints/ip_addresses"
+require "mihari/web/endpoints/sources"
+require "mihari/web/endpoints/tags"
+
+module Mihari
+  class API < Grape::API
+    prefix "api"
+    format :json
+
+    mount Endpoints::Alerts
+    mount Endpoints::Artifacts
+    mount Endpoints::Command
+    mount Endpoints::Configs
+    mount Endpoints::IPAddresses
+    mount Endpoints::Sources
+    mount Endpoints::Tags
+
+    add_swagger_documentation(api_version: "v1", info: { title: "Mihari API" })
+  end
+end

data/lib/mihari/web/app.rb

@@ -2,46 +2,47 @@
 
 require "launchy"
 require "rack"
+require "rack/contrib"
 require "rack/handler/puma"
-require "sinatra"
+require "rack/cors"
 
-require "mihari/web/helpers/json"
+require "grape"
+require "grape-entity"
+require "grape-swagger"
+require "grape-swagger-entity"
 
-require "mihari/web/controllers/base_controller"
-
-require "mihari/web/controllers/alerts_controller"
-require "mihari/web/controllers/analyzers_controller"
-require "mihari/web/controllers/artifacts_controller"
-require "mihari/web/controllers/command_controller"
-require "mihari/web/controllers/config_controller"
-require "mihari/web/controllers/ip_address_controller"
-require "mihari/web/controllers/sources_controller"
-require "mihari/web/controllers/tags_controller"
+require "mihari/web/api"
 
 module Mihari
-  class App < Sinatra::Base
-    set :root, File.dirname(__FILE__)
-    set :public_folder, File.join(root, "public")
-
-    get "/" do
-      send_file File.join(settings.public_folder, "index.html")
+  class App
+    def initialize
+      @filenames = ["", ".html", "index.html", "/index.html"]
+      @rack_static = ::Rack::Static.new(
+        -> { [404, {}, []] },
+        root: File.expand_path("./public", __dir__),
+        urls: ["/"]
+      )
     end
 
-    use Mihari::Controllers::AlertsController
-    use Mihari::Controllers::AnalyzersController
-    use Mihari::Controllers::ArtifactsController
-    use Mihari::Controllers::CommandController
-    use Mihari::Controllers::ConfigController
-    use Mihari::Controllers::IPAddressController
-    use Mihari::Controllers::SourcesController
-    use Mihari::Controllers::TagsController
-
     class << self
+      def instance
+        @instance ||= Rack::Builder.new do
+          use Rack::Cors do
+            allow do
+              origins "*"
+              resource "*", headers: :any, methods: [:get, :post, :put, :delete, :options]
+            end
+          end
+
+          run App.new
+        end.to_app
+      end
+
       def run!(port: 9292, host: "localhost", threads: "0:16", verbose: false)
         url = "http://#{host}:#{port}"
 
-        Rack::Handler::Puma.run(self, Port: port, Host: host, Threads: threads, Verbose: verbose) do |server|
-          Launchy.open url
+        Rack::Handler::Puma.run(instance, Port: port, Host: host, Threads: threads, Verbose: verbose) do |server|
+          Launchy.open(url) if ENV["RACK_ENV"] != "development"
 
           [:INT, :TERM].each do |sig|
             trap(sig) do
@@ -51,5 +52,22 @@ module Mihari
         end
       end
     end
+
+    def call(env)
+      # api
+      api_response = API.call(env)
+
+      # Check if the App wants us to pass the response along to others
+      if api_response[1]["X-Cascade"] == "pass"
+        # static files
+        request_path = env["PATH_INFO"]
+        @filenames.each do |path|
+          response = @rack_static.call(env.merge("PATH_INFO" => request_path + path))
+          return response if response[0] != 404
+        end
+      end
+
+      api_response
+    end
   end
 end

data/lib/mihari/web/endpoints/alerts.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Endpoints
+    class Alerts < Grape::API
+      namespace :alerts do
+        desc "Search alerts", {
+          is_array: true,
+          success: Entities::Alert,
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+        }
+        params do
+          optional :page, type: Integer
+          optional :artifact, type: String
+          optional :description, type: String
+          optional :source, type: String
+          optional :tag, type: String
+
+          optional :fromAt, type: DateTime
+          optional :toAt, type: DateTime
+
+          optional :asn, type: Integer
+          optional :dnsRecord, type: String
+          optional :reverseDnsName, type: String
+        end
+        get "/" do
+          filter = params.to_h.to_snake_keys
+
+          # set page & limit
+          page = filter["page"] || 1
+          filter["page"] = page.to_i
+
+          limit = 10
+          filter["limit"] = 10
+
+          # normalize keys
+          filter["artifact_data"] = filter["artifact"]
+          filter["tag_name"] = filter["tag"]
+
+          # symbolize hash keys
+          filter = filter.to_h.transform_keys(&:to_sym)
+
+          search_filter_with_pagenation = Structs::Alert::SearchFilterWithPagination.new(**filter)
+          alerts = Mihari::Alert.search(search_filter_with_pagenation)
+          total = Mihari::Alert.count(search_filter_with_pagenation.without_pagination)
+
+          present({ alerts: alerts, total: total, current_page: page, page_size: limit }, with: Entities::AlertsWithPagination)
+        end
+
+        desc "Delete an alert", {
+          success: Entities::Message,
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+        }
+        params do
+          requires :id, type: Integer
+        end
+        delete "/:id" do
+          id = params["id"].to_i
+
+          begin
+            alert = Mihari::Alert.find(id)
+          rescue ActiveRecord::RecordNotFound
+            error!({ message: "ID:#{id} is not found" }, 404)
+          end
+
+          alert.destroy
+
+          status 204
+          present({ message: "" }, with: Entities::Message)
+        end
+      end
+    end
+  end
+end