mihari 3.9.1 → 3.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2e67ff3e1ae2bf328a9a77ef7c9a88dce779749c422490a97106d3529a9a3b1
-  data.tar.gz: a71ee49c8fcb0b06e180739a588930783dabbc40078d234f791314dd3f8af9b4
+  metadata.gz: 2c8fda839e1bb1ec4733b7da17e20e3b47bd5795d5aca25371d09e5a0fa9a575
+  data.tar.gz: a4551c61c625bd08167608051750bf1785a31ad5215e95e857753fc5fed31d00
 SHA512:
-  metadata.gz: 41c30a97d80e6d96f425230401b7f0ee947979dff6b2a8c458bb72c38ed34c2577bed1b007f25e2607cacc1c70f6d7de183722bac0b604f0c54f582758db1e53
-  data.tar.gz: 6cb1d47e4efec3fb54bd93ddacf5e8f55e423d3b28c1e3c6ae2ea852898b88d3995d8eb767e89535ba50bfffb6b533ce946068eb348cfbd33afe41ace587a5df
+  metadata.gz: 81a1ca91b65b03f98bb3504a85c8113b2cf189a4911109e113da1058fb9a1bd61c8807e6f3bec72484786b89e68d0027782d38013676bd6ae78aa57434315972
+  data.tar.gz: b42b1658310a7ee87940fbebb6507edca2187d6be92957a149f9234af8bf32ba06e702c401d321b553901dc56094964e78ff602549734f51abbf24b4b7a1fa4d

data/README.md

@@ -38,6 +38,7 @@ Mihari supports the following services by default.
 - [crt.sh](https://crt.sh/)
 - [DN Pedia](https://dnpedia.com/)
 - [dnstwister](https://dnstwister.report/)
+- [GreyNoise](https://www.greynoise.io/)
 - [Onyphe](https://onyphe.io)
 - [OTX](https://otx.alienvault.com/)
 - [PassiveTotal](https://community.riskiq.com/)

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.0.2-alpine3.13
+FROM ruby:3.0.3-alpine3.13
 
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
   && gem install pg mysql2 \

data/lib/mihari/analyzers/binaryedge.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         results = search
         return [] unless results || results.empty?
@@ -55,6 +57,9 @@ module Mihari
 
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         responses
       end

data/lib/mihari/analyzers/censys.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         search
       end
@@ -33,6 +35,9 @@ module Mihari
 
           cursor = response.result.links.next
           break if cursor == ""
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
 
         artifacts.flatten.uniq(&:data)

data/lib/mihari/analyzers/greynoise.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "greynoise"
+
+module Mihari
+  module Analyzers
+    class GreyNoise < Base
+      param :query
+      option :title, default: proc { "GreyNoise search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
+
+      def artifacts
+        res = Structs::GreyNoise::Response.from_dynamic!(search)
+        res.data.map do |datum|
+          build_artifact datum
+        end
+      end
+
+      private
+
+      PAGE_SIZE = 10_000
+
+      def configuration_keys
+        %w[greynoise_api_key]
+      end
+
+      def api
+        @api ||= ::GreyNoise::API.new(key: Mihari.config.greynoise_api_key)
+      end
+
+      #
+      # Search
+      #
+      # @return [Hash]
+      #
+      def search
+        api.experimental.gnql(query, size: PAGE_SIZE)
+      end
+
+      #
+      # Build an artifact from a GreyNoise search API response
+      #
+      # @param [Structs::GreyNoise::Datum] datum
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(datum)
+        as = AutonomousSystem.new(asn: normalize_asn(datum.metadata.asn))
+
+        geolocation = Geolocation.new(
+          country: datum.metadata.country,
+          country_code: datum.metadata.country_code
+        )
+
+        Artifact.new(
+          data: datum.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/onyphe.rb

@@ -11,6 +11,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         responses = search
         return [] unless responses
@@ -59,6 +61,9 @@ module Mihari
 
           total = res.total
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         responses
       end

data/lib/mihari/analyzers/rule.rb

@@ -11,6 +11,7 @@ module Mihari
       "crtsh" => Crtsh,
       "dnpedia" => DNPedia,
       "dnstwister" => DNSTwister,
+      "greynoise" => GreyNoise,
       "onyphe" => Onyphe,
       "otx" => OTX,
       "passivetotal" => PassiveTotal,
@@ -63,6 +64,12 @@ module Mihari
           klass = get_analyzer_class(analyzer_name)
 
           query = params[:query]
+
+          # set interval in the top level
+          options = params[:options] || {}
+          interval = options[:interval]
+          params[:interval] = interval
+
           analyzer = klass.new(query, **params)
 
           # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>

data/lib/mihari/analyzers/shodan.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         results = search
         return [] unless results || results.empty?
@@ -58,10 +60,14 @@ module Mihari
         responses = []
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
+
           break unless res
 
           responses << res
           break if res["total"].to_i <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         rescue JSON::ParserError
           # ignore JSON::ParserError
           # ref. https://github.com/ninoseki/mihari/issues/197
@@ -78,11 +84,16 @@ module Mihari
       # @return [Artifact]
       #
       def build_artifact(match)
-        as = AutonomousSystem.new(asn: normalize_asn(match.asn))
-        geolocation = Geolocation.new(
-          country: match.location.country_name,
-          country_code: match.location.country_code
-        )
+        as = nil
+        as = AutonomousSystem.new(asn: normalize_asn(match.asn)) unless match.asn.nil?
+
+        geolocation = nil
+        if !match.location.country_name.nil? && !match.location.country_code.nil?
+          geolocation = Geolocation.new(
+            country: match.location.country_name,
+            country_code: match.location.country_code
+          )
+        end
 
         Artifact.new(
           data: match.ip_str,

data/lib/mihari/analyzers/urlscan.rb

@@ -2,8 +2,6 @@
 
 require "urlscan"
 
-SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
-
 module Mihari
   module Analyzers
     class Urlscan < Base
@@ -12,7 +10,11 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
       option :allowed_data_types, default: proc { SUPPORTED_DATA_TYPES }
-      option :use_similarity, default: proc { false }
+
+      option :interval, default: proc { 0 }
+
+      SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
+      SIZE = 1000
 
       def initialize(*args, **kwargs)
         super
@@ -21,16 +23,15 @@ module Mihari
       end
 
       def artifacts
-        result = search
-        return [] unless result
-
-        results = result["results"] || []
+        responses = search
+        results = responses.map(&:results).flatten
 
         allowed_data_types.map do |type|
-          results.filter_map do |match|
-            match.dig "page", type
+          results.filter_map do |result|
+            page = result.page
+            page.send(type.to_sym)
           end.uniq
-        end.flatten
+        end.flatten.compact
       end
 
       private
@@ -43,15 +44,38 @@ module Mihari
         @api ||= ::UrlScan::API.new(Mihari.config.urlscan_api_key)
       end
 
+      #
+      # Search with search_after option
+      #
+      # @return [Structs::Urlscan::Response]
+      #
+      def search_with_search_after(search_after: nil)
+        res = api.search(query, size: SIZE, search_after: search_after)
+        Structs::Urlscan::Response.from_dynamic! res
+      end
+
       #
       # Search
       #
-      # @return [Array<Hash>]
+      # @return [Array<Structs::Urlscan::Response>]
       #
       def search
-        return api.pro.similar(query) if use_similarity
+        responses = []
+
+        search_after = nil
+        loop do
+          res = search_with_search_after(search_after: search_after)
+          responses << res
+
+          break if res.results.length < SIZE
+
+          search_after = res.results.last.sort.join(",")
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
+        end
 
-        api.search(query, size: 10_000)
+        responses
       end
 
       #

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def initialize(*args, **kwargs)
         super
 
@@ -54,6 +56,9 @@ module Mihari
           break if response.meta.cursor.nil?
 
           cursor = response.meta.cursor
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
 
         responses

data/lib/mihari/analyzers/zoomeye.rb

@@ -11,6 +11,8 @@ module Mihari
       option :tags, default: proc { [] }
       option :type, default: proc { "host" }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         case type
         when "host"
@@ -87,6 +89,9 @@ module Mihari
           total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         convert_responses responses.compact
       end
@@ -119,6 +124,9 @@ module Mihari
           total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         convert_responses responses.compact
       end

data/lib/mihari/cli/analyzer.rb

@@ -6,6 +6,7 @@ require "mihari/commands/circl"
 require "mihari/commands/crtsh"
 require "mihari/commands/dnpedia"
 require "mihari/commands/dnstwister"
+require "mihari/commands/greynoise"
 require "mihari/commands/onyphe"
 require "mihari/commands/otx"
 require "mihari/commands/passivetotal"
@@ -25,6 +26,7 @@ module Mihari
     class Analyzer < Base
       class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not."
       class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not."
+      class_option :interval, type: :numeric, default: 0, desc: "Seconds of the interval while calling API in a row."
       class_option :config, type: :string, desc: "Path to the config file"
 
       include Mihari::Commands::BinaryEdge
@@ -33,6 +35,7 @@ module Mihari
       include Mihari::Commands::Crtsh
       include Mihari::Commands::DNPedia
       include Mihari::Commands::DNSTwister
+      include Mihari::Commands::GreyNoise
       include Mihari::Commands::JSON
       include Mihari::Commands::Onyphe
       include Mihari::Commands::OTX

data/lib/mihari/commands/greynoise.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module GreyNoise
+      def self.included(thor)
+        thor.class_eval do
+          desc "greynoise [QUERY]", "GreyNoise search"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          def greynoise(query)
+            with_error_handling do
+              run_analyzer Analyzers::GreyNoise, query: query, options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/search.rb

@@ -10,6 +10,9 @@ module Mihari
           desc "search [RULE]", "Search by a rule"
           method_option :config, type: :string, desc: "Path to the config file"
           def search_by_rule(rule)
+            # load configuration
+            load_configuration
+
             # convert str(YAML) to hash or str(path/YAML file) to hash
             rule = load_rule(rule)
 
@@ -77,8 +80,6 @@ module Mihari
       # @return [nil]
       #
       def run_rule_analyzer(analyzer, ignore_old_artifacts: false, ignore_threshold: 0)
-        load_configuration
-
         analyzer.ignore_old_artifacts = ignore_old_artifacts
         analyzer.ignore_threshold = ignore_threshold
 

data/lib/mihari/commands/urlscan.rb

@@ -9,8 +9,7 @@ module Mihari
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
-          method_option :target_type, type: :string, default: "url", desc: "target type to fetch from search results (target type should be 'url', 'domain' or 'ip')"
-          method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
+          method_option :allowed_data_types, type: :array, default: ["url", "ip", "domain"], desc: "types to fetch from search results ('url', 'domain' or 'ip')"
           def urlscan(query)
             with_error_handling do
               run_analyzer Analyzers::Urlscan, query: query, options: options

data/lib/mihari/schemas/configuration.rb

@@ -13,6 +13,8 @@ module Mihari
       optional(:censys_secret).value(:string)
       optional(:circl_passive_password).value(:string)
       optional(:circl_passive_username).value(:string)
+      optional(:database).value(:string)
+      optional(:greynoise_api_key).value(:string)
       optional(:ipinfo_api_key).value(:string)
       optional(:misp_api_endpoint).value(:string)
       optional(:misp_api_key).value(:string)
@@ -30,10 +32,9 @@ module Mihari
       optional(:thehive_api_key).value(:string)
       optional(:urlscan_api_key).value(:string)
       optional(:virustotal_api_key).value(:string)
-      optional(:zoomeye_api_key).value(:string)
       optional(:webhook_url).value(:string)
       optional(:webhook_use_json_body).value(:bool)
-      optional(:database).value(:string)
+      optional(:zoomeye_api_key).value(:string)
     end
 
     class ConfigurationContract < Dry::Validation::Contract

data/lib/mihari/schemas/rule.rb

@@ -7,33 +7,41 @@ require "mihari/schemas/macros"
 
 module Mihari
   module Schemas
+    AnalyzerOptions = Dry::Schema.Params do
+      optional(:interval).value(:integer)
+    end
+
     Analyzer = Dry::Schema.Params do
       required(:analyzer).value(Types::AnalyzerTypes)
       required(:query).value(:string)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Spyse = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("spyse"))
       required(:query).value(:string)
       required(:type).value(Types::String.enum("ip", "domain"))
+      optional(:options).hash(AnalyzerOptions)
     end
 
     ZoomEye = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("zoomeye"))
       required(:query).value(:string)
       required(:type).value(Types::String.enum("host", "web"))
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Crtsh = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("crtsh"))
       required(:query).value(:string)
       optional(:exclude_expired).value(:bool).default(true)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Urlscan = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("urlscan"))
       required(:query).value(:string)
-      optional(:use_similarity).value(:bool).default(true)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Rule = Dry::Schema.Params do

data/lib/mihari/structs/greynoise.rb

@@ -0,0 +1,55 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module GreyNoise
+      class Metadata < Dry::Struct
+        attribute :country, Types::String
+        attribute :country_code, Types::String
+        attribute :asn, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            country: d.fetch("country"),
+            country_code: d.fetch("country_code"),
+            asn: d.fetch("asn")
+          )
+        end
+      end
+
+      class Datum < Dry::Struct
+        attribute :ip, Types::String
+        attribute :metadata, Metadata
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            ip: d.fetch("ip"),
+            metadata: Metadata.from_dynamic!(d.fetch("metadata"))
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :complete, Types::Bool
+        attribute :count, Types::Int
+        attribute :data, Types.Array(Datum)
+        attribute :message, Types::String
+        attribute :query, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            complete: d.fetch("complete"),
+            count: d.fetch("count"),
+            data: d.fetch("data").map { |x| Datum.from_dynamic!(x) },
+            message: d.fetch("message"),
+            query: d.fetch("query")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/shodan.rb

@@ -5,20 +5,20 @@ module Mihari
   module Structs
     module Shodan
       class Location < Dry::Struct
-        attribute :country_code, Types::String
-        attribute :country_name, Types::String
+        attribute :country_code, Types::String.optional
+        attribute :country_name, Types::String.optional
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
-            country_code: d.fetch("country_code"),
-            country_name: d.fetch("country_name")
+            country_code: d["country_code"],
+            country_name: d["country_name"]
           )
         end
       end
 
       class Match < Dry::Struct
-        attribute :asn, Types::String
+        attribute :asn, Types::String.optional
         attribute :hostnames, Types.Array(Types::String)
         attribute :location, Location
         attribute :domains, Types.Array(Types::String)
@@ -27,7 +27,7 @@ module Mihari
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
-            asn: d.fetch("asn"),
+            asn: d["asn"],
             hostnames: d.fetch("hostnames"),
             location: Location.from_dynamic!(d.fetch("location")),
             domains: d.fetch("domains"),

data/lib/mihari/structs/urlscan.rb

@@ -0,0 +1,51 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module Urlscan
+      class Page < Dry::Struct
+        attribute :domain, Types::String.optional
+        attribute :ip, Types::String.optional
+        attribute :url, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            domain: d["domain"],
+            ip: d["ip"],
+            url: d.fetch("url")
+          )
+        end
+      end
+
+      class Result < Dry::Struct
+        attribute :page, Page
+        attribute :id, Types::String
+        attribute :sort, Types.Array(Types::String | Types::Integer)
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            page: Page.from_dynamic!(d.fetch("page")),
+            id: d.fetch("_id"),
+            sort: d.fetch("sort")
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :results, Types.Array(Result)
+        attribute :has_more, Types::Bool
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            results: d.fetch("results").map { |x| Result.from_dynamic!(x) },
+            has_more: d.fetch("has_more")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/types.rb

@@ -8,17 +8,21 @@ module Mihari
     Nil = Strict::Nil
     Hash = Strict::Hash
     String = Strict::String
+    Bool = Strict::Bool
     Double = Strict::Float | Strict::Integer
     DateTime = Strict::DateTime
 
     DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
 
+    UrlscanDataTypes = Types::String.enum("ip", "domain", "url")
+
     AnalyzerTypes = Types::String.enum(
       "binaryedge",
       "censys",
       "circl",
       "dnpedia",
       "dnstwister",
+      "greynoise",
       "onyphe",
       "otx",
       "passivetotal",

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.9.1"
+  VERSION = "3.11.0"
 end

data/lib/mihari/web/public/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/static/favicon.ico"><title>Mihari</title><link href="/static/js/app.6b636b62.js" rel="preload" as="script"></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/static/js/app.6b636b62.js"></script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/static/favicon.ico"><title>Mihari</title><link href="/static/js/app.fbc19869.js" rel="preload" as="script"></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/static/js/app.fbc19869.js"></script></body></html>