mihari 2.1.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dff04f8065ceff1c6b9fa68bd606db9cc4789ed034a330019143a69c4ec2c037
-  data.tar.gz: d307e3e48eedf5d17245da14a20bd578468e03a70b2ba302fcc4ea68aab5651c
+  metadata.gz: 388f48a9001d38fd83f4a6d527d7c5826a490be1d339596d174a9f619a78cb0c
+  data.tar.gz: 7e0a6e2fcbe9ad1792c21472b8be7706a86b09fbbce951edb1829a76493aaedc
 SHA512:
-  metadata.gz: 97f8237d491778739e307e15d0b5b96fba510a548a0a42ae1760e41469cef7fad551b6e876025d7daed58e449c77bee6d3573d659c85e4b2886712229e2ee7ef
-  data.tar.gz: 2d5f9125f2039ce7d8727661a8f71f1a7c0b9021a0eaf613242598371fb6066234fa5a8959528190bc7275b35da026932700d13929cfd9f94aa04eded6881951
+  metadata.gz: dfcde6c4fa80ae12c56606157c6800c7e321cef71ed3e4aa9250805ea51126c74a19b3f73040630d169966fb17d834d8c45b37cc6f7baa808d7eea3e7c585fb9
+  data.tar.gz: d477cdcc4b4075e7671263f32ed5e81daad42e499eded5dffcecfba2d7568b779e99010a64a271a652db3f928800506cb6a4c7cf4060816742d4bb8d5bbec86a

data/.rubocop.yml

@@ -4,6 +4,9 @@
 require:
   - rubocop-performance
 
+AllCops:
+  NewCops: enable
+
 Style/Alias:
   Enabled: false
   StyleGuide: https://relaxed.ruby.style/#stylealias
@@ -151,5 +154,8 @@ Lint/AssignmentInCondition:
 Layout/LineLength:
   Enabled: false
 
+Style/StringLiteralsInInterpolation:
+  Enabled: false
+
 Metrics:
   Enabled: false

data/README.md

@@ -8,10 +8,14 @@
 
 ![img](https://github.com/ninoseki/mihari/raw/master/images/logo.png)
 
+[![](images/tines.png)](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki)
+
 Mihari is a framework for continuous OSINT based threat hunting.
 
 ## How it works
 
+![img](https://github.com/ninoseki/mihari/raw/master/images/overview.png)
+
 - Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs or hashes).
 - Mihari checks whether a DB (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
@@ -59,3 +63,7 @@ See [Usage](https://github.com/ninoseki/mihari/wiki/Usage) for more information.
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Acknowledgement
+
+Mihari is proudly supported by [Tines.io](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki), The SOAR Platform for Enterprise Security Teams.

data/bin/console

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require "bundler/setup"
 require "mihari"

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.0.0-alpine3.13
+FROM ruby:3.0.1-alpine3.13
 
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
   && gem install pg mysql2 \

data/examples/ipinfo_hosted_domains.rb

@@ -34,7 +34,7 @@ module Mihari
         uri = URI("#{IPINFO_API_ENDPOINT}/domains/#{ip}?token=#{token}")
         res = uri.read
         json = JSON.parse(res)
-        json.dig("domains") || []
+        json["domains"] || []
       end
     end
   end

data/lib/mihari.rb

@@ -78,6 +78,7 @@ require "mihari/emitters/misp"
 require "mihari/emitters/slack"
 require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
+require "mihari/emitters/webhook"
 
 require "mihari/status"
 

data/lib/mihari/analyzers/base.rb

@@ -8,6 +8,13 @@ module Mihari
       include Configurable
       include Retriable
 
+      attr_accessor :ignore_old_artifacts, :ignore_threshold
+
+      def initialize
+        @ignore_old_artifacts = false
+        @ignore_threshold = 0
+      end
+
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -61,7 +68,9 @@ module Mihari
 
       # @return [Array<Mihari::Artifact>]
       def unique_artifacts
-        @unique_artifacts ||= normalized_artifacts.select(&:unique?)
+        @unique_artifacts ||= normalized_artifacts.select do |artifact|
+          artifact.unique?(ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold)
+        end
       end
 
       def set_unique_artifacts

data/lib/mihari/analyzers/circl.rb

@@ -46,14 +46,14 @@ module Mihari
       def passive_dns_lookup
         results = api.dns.query(@query)
         results.map do |result|
-          type = result.dig("rrtype")
-          type == "A" ? result.dig("rdata") : nil
+          type = result["rrtype"]
+          type == "A" ? result["rdata"] : nil
         end.compact.uniq
       end
 
       def passive_ssl_lookup
         result = api.ssl.cquery(@query)
-        seen = result.dig("seen") || []
+        seen = result["seen"] || []
         seen.uniq
       end
     end

data/lib/mihari/analyzers/onyphe.rb

@@ -21,10 +21,10 @@ module Mihari
         return [] unless results
 
         flat_results = results.map do |result|
-          result.dig("results")
+          result["results"]
         end.flatten.compact
 
-        flat_results.map { |result| result.dig("ip") }.compact.uniq
+        flat_results.map { |result| result["ip"] }.compact.uniq
       end
 
       private

data/lib/mihari/analyzers/urlscan.rb

@@ -5,16 +5,14 @@ require "urlscan"
 module Mihari
   module Analyzers
     class Urlscan < Base
-      attr_reader :title, :description, :query, :tags, :filter, :target_type, :use_pro, :use_similarity
+      attr_reader :title, :description, :query, :tags, :target_type, :use_similarity
 
       def initialize(
         query,
         description: nil,
-        filter: nil,
         tags: [],
         target_type: "url",
         title: nil,
-        use_pro: false,
         use_similarity: false
       )
         super()
@@ -24,9 +22,7 @@ module Mihari
         @description = description || "query = #{query}"
         @tags = tags
 
-        @filter = filter
         @target_type = target_type
-        @use_pro = use_pro
         @use_similarity = use_similarity
 
         raise InvalidInputError, "type should be url, domain or ip." unless valid_target_type?
@@ -54,7 +50,6 @@ module Mihari
 
       def search
         return api.pro.similar(query) if use_similarity
-        return api.pro.search(query: query, filter: filter, size: 10_000) if use_pro
 
         api.search(query, size: 10_000)
       end

data/lib/mihari/analyzers/zoomeye.rb

@@ -37,11 +37,11 @@ module Mihari
       end
 
       def config_keys
-        %w[zoomeye_password zoomeye_username]
+        %w[zoomeye_api_key]
       end
 
       def api
-        @api ||= ::ZoomEye::API.new(username: Mihari.config.zoomeye_username, password: Mihari.config.zoomeye_password)
+        @api ||= ::ZoomEye::API.new(api_key: Mihari.config.zoomeye_api_key)
       end
 
       def convert_responses(responses)

data/lib/mihari/cli.rb

@@ -33,7 +33,10 @@ require "mihari/commands/web"
 
 module Mihari
   class CLI < Thor
-    class_option :config, type: :string, desc: "path to the config file"
+    class_option :config, type: :string, desc: "Path to the config file"
+
+    class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not. Only affects with analyze commands."
+    class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not. Only affects with analyze commands."
 
     include Mihari::Commands::BinaryEdge
     include Mihari::Commands::Censys
@@ -96,16 +99,22 @@ module Mihari
         options = normalize_options(options)
 
         analyzer = analyzer_class.new(query, **options)
+
+        analyzer.ignore_old_artifacts = options[:ignore_old_artifacts] || false
+        analyzer.ignore_threshold = options[:ignore_threshold] || 0
+
         analyzer.run
       end
 
       def symbolize_hash_keys(hash)
-        hash.map { |k, v| [k.to_sym, v] }.to_h
+        hash.transform_keys(&:to_sym)
       end
 
       def normalize_options(options)
         # Delete :config because it is not intended to use for running an analyzer
-        options.delete(:config)
+        [:config, :ignore_old_artifacts, :ignore_threshold].each do |ignore_key|
+          options.delete(ignore_key)
+        end
         options
       end
 

data/lib/mihari/commands/config.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require "colorize"
+
 module Mihari
   module Commands
     module Config
@@ -8,11 +12,13 @@ module Mihari
           def init_config
             filename = options["filename"]
 
-            if File.exist?(filename) && !(yes? "#{filename} exists. Do you want to overwrite it? (y/n)")
+            warning = "#{filename} exists. Do you want to overwrite it? (y/n)"
+            if File.exist?(filename) && !(yes? warning)
               return
             end
 
             Mihari::Config.initialize_yaml filename
+            puts "The config file is initialized as #{filename}.".colorize(:blue)
           end
         end
       end

data/lib/mihari/commands/dnstwister.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module DNSTwister

data/lib/mihari/commands/json.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module JSON
@@ -18,6 +20,10 @@ module Mihari
               tags = json["tags"] || []
 
               basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
+
+              basic.ignore_old_artifacts = options["ignore_old_artifacts"] || false
+              basic.ignore_threshold = options["ignore_threshold"] || 0
+
               basic.run
             end
           end

data/lib/mihari/commands/onyphe.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module Onyphe

data/lib/mihari/commands/passive_dns.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module PassiveDNS

data/lib/mihari/commands/securitytrails.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module SecurityTrails

data/lib/mihari/commands/shodan.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module Shodan

data/lib/mihari/commands/spyse.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module Spyse

data/lib/mihari/commands/urlscan.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module Urlscan
@@ -7,9 +9,7 @@ module Mihari
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
-          method_option :filter, type: :string, desc: "filter for urlscan pro search"
           method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
-          method_option :use_pro, type: :boolean, default: false, desc: "use pro search API or not"
           method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
           def urlscan(query)
             with_error_handling do

data/lib/mihari/commands/virustotal.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module VirusTotal

data/lib/mihari/commands/web.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module Web

data/lib/mihari/commands/zoomeye.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module ZoomEye

data/lib/mihari/config.rb

@@ -4,7 +4,7 @@ require "yaml"
 
 module Mihari
   class Config
-    attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_password, :zoomeye_username, :database
+    attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_api_key, :webhook_url, :webhook_use_json_body, :database
 
     def initialize
       load_from_env
@@ -32,8 +32,9 @@ module Mihari
       @thehive_api_key = ENV["THEHIVE_API_KEY"]
       @urlscan_api_key = ENV["URLSCAN_API_KEY"]
       @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
-      @zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
-      @zoomeye_username = ENV["ZOOMEYE_USERNAME"]
+      @zoomeye_api_key = ENV["ZOOMEYE_API_KEY"]
+      @webhook_url = ENV["WEBHOOK_URL"]
+      @webhook_use_json_body = ENV["WEBHOOK_USE_JSON_BODY"]
 
       @database = ENV["DATABASE"] || "mihari.db"
     end
@@ -58,7 +59,7 @@ module Mihari
 
       def initialize_yaml(filename)
         keys = new.instance_variables.map do |key|
-          key.to_s[1..-1]
+          key.to_s[1..]
         end
 
         config = keys.map do |key|

data/lib/mihari/emitters/slack.rb

@@ -2,9 +2,6 @@
 
 require "slack-notifier"
 require "digest/sha2"
-require "mem"
-
-require "mihari/slack_monkeypatch"
 
 module Mihari
   module Emitters

data/lib/mihari/emitters/webhook.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+module Mihari
+  module Emitters
+    class Webhook < Base
+      # @return [true, false]
+      def valid?
+        webhook_url?
+      end
+
+      def emit(title:, description:, artifacts:, source:, tags:)
+        return if artifacts.empty?
+
+        uri = URI(Mihari.config.webhook_url)
+        data = {
+          title: title,
+          description: description,
+          artifacts: artifacts.map(&:data),
+          source: source,
+          tags: tags
+        }
+
+        if use_json_body
+          Net::HTTP.post(uri, data.to_json, "Content-Type" => "application/json")
+        else
+          Net::HTTP.post_form(uri, data)
+        end
+      end
+
+      private
+
+      def config_keys
+        %w[webhook_url]
+      end
+
+      def webhook_url
+        @webhook_url ||= Mihari.config.webhook_url
+      end
+
+      def webhook_url?
+        !webhook_url.nil?
+      end
+
+      def use_json_body
+        @use_json_body ||= truthy?(Mihari.config.webhook_use_json_body || 'false')
+      end
+
+      def truthy?(value)
+        return true if value == "true"
+        return true if value == true
+
+        false
+      end
+    end
+  end
+end