mihari 1.5.0 → 2.2.1

data/bin/console

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require "bundler/setup"
 require "mihari"

data/build_frontend.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+CURRENT_DIR=${PWD}
+
+mkdir -p tmp
+cd tmp
+git clone https://github.com/ninoseki/mihari-frontend.git
+cd mihari-frontend
+npm install
+npm run build
+
+cp -r dist/* ${CURRENT_DIR}/lib/mihari/web/public
+
+rm -rf ${CURRENT_DIR}/tmp/mihari-frontend

data/docker/Dockerfile

@@ -1,5 +1,7 @@
-FROM ruby:3.0.0-alpine3.13
-RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
+FROM ruby:3.0.1-alpine3.13
+
+RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
+  && gem install pg mysql2 \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/mihari.git \
   && cd mihari \
@@ -10,4 +12,4 @@ RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
 
 ENTRYPOINT ["mihari"]
 
-CMD ["--help"]
+CMD ["--help"]

data/examples/ipinfo_hosted_domains.rb

@@ -34,7 +34,7 @@ module Mihari
         uri = URI("#{IPINFO_API_ENDPOINT}/domains/#{ip}?token=#{token}")
         res = uri.read
         json = JSON.parse(res)
-        json.dig("domains") || []
+        json["domains"] || []
       end
     end
   end

data/lib/mihari.rb

@@ -79,8 +79,8 @@ require "mihari/emitters/slack"
 require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
 
-require "mihari/alert_viewer"
-
 require "mihari/status"
 
+require "mihari/web/app"
+
 require "mihari/cli"

data/lib/mihari/analyzers/base.rb

@@ -8,6 +8,13 @@ module Mihari
       include Configurable
       include Retriable
 
+      attr_accessor :ignore_old_artifacts, :ignore_threshold
+
+      def initialize
+        @ignore_old_artifacts = false
+        @ignore_threshold = 0
+      end
+
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -42,7 +49,7 @@ module Mihari
 
       def run_emitter(emitter)
         emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
-      rescue => e
+      rescue StandardError => e
         puts "Emission by #{emitter.class} is failed: #{e}"
       end
 
@@ -61,7 +68,9 @@ module Mihari
 
       # @return [Array<Mihari::Artifact>]
       def unique_artifacts
-        @unique_artifacts ||= normalized_artifacts.select(&:unique?)
+        @unique_artifacts ||= normalized_artifacts.select do |artifact|
+          artifact.unique?(ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold)
+        end
       end
 
       def set_unique_artifacts

data/lib/mihari/analyzers/circl.rb

@@ -46,14 +46,14 @@ module Mihari
       def passive_dns_lookup
         results = api.dns.query(@query)
         results.map do |result|
-          type = result.dig("rrtype")
-          type == "A" ? result.dig("rdata") : nil
+          type = result["rrtype"]
+          type == "A" ? result["rdata"] : nil
         end.compact.uniq
       end
 
       def passive_ssl_lookup
         result = api.ssl.cquery(@query)
-        seen = result.dig("seen") || []
+        seen = result["seen"] || []
         seen.uniq
       end
     end

data/lib/mihari/analyzers/onyphe.rb

@@ -21,10 +21,10 @@ module Mihari
         return [] unless results
 
         flat_results = results.map do |result|
-          result.dig("results")
+          result["results"]
         end.flatten.compact
 
-        flat_results.map { |result| result.dig("ip") }.compact.uniq
+        flat_results.map { |result| result["ip"] }.compact.uniq
       end
 
       private

data/lib/mihari/analyzers/securitytrails.rb

@@ -58,13 +58,13 @@ module Mihari
       end
 
       def ip_lookup
-        result = api.domains.search(filter: {ipv4: query})
+        result = api.domains.search(filter: { ipv4: query })
         records = result["records"] || []
         records.map { |record| record["hostname"] }.compact.uniq
       end
 
       def mail_lookup
-        result = api.domains.search(filter: {whois_email: query})
+        result = api.domains.search(filter: { whois_email: query })
         records = result["records"] || []
         records.map { |record| record["hostname"] }.compact.uniq
       end

data/lib/mihari/analyzers/shodan.rb

@@ -56,6 +56,10 @@ module Mihari
 
           responses << res
           break if res["total"].to_i <= page * PAGE_SIZE
+        rescue JSON::ParserError
+          # ignore JSON::ParserError
+          # ref. https://github.com/ninoseki/mihari/issues/197
+          next
         end
         responses
       end

data/lib/mihari/cli.rb

@@ -1,318 +1,84 @@
 # frozen_string_literal: true
 
 require "thor"
-require "json"
+
+require "mihari/commands/binaryedge"
+require "mihari/commands/censys"
+require "mihari/commands/circl"
+require "mihari/commands/crtsh"
+require "mihari/commands/dnpedia"
+require "mihari/commands/dnstwister"
+require "mihari/commands/onyphe"
+require "mihari/commands/otx"
+require "mihari/commands/passivetotal"
+require "mihari/commands/pulsedive"
+require "mihari/commands/securitytrails_domain_feed"
+require "mihari/commands/securitytrails"
+require "mihari/commands/shodan"
+require "mihari/commands/spyse"
+require "mihari/commands/urlscan"
+require "mihari/commands/virustotal"
+require "mihari/commands/zoomeye"
+
+require "mihari/commands/free_text"
+require "mihari/commands/http_hash"
+require "mihari/commands/passive_dns"
+require "mihari/commands/passive_ssl"
+require "mihari/commands/reverse_whois"
+require "mihari/commands/ssh_fingerprint"
+
+require "mihari/commands/config"
+require "mihari/commands/json"
+require "mihari/commands/web"
 
 module Mihari
   class CLI < Thor
-    class_option :config, type: :string, desc: "path to config file"
-
-    def self.exit_on_failure?
-      true
-    end
-
-    desc "censys [QUERY]", "Censys IPv4 search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, desc: "type to search (ipv4 / websites / certificates)", default: "ipv4"
-    def censys(query)
-      with_error_handling do
-        run_analyzer Analyzers::Censys, query: query, options: options
-      end
-    end
-
-    desc "shodan [QUERY]", "Shodan host search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def shodan(query)
-      with_error_handling do
-        run_analyzer Analyzers::Shodan, query: query, options: options
-      end
-    end
-
-    desc "onyphe [QUERY]", "Onyphe datascan search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def onyphe(query)
-      with_error_handling do
-        run_analyzer Analyzers::Onyphe, query: query, options: options
-      end
-    end
-
-    desc "urlscan [QUERY]", "urlscan search by a given query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :filter, type: :string, desc: "filter for urlscan pro search"
-    method_option :target_type, type: :string, default: "url", desc: "target type to fetch from lookup results (target type should be 'url', 'domain' or 'ip')"
-    method_option :use_pro, type: :boolean, default: false, desc: "use pro search API or not"
-    method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
-    def urlscan(query)
-      with_error_handling do
-        run_analyzer Analyzers::Urlscan, query: query, options: options
-      end
-    end
-
-    desc "virustotal [IP|DOMAIN]", "VirusTotal resolutions lookup by an ip or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def virustotal(indiactor)
-      with_error_handling do
-        run_analyzer Analyzers::VirusTotal, query: refang(indiactor), options: options
-      end
-    end
-
-    desc "securitytrails [IP|DOMAIN|EMAIL]", "SecurityTrails lookup by an ip, domain or email"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def securitytrails(indiactor)
-      with_error_handling do
-        run_analyzer Analyzers::SecurityTrails, query: refang(indiactor), options: options
-      end
-    end
-    map "st" => :securitytrails
-
-    desc "securitytrails_domain_feed [REGEXP]", "SecurityTrails new domain feed search by a regexp"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, default: "registered", desc: "A type of domain feed ('all', 'new' or 'registered')"
-    def securitytrails_domain_feed(regexp)
-      with_error_handling do
-        run_analyzer Analyzers::SecurityTrailsDomainFeed, query: regexp, options: options
-      end
-    end
-    map "st_domain_feed" => :securitytrails_domain_feed
-
-    desc "crtsh [QUERY]", "crt.sh search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :exclude_expired, type: :boolean, desc: "exclude expired certificates"
-    def crtsh(query)
-      with_error_handling do
-        run_analyzer Analyzers::Crtsh, query: query, options: options
-      end
-    end
-
-    desc "dnpedia [QUERY]", "DNPedia domain search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def dnpedia(query)
-      with_error_handling do
-        run_analyzer Analyzers::DNPedia, query: query, options: options
-      end
-    end
-
-    desc "circl [DOMAIN|SHA1]", "CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def circl(query)
-      with_error_handling do
-        run_analyzer Analyzers::CIRCL, query: refang(query), options: options
-      end
-    end
-
-    desc "passivetotal [IP|DOMAIN|EMAIL|SHA1]", "PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def passivetotal(indicator)
-      with_error_handling do
-        run_analyzer Analyzers::PassiveTotal, query: refang(indicator), options: options
-      end
-    end
-
-    desc "zoomeye [QUERY]", "ZoomEye search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, desc: "type to search(host / web)", default: "host"
-    def zoomeye(query)
-      with_error_handling do
-        run_analyzer Analyzers::ZoomEye, query: query, options: options
-      end
-    end
-
-    desc "binaryedge [QUERY]", "BinaryEdge host search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def binaryedge(query)
-      with_error_handling do
-        run_analyzer Analyzers::BinaryEdge, query: query, options: options
-      end
-    end
-
-    desc "pulsedive [IP|DOMAIN]", "Pulsedive lookup by an ip or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def pulsedive(indiactor)
-      with_error_handling do
-        run_analyzer Analyzers::Pulsedive, query: refang(indiactor), options: options
-      end
-    end
-
-    desc "dnstwister [DOMAIN]", "dnstwister lookup by a domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def dnstwister(domain)
-      with_error_handling do
-        run_analyzer Analyzers::DNSTwister, query: refang(domain), options: options
-      end
-    end
-
-    desc "otx [IP|DOMAIN]", "OTX lookup by an IP or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def otx(domain)
-      with_error_handling do
-        run_analyzer Analyzers::OTX, query: refang(domain), options: options
-      end
-    end
-
-    desc "spyse [QUERY]", "Spyse search by a query"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :type, type: :string, desc: "type to search (ip or domain)", default: "doamin"
-    def spyse(query)
-      with_error_handling do
-        run_analyzer Analyzers::Spyse, query: query, options: options
-      end
-    end
-
-    desc "passive_dns [IP|DOMAIN]", "Cross search with passive DNS services by an ip or domain"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def passive_dns(query)
-      with_error_handling do
-        run_analyzer Analyzers::PassiveDNS, query: refang(query), options: options
-      end
-    end
-
-    desc "passive_ssl [SHA1]", "Cross search with passive SSL services by an SHA1 certificate fingerprint"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def passive_ssl(query)
-      with_error_handling do
-        run_analyzer Analyzers::PassiveSSL, query: query, options: options
-      end
-    end
-
-    desc "reverse_whois [EMAIL]", "Cross search with reverse whois services by an email"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def reverse_whois(query)
-      with_error_handling do
-        run_analyzer Analyzers::ReveseWhois, query: refang(query), options: options
-      end
-    end
-
-    desc "http_hash", "Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    method_option :md5, type: :string, desc: "MD5 hash"
-    method_option :sha256, type: :string, desc: "SHA256 hash"
-    method_option :mmh3, type: :numeric, desc: "MurmurHash3 hash"
-    method_option :html, type: :string, desc: "path to an HTML file"
-    def http_hash
-      with_error_handling do
-        run_analyzer Analyzers::HTTPHash, query: nil, options: options
-      end
-    end
-
-    desc "free_text [TEXT]", "Cross search with search engines by a free text"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def free_text(text)
-      with_error_handling do
-        run_analyzer Analyzers::FreeText, query: text, options: options
-      end
-    end
-
-    desc "ssh_fingerprint [FINGERPRINT]", "Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)"
-    method_option :title, type: :string, desc: "title"
-    method_option :description, type: :string, desc: "description"
-    method_option :tags, type: :array, desc: "tags"
-    def ssh_fingerprint(fingerprint)
-      with_error_handling do
-        run_analyzer Analyzers::SSHFingerprint, query: fingerprint, options: options
-      end
-    end
-
-    desc "import_from_json", "Give a JSON input via STDIN"
-    def import_from_json(input = nil)
-      with_error_handling do
-        json = input || $stdin.gets.chomp
-        raise ArgumentError, "Input not found: please give an input in a JSON format" unless json
-
-        json = parse_as_json(json)
-        raise ArgumentError, "Invalid input format: an input JSON data should have title, description and artifacts key" unless valid_json?(json)
-
-        title = json["title"]
-        description = json["description"]
-        artifacts = json["artifacts"]
-        tags = json["tags"] || []
-
-        basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
-        basic.run
-      end
-    end
-
-    desc "alerts", "Show the alerts on TheHive"
-    method_option :limit, type: :string, default: "5", desc: "Number of alerts to show (or 'all' to show all the alerts)"
-    method_option :title, type: :string, desc: "Title to filter"
-    method_option :source, type: :string, desc: "Source to filter"
-    method_option :tag, type: :string, desc: "Tag to filter"
-    def alerts
-      with_error_handling do
-        load_configuration
-
-        viewer = AlertViewer.new
-        alerts = viewer.list(limit: options["limit"], title: options["title"], source: options["source"], tag: options[:tag])
-        puts JSON.pretty_generate(alerts)
-      end
-    end
-
-    desc "status", "Show the current configuration status"
-    def status
-      with_error_handling do
-        load_configuration
-
-        puts JSON.pretty_generate(Status.check)
+    class_option :config, type: :string, desc: "Path to the config file"
+
+    class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not. Only affects with analyze commands."
+    class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not. Only affects with analyze commands."
+
+    include Mihari::Commands::BinaryEdge
+    include Mihari::Commands::Censys
+    include Mihari::Commands::CIRCL
+    include Mihari::Commands::Config
+    include Mihari::Commands::Crtsh
+    include Mihari::Commands::DNPedia
+    include Mihari::Commands::DNSTwister
+    include Mihari::Commands::FreeText
+    include Mihari::Commands::HTTPHash
+    include Mihari::Commands::JSON
+    include Mihari::Commands::Onyphe
+    include Mihari::Commands::OTX
+    include Mihari::Commands::PassiveDNS
+    include Mihari::Commands::PassiveSSL
+    include Mihari::Commands::PassiveTotal
+    include Mihari::Commands::Pulsedive
+    include Mihari::Commands::ReverseWhois
+    include Mihari::Commands::SecurityTrails
+    include Mihari::Commands::SecurityTrailsDomainFeed
+    include Mihari::Commands::Shodan
+    include Mihari::Commands::Spyse
+    include Mihari::Commands::SSHFingerprint
+    include Mihari::Commands::Urlscan
+    include Mihari::Commands::VirusTotal
+    include Mihari::Commands::Web
+    include Mihari::Commands::ZoomEye
+
+    class << self
+      def exit_on_failure?
+        true
       end
     end
 
     no_commands do
       def with_error_handling
         yield
-      rescue => e
+      rescue StandardError => e
         notifier = Notifiers::ExceptionNotifier.new
         notifier.notify e
       end
 
-      def parse_as_json(input)
-        JSON.parse input
-      rescue JSON::ParserError => _e
-        nil
-      end
-
       # @return [true, false]
       def valid_json?(json)
         %w[title description artifacts].all? { |key| json.key? key }
@@ -333,16 +99,22 @@ module Mihari
         options = normalize_options(options)
 
         analyzer = analyzer_class.new(query, **options)
+
+        analyzer.ignore_old_artifacts = options[:ignore_old_artifacts] || false
+        analyzer.ignore_threshold = options[:ignore_threshold] || 0
+
         analyzer.run
       end
 
       def symbolize_hash_keys(hash)
-        hash.map { |k, v| [k.to_sym, v] }.to_h
+        hash.transform_keys(&:to_sym)
       end
 
       def normalize_options(options)
         # Delete :config because it is not intended to use for running an analyzer
-        options.delete(:config)
+        [:config, :ignore_old_artifacts, :ignore_threshold].each do |ignore_key|
+          options.delete(ignore_key)
+        end
         options
       end