mihari 2.2.0 → 3.0.0

data/lib/mihari/analyzers/securitytrails_domain_feed.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require "securitytrails"
-
-module Mihari
-  module Analyzers
-    class SecurityTrailsDomainFeed < Base
-      attr_reader :type, :title, :description, :tags
-
-      def initialize(regexp, type: "registered", title: nil, description: nil, tags: [])
-        super()
-
-        @_regexp = regexp
-        @type = type
-
-        raise InvalidInputError, "#{@_regexp} is not a valid regexp" unless regexp
-        raise InvalidInputError, "#{type} is not a valid type" unless valid_type?
-
-        @title = title || "SecurityTrails domain feed lookup"
-        @description = description || "Regexp = /#{@_regexp}/"
-        @tags = tags
-      end
-
-      def artifacts
-        lookup || []
-      end
-
-      private
-
-      def config_keys
-        %w[securitytrails_api_key]
-      end
-
-      def api
-        @api ||= ::SecurityTrails::API.new(Mihari.config.securitytrails_api_key)
-      end
-
-      def valid_type?
-        %w[all new registered].include? type
-      end
-
-      def regexp
-        @regexp ||= Regexp.compile(@_regexp)
-      rescue InvalidInputError => _e
-        nil
-      end
-
-      def lookup
-        new_domains.select do |domain|
-          regexp.match? domain
-        end
-      end
-
-      def new_domains
-        api.feeds.domains type
-      end
-    end
-  end
-end

data/lib/mihari/analyzers/ssh_fingerprint.rb

@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-require "parallel"
-
-module Mihari
-  module Analyzers
-    class SSHFingerprint < Base
-      attr_reader :fingerprint, :title, :description, :tags
-
-      def initialize(fingerprint, title: nil, description: nil, tags: [])
-        super()
-
-        @fingerprint = fingerprint
-
-        @title = title || "SSH fingerprint cross search"
-        @description = description || "fingerprint = #{fingerprint}"
-        @tags = tags
-      end
-
-      def artifacts
-        Parallel.map(analyzers) do |analyzer|
-          run_analyzer analyzer
-        end.flatten
-      end
-
-      private
-
-      def valid_fingerprint?
-        /^([0-9a-f]{2}:){15}[0-9a-f]{2}$/.match? fingerprint
-      end
-
-      def binary_edge
-        BinaryEdge.new "ssh.fingerprint:\"#{fingerprint}\""
-      end
-
-      def shodan
-        Shodan.new fingerprint
-      end
-
-      def analyzers
-        raise InvalidInputError, "Invalid fingerprint is given." unless valid_fingerprint?
-
-        [
-          binary_edge,
-          shodan
-        ].compact
-      end
-
-      def run_analyzer(analyzer)
-        analyzer.artifacts
-      rescue ArgumentError, InvalidInputError => _e
-        nil
-      rescue ::BinaryEdge::Error, ::Shodan::Error => _e
-        nil
-      end
-    end
-  end
-end

data/lib/mihari/cli.rb

@@ -1,124 +0,0 @@
-# frozen_string_literal: true
-
-require "thor"
-
-require "mihari/commands/binaryedge"
-require "mihari/commands/censys"
-require "mihari/commands/circl"
-require "mihari/commands/crtsh"
-require "mihari/commands/dnpedia"
-require "mihari/commands/dnstwister"
-require "mihari/commands/onyphe"
-require "mihari/commands/otx"
-require "mihari/commands/passivetotal"
-require "mihari/commands/pulsedive"
-require "mihari/commands/securitytrails_domain_feed"
-require "mihari/commands/securitytrails"
-require "mihari/commands/shodan"
-require "mihari/commands/spyse"
-require "mihari/commands/urlscan"
-require "mihari/commands/virustotal"
-require "mihari/commands/zoomeye"
-
-require "mihari/commands/free_text"
-require "mihari/commands/http_hash"
-require "mihari/commands/passive_dns"
-require "mihari/commands/passive_ssl"
-require "mihari/commands/reverse_whois"
-require "mihari/commands/ssh_fingerprint"
-
-require "mihari/commands/config"
-require "mihari/commands/json"
-require "mihari/commands/web"
-
-module Mihari
-  class CLI < Thor
-    class_option :config, type: :string, desc: "Path to the config file"
-
-    class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not. Only affects with analyze commands."
-    class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not. Only affects with analyze commands."
-
-    include Mihari::Commands::BinaryEdge
-    include Mihari::Commands::Censys
-    include Mihari::Commands::CIRCL
-    include Mihari::Commands::Config
-    include Mihari::Commands::Crtsh
-    include Mihari::Commands::DNPedia
-    include Mihari::Commands::DNSTwister
-    include Mihari::Commands::FreeText
-    include Mihari::Commands::HTTPHash
-    include Mihari::Commands::JSON
-    include Mihari::Commands::Onyphe
-    include Mihari::Commands::OTX
-    include Mihari::Commands::PassiveDNS
-    include Mihari::Commands::PassiveSSL
-    include Mihari::Commands::PassiveTotal
-    include Mihari::Commands::Pulsedive
-    include Mihari::Commands::ReverseWhois
-    include Mihari::Commands::SecurityTrails
-    include Mihari::Commands::SecurityTrailsDomainFeed
-    include Mihari::Commands::Shodan
-    include Mihari::Commands::Spyse
-    include Mihari::Commands::SSHFingerprint
-    include Mihari::Commands::Urlscan
-    include Mihari::Commands::VirusTotal
-    include Mihari::Commands::Web
-    include Mihari::Commands::ZoomEye
-
-    class << self
-      def exit_on_failure?
-        true
-      end
-    end
-
-    no_commands do
-      def with_error_handling
-        yield
-      rescue StandardError => e
-        notifier = Notifiers::ExceptionNotifier.new
-        notifier.notify e
-      end
-
-      # @return [true, false]
-      def valid_json?(json)
-        %w[title description artifacts].all? { |key| json.key? key }
-      end
-
-      def load_configuration
-        config = options["config"]
-        return unless config
-
-        Config.load_from_yaml(config)
-        Database.connect
-      end
-
-      def run_analyzer(analyzer_class, query:, options:)
-        load_configuration
-
-        options = symbolize_hash_keys(options)
-        options = normalize_options(options)
-
-        analyzer = analyzer_class.new(query, **options)
-
-        analyzer.ignore_old_artifacts = options["ignore_old_artifacts"] || false
-        analyzer.ignore_threshold = options["ignore_threshold"] || 0
-
-        analyzer.run
-      end
-
-      def symbolize_hash_keys(hash)
-        hash.transform_keys(&:to_sym)
-      end
-
-      def normalize_options(options)
-        # Delete :config because it is not intended to use for running an analyzer
-        options.delete(:config)
-        options
-      end
-
-      def refang(indicator)
-        indicator.gsub("[.]", ".").gsub("(.)", ".")
-      end
-    end
-  end
-end

data/lib/mihari/commands/config.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require "colorize"
-
-module Mihari
-  module Commands
-    module Config
-      def self.included(thor)
-        thor.class_eval do
-          desc "init_config", "Create a config file"
-          method_option :filename, type: :string, default: "mihari.yml"
-          def init_config
-            filename = options["filename"]
-
-            warning = "#{filename} exists. Do you want to overwrite it? (y/n)"
-            if File.exist?(filename) && !(yes? warning)
-              return
-            end
-
-            Mihari::Config.initialize_yaml filename
-            puts "The config file is initialized as #{filename}.".colorize(:blue)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/free_text.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module FreeText
-      def self.included(thor)
-        thor.class_eval do
-          desc "free_text [TEXT]", "Cross search with search engines by a free text"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def free_text(text)
-            with_error_handling do
-              run_analyzer Analyzers::FreeText, query: text, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/http_hash.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module HTTPHash
-      def self.included(thor)
-        thor.class_eval do
-          desc "http_hash", "Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          method_option :md5, type: :string, desc: "MD5 hash"
-          method_option :sha256, type: :string, desc: "SHA256 hash"
-          method_option :mmh3, type: :numeric, desc: "MurmurHash3 hash"
-          method_option :html, type: :string, desc: "path to an HTML file"
-          def http_hash
-            with_error_handling do
-              run_analyzer Analyzers::HTTPHash, query: nil, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/passive_dns.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module PassiveDNS
-      def self.included(thor)
-        thor.class_eval do
-          desc "passive_dns [IP|DOMAIN]", "Cross search with passive DNS services by an ip or domain"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def passive_dns(query)
-            with_error_handling do
-              run_analyzer Analyzers::PassiveDNS, query: refang(query), options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/passive_ssl.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module PassiveSSL
-      def self.included(thor)
-        thor.class_eval do
-          desc "passive_ssl [SHA1]", "Cross search with passive SSL services by an SHA1 certificate fingerprint"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def passive_ssl(query)
-            with_error_handling do
-              run_analyzer Analyzers::PassiveSSL, query: query, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/reverse_whois.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module ReverseWhois
-      def self.included(thor)
-        thor.class_eval do
-          desc "reverse_whois [EMAIL]", "Cross search with reverse whois services by an email"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def reverse_whois(query)
-            with_error_handling do
-              run_analyzer Analyzers::ReveseWhois, query: refang(query), options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/securitytrails_domain_feed.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module SecurityTrailsDomainFeed
-      def self.included(thor)
-        thor.class_eval do
-          desc "securitytrails_domain_feed [REGEXP]", "SecurityTrails new domain feed search by a regexp"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          method_option :type, type: :string, default: "registered", desc: "A type of domain feed ('all', 'new' or 'registered')"
-          def securitytrails_domain_feed(regexp)
-            with_error_handling do
-              run_analyzer Analyzers::SecurityTrailsDomainFeed, query: regexp, options: options
-            end
-          end
-          map "st_domain_feed" => :securitytrails_domain_feedd
-        end
-      end
-    end
-  end
-end

data/lib/mihari/commands/ssh_fingerprint.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Mihari
-  module Commands
-    module SSHFingerprint
-      def self.included(thor)
-        thor.class_eval do
-          desc "ssh_fingerprint [FINGERPRINT]", "Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)"
-          method_option :title, type: :string, desc: "title"
-          method_option :description, type: :string, desc: "description"
-          method_option :tags, type: :array, desc: "tags"
-          def ssh_fingerprint(fingerprint)
-            with_error_handling do
-              run_analyzer Analyzers::SSHFingerprint, query: fingerprint, options: options
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mihari/config.rb

@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-require "yaml"
-
-module Mihari
-  class Config
-    attr_accessor :binaryedge_api_key, :censys_id, :censys_secret, :circl_passive_password, :circl_passive_username, :misp_api_endpoint, :misp_api_key, :onyphe_api_key, :otx_api_key, :passivetotal_api_key, :passivetotal_username, :pulsedive_api_key, :securitytrails_api_key, :shodan_api_key, :slack_channel, :slack_webhook_url, :spyse_api_key, :thehive_api_endpoint, :thehive_api_key, :urlscan_api_key, :virustotal_api_key, :zoomeye_password, :zoomeye_username, :database
-
-    def initialize
-      load_from_env
-    end
-
-    def load_from_env
-      @binaryedge_api_key = ENV["BINARYEDGE_API_KEY"]
-      @censys_id = ENV["CENSYS_ID"]
-      @censys_secret = ENV["CENSYS_SECRET"]
-      @circl_passive_password = ENV["CIRCL_PASSIVE_PASSWORD"]
-      @circl_passive_username = ENV["CIRCL_PASSIVE_USERNAME"]
-      @misp_api_endpoint = ENV["MISP_API_ENDPOINT"]
-      @misp_api_key = ENV["MISP_API_KEY"]
-      @onyphe_api_key = ENV["ONYPHE_API_KEY"]
-      @otx_api_key = ENV["OTX_API_KEY"]
-      @passivetotal_api_key = ENV["PASSIVETOTAL_API_KEY"]
-      @passivetotal_username = ENV["PASSIVETOTAL_USERNAME"]
-      @pulsedive_api_key = ENV["PULSEDIVE_API_KEY"]
-      @securitytrails_api_key = ENV["SECURITYTRAILS_API_KEY"]
-      @shodan_api_key = ENV["SHODAN_API_KEY"]
-      @slack_channel = ENV["SLACK_CHANNEL"]
-      @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
-      @spyse_api_key = ENV["SPYSE_API_KEY"]
-      @thehive_api_endpoint = ENV["THEHIVE_API_ENDPOINT"]
-      @thehive_api_key = ENV["THEHIVE_API_KEY"]
-      @urlscan_api_key = ENV["URLSCAN_API_KEY"]
-      @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]
-      @zoomeye_password = ENV["ZOOMEYE_PASSWORD"]
-      @zoomeye_username = ENV["ZOOMEYE_USERNAME"]
-
-      @database = ENV["DATABASE"] || "mihari.db"
-    end
-
-    class << self
-      def load_from_yaml(path)
-        raise ArgumentError, "#{path} does not exist." unless File.exist?(path)
-
-        data = File.read(path)
-        begin
-          yaml = YAML.safe_load(data)
-        rescue TypeError => _e
-          return
-        end
-
-        Mihari.configure do |config|
-          yaml.each do |key, value|
-            config.send("#{key.downcase}=".to_sym, value)
-          end
-        end
-      end
-
-      def initialize_yaml(filename)
-        keys = new.instance_variables.map do |key|
-          key.to_s[1..]
-        end
-
-        config = keys.map do |key|
-          [key, nil]
-        end.to_h
-
-        YAML.dump(config, File.open(filename, "w"))
-      end
-    end
-  end
-
-  class << self
-    def config
-      @config ||= Config.new
-    end
-
-    attr_writer :config
-
-    def configure
-      yield config
-    end
-  end
-end