mihari 1.4.1 → 2.2.0

data/Rakefile

@@ -2,6 +2,7 @@
 
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "standard/rake"
 
 RSpec::Core::RakeTask.new(:spec)
 

data/bin/console

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require "bundler/setup"
 require "mihari"

data/build_frontend.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+CURRENT_DIR=${PWD}
+
+mkdir -p tmp
+cd tmp
+git clone https://github.com/ninoseki/mihari-frontend.git
+cd mihari-frontend
+npm install
+npm run build
+
+cp -r dist/* ${CURRENT_DIR}/lib/mihari/web/public
+
+rm -rf ${CURRENT_DIR}/tmp/mihari-frontend

data/docker/Dockerfile

@@ -1,5 +1,7 @@
-FROM ruby:2.7-alpine3.10
-RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
+FROM ruby:3.0.1-alpine3.13
+
+RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
+  && gem install pg mysql2 \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/mihari.git \
   && cd mihari \
@@ -10,4 +12,4 @@ RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
 
 ENTRYPOINT ["mihari"]
 
-CMD ["--help"]
+CMD ["--help"]

data/examples/ipinfo_hosted_domains.rb

@@ -34,7 +34,7 @@ module Mihari
         uri = URI("#{IPINFO_API_ENDPOINT}/domains/#{ip}?token=#{token}")
         res = uri.read
         json = JSON.parse(res)
-        json.dig("domains") || []
+        json["domains"] || []
       end
     end
   end

data/lib/mihari.rb

@@ -79,8 +79,8 @@ require "mihari/emitters/slack"
 require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
 
-require "mihari/alert_viewer"
-
 require "mihari/status"
 
+require "mihari/web/app"
+
 require "mihari/cli"

data/lib/mihari/analyzers/base.rb

@@ -8,6 +8,13 @@ module Mihari
       include Configurable
       include Retriable
 
+      attr_accessor :ignore_old_artifacts, :ignore_threshold
+
+      def initialize
+        @ignore_old_artifacts = false
+        @ignore_threshold = 0
+      end
+
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -61,7 +68,9 @@ module Mihari
 
       # @return [Array<Mihari::Artifact>]
       def unique_artifacts
-        @unique_artifacts ||= normalized_artifacts.select(&:unique?)
+        @unique_artifacts ||= normalized_artifacts.select do |artifact|
+          artifact.unique?(ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold)
+        end
       end
 
       def set_unique_artifacts

data/lib/mihari/analyzers/basic.rb

@@ -4,12 +4,11 @@ module Mihari
   module Analyzers
     class Basic < Base
       attr_accessor :title
-      attr_reader :description
-      attr_reader :artifacts
-      attr_reader :source
-      attr_reader :tags
+      attr_reader :description, :artifacts, :source, :tags
 
       def initialize(title:, description:, artifacts:, source:, tags: [])
+        super()
+
         @title = title
         @description = description
         @artifacts = artifacts

data/lib/mihari/analyzers/binaryedge.rb

@@ -5,10 +5,7 @@ require "binaryedge"
 module Mihari
   module Analyzers
     class BinaryEdge < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
+      attr_reader :title, :description, :query, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -24,7 +21,7 @@ module Mihari
         return [] unless results || results.empty?
 
         results.map do |result|
-          events = result.dig("events") || []
+          events = result["events"] || []
           events.map do |event|
             event.dig "target", "ip"
           end.compact
@@ -47,7 +44,7 @@ module Mihari
         responses = []
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
-          total = res.dig("total").to_i
+          total = res["total"].to_i
 
           responses << res
           break if total <= page * PAGE_SIZE
@@ -56,7 +53,7 @@ module Mihari
       end
 
       def config_keys
-        %w(binaryedge_api_key)
+        %w[binaryedge_api_key]
       end
 
       def api

data/lib/mihari/analyzers/censys.rb

@@ -5,11 +5,7 @@ require "censu"
 module Mihari
   module Analyzers
     class Censys < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
-      attr_reader :type
+      attr_reader :title, :description, :query, :tags, :type
 
       def initialize(query, title: nil, description: nil, tags: [], type: "ipv4")
         super()
@@ -37,7 +33,7 @@ module Mihari
       private
 
       def valid_type?
-        %w(ipv4 websites certificates).include? type
+        %w[ipv4 websites certificates].include? type
       end
 
       def normalize(domain)
@@ -86,7 +82,7 @@ module Mihari
       end
 
       def config_keys
-        %w(censys_id censys_secret)
+        %w[censys_id censys_secret]
       end
 
       def api

data/lib/mihari/analyzers/circl.rb

@@ -5,9 +5,7 @@ require "passive_circl"
 module Mihari
   module Analyzers
     class CIRCL < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -27,7 +25,7 @@ module Mihari
       private
 
       def config_keys
-        %w(circl_passive_password circl_passive_username)
+        %w[circl_passive_password circl_passive_username]
       end
 
       def api
@@ -41,21 +39,21 @@ module Mihari
         when "hash"
           passive_ssl_lookup
         else
-          raise InvalidInputError, "#{@query}(type: #{@type || 'unknown'}) is not supported."
+          raise InvalidInputError, "#{@query}(type: #{@type || "unknown"}) is not supported."
         end
       end
 
       def passive_dns_lookup
         results = api.dns.query(@query)
         results.map do |result|
-          type = result.dig("rrtype")
-          type == "A" ? result.dig("rdata") : nil
+          type = result["rrtype"]
+          type == "A" ? result["rdata"] : nil
         end.compact.uniq
       end
 
       def passive_ssl_lookup
         result = api.ssl.cquery(@query)
-        seen = result.dig("seen") || []
+        seen = result["seen"] || []
         seen.uniq
       end
     end

data/lib/mihari/analyzers/crtsh.rb

@@ -5,11 +5,7 @@ require "crtsh"
 module Mihari
   module Analyzers
     class Crtsh < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
-      attr_reader :exclude_expired
+      attr_reader :title, :description, :query, :tags, :exclude_expired
 
       def initialize(query, title: nil, description: nil, tags: [], exclude_expired: nil)
         super()
@@ -24,7 +20,7 @@ module Mihari
 
       def artifacts
         results = search
-        name_values = results.map { |result| result.dig("name_value") }.compact
+        name_values = results.map { |result| result["name_value"] }.compact
         name_values.map(&:lines).flatten.uniq.map(&:chomp)
       end
 

data/lib/mihari/analyzers/dnpedia.rb

@@ -5,10 +5,7 @@ require "dnpedia"
 module Mihari
   module Analyzers
     class DNPedia < Base
-      attr_reader :query
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -31,9 +28,9 @@ module Mihari
 
       def lookup
         res = api.search(query)
-        rows = res.dig("rows") || []
+        rows = res["rows"] || []
         rows.map do |row|
-          [row.dig("name"), row.dig("zoneid")].join(".")
+          [row["name"], row["zoneid"]].join(".")
         end
       end
     end

data/lib/mihari/analyzers/dnstwister.rb

@@ -7,12 +7,7 @@ require "parallel"
 module Mihari
   module Analyzers
     class DNSTwister < Base
-      attr_reader :query
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :type, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -47,11 +42,11 @@ module Mihari
       end
 
       def lookup
-        raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         res = api.fuzz(query)
-        fuzzy_domains = res.dig("fuzzy_domains") || []
-        domains = fuzzy_domains.map { |domain| domain.dig("domain") }
+        fuzzy_domains = res["fuzzy_domains"] || []
+        domains = fuzzy_domains.map { |domain| domain["domain"] }
         Parallel.map(domains) do |domain|
           resolvable?(domain) ? domain : nil
         end.compact

data/lib/mihari/analyzers/free_text.rb

@@ -5,15 +5,11 @@ require "parallel"
 module Mihari
   module Analyzers
     class FreeText < Base
-      attr_reader :query
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :title, :description, :tags
 
       ANALYZERS = [
         Mihari::Analyzers::BinaryEdge,
-        Mihari::Analyzers::Censys,
+        Mihari::Analyzers::Censys
       ].freeze
 
       def initialize(query, title: nil, description: nil, tags: [])

data/lib/mihari/analyzers/http_hash.rb

@@ -5,15 +5,7 @@ require "parallel"
 module Mihari
   module Analyzers
     class HTTPHash < Base
-      attr_reader :md5
-      attr_reader :sha256
-      attr_reader :mmh3
-
-      attr_reader :html
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :md5, :sha256, :mmh3, :html, :title, :description, :tags
 
       def initialize(_query, md5: nil, sha256: nil, mmh3: nil, html: nil, title: nil, description: nil, tags: [])
         super()
@@ -57,7 +49,7 @@ module Mihari
         [
           md5 ? "md5:#{md5}" : nil,
           sha256 ? "sha256:#{sha256}" : nil,
-          mmh3 ? "mmh3:#{mmh3}" : nil,
+          mmh3 ? "mmh3:#{mmh3}" : nil
         ].compact.join(",")
       end
 
@@ -92,7 +84,7 @@ module Mihari
           binary_edge,
           censys,
           onyphe,
-          shodan,
+          shodan
         ].compact
       end
 

data/lib/mihari/analyzers/onyphe.rb

@@ -5,10 +5,7 @@ require "onyphe"
 module Mihari
   module Analyzers
     class Onyphe < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
+      attr_reader :title, :description, :query, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -24,10 +21,10 @@ module Mihari
         return [] unless results
 
         flat_results = results.map do |result|
-          result.dig("results")
+          result["results"]
         end.flatten.compact
 
-        flat_results.map { |result| result.dig("ip") }.compact.uniq
+        flat_results.map { |result| result["ip"] }.compact.uniq
       end
 
       private
@@ -35,7 +32,7 @@ module Mihari
       PAGE_SIZE = 10
 
       def config_keys
-        %w(onyphe_api_key)
+        %w[onyphe_api_key]
       end
 
       def api
@@ -51,7 +48,7 @@ module Mihari
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
           responses << res
-          total = res.dig("total").to_i
+          total = res["total"].to_i
           break if total <= page * PAGE_SIZE
         end
         responses

data/lib/mihari/analyzers/otx.rb

@@ -5,12 +5,7 @@ require "otx_ruby"
 module Mihari
   module Analyzers
     class OTX < Base
-      attr_reader :query
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :type, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -30,7 +25,7 @@ module Mihari
       private
 
       def config_keys
-        %w(otx_api_key)
+        %w[otx_api_key]
       end
 
       def domain_client
@@ -42,7 +37,7 @@ module Mihari
       end
 
       def valid_type?
-        %w(ip domain).include? type
+        %w[ip domain].include? type
       end
 
       def lookup
@@ -52,7 +47,7 @@ module Mihari
         when "ip"
           ip_lookup
         else
-          raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end