mihari 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dff04f8065ceff1c6b9fa68bd606db9cc4789ed034a330019143a69c4ec2c037
-  data.tar.gz: d307e3e48eedf5d17245da14a20bd578468e03a70b2ba302fcc4ea68aab5651c
+  metadata.gz: c5859ab6b5d161359ea07c4af14de087013befa3b51d91844a60127137882e28
+  data.tar.gz: 481f34619d14e04f3187a9d2c3a05b3a1d109c7809622accebc47b100a506bc4
 SHA512:
-  metadata.gz: 97f8237d491778739e307e15d0b5b96fba510a548a0a42ae1760e41469cef7fad551b6e876025d7daed58e449c77bee6d3573d659c85e4b2886712229e2ee7ef
-  data.tar.gz: 2d5f9125f2039ce7d8727661a8f71f1a7c0b9021a0eaf613242598371fb6066234fa5a8959528190bc7275b35da026932700d13929cfd9f94aa04eded6881951
+  metadata.gz: 13693510f6f7d3560d207bb97dec50bb4851514e3b35918a82924e4e73d18e4bdcf963c3cd8928cd17559187b69419c8aa2a8e11e9d857965473dabfdf12ac59
+  data.tar.gz: 8786ad5973687848a89c8737eb92e52dd636e912b6af0ddf339dccccbff28c42f55e4111d1967b1d9a7b8aca07618788e82d98c24a56ca54102b36c84231d21e

data/.rubocop.yml

@@ -4,6 +4,9 @@
 require:
   - rubocop-performance
 
+AllCops:
+  NewCops: enable
+
 Style/Alias:
   Enabled: false
   StyleGuide: https://relaxed.ruby.style/#stylealias
@@ -151,5 +154,8 @@ Lint/AssignmentInCondition:
 Layout/LineLength:
   Enabled: false
 
+Style/StringLiteralsInInterpolation:
+  Enabled: false
+
 Metrics:
   Enabled: false

data/README.md

@@ -12,6 +12,8 @@ Mihari is a framework for continuous OSINT based threat hunting.
 
 ## How it works
 
+![img](https://github.com/ninoseki/mihari/raw/master/images/overview.png)
+
 - Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs or hashes).
 - Mihari checks whether a DB (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:

data/bin/console

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require "bundler/setup"
 require "mihari"

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.0.0-alpine3.13
+FROM ruby:3.0.1-alpine3.13
 
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
   && gem install pg mysql2 \

data/examples/ipinfo_hosted_domains.rb

@@ -34,7 +34,7 @@ module Mihari
         uri = URI("#{IPINFO_API_ENDPOINT}/domains/#{ip}?token=#{token}")
         res = uri.read
         json = JSON.parse(res)
-        json.dig("domains") || []
+        json["domains"] || []
       end
     end
   end

data/lib/mihari/analyzers/base.rb

@@ -8,6 +8,13 @@ module Mihari
       include Configurable
       include Retriable
 
+      attr_accessor :ignore_old_artifacts, :ignore_threshold
+
+      def initialize
+        @ignore_old_artifacts = false
+        @ignore_threshold = 0
+      end
+
       # @return [Array<String>, Array<Mihari::Artifact>]
       def artifacts
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
@@ -61,7 +68,9 @@ module Mihari
 
       # @return [Array<Mihari::Artifact>]
       def unique_artifacts
-        @unique_artifacts ||= normalized_artifacts.select(&:unique?)
+        @unique_artifacts ||= normalized_artifacts.select do |artifact|
+          artifact.unique?(ignore_old_artifacts: ignore_old_artifacts, ignore_threshold: ignore_threshold)
+        end
       end
 
       def set_unique_artifacts

data/lib/mihari/analyzers/circl.rb

@@ -46,14 +46,14 @@ module Mihari
       def passive_dns_lookup
         results = api.dns.query(@query)
         results.map do |result|
-          type = result.dig("rrtype")
-          type == "A" ? result.dig("rdata") : nil
+          type = result["rrtype"]
+          type == "A" ? result["rdata"] : nil
         end.compact.uniq
       end
 
       def passive_ssl_lookup
         result = api.ssl.cquery(@query)
-        seen = result.dig("seen") || []
+        seen = result["seen"] || []
         seen.uniq
       end
     end

data/lib/mihari/analyzers/onyphe.rb

@@ -21,10 +21,10 @@ module Mihari
         return [] unless results
 
         flat_results = results.map do |result|
-          result.dig("results")
+          result["results"]
         end.flatten.compact
 
-        flat_results.map { |result| result.dig("ip") }.compact.uniq
+        flat_results.map { |result| result["ip"] }.compact.uniq
       end
 
       private

data/lib/mihari/cli.rb

@@ -33,7 +33,10 @@ require "mihari/commands/web"
 
 module Mihari
   class CLI < Thor
-    class_option :config, type: :string, desc: "path to the config file"
+    class_option :config, type: :string, desc: "Path to the config file"
+
+    class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not. Only affects with analyze commands."
+    class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not. Only affects with analyze commands."
 
     include Mihari::Commands::BinaryEdge
     include Mihari::Commands::Censys
@@ -96,11 +99,15 @@ module Mihari
         options = normalize_options(options)
 
         analyzer = analyzer_class.new(query, **options)
+
+        analyzer.ignore_old_artifacts = options["ignore_old_artifacts"] || false
+        analyzer.ignore_threshold = options["ignore_threshold"] || 0
+
         analyzer.run
       end
 
       def symbolize_hash_keys(hash)
-        hash.map { |k, v| [k.to_sym, v] }.to_h
+        hash.transform_keys(&:to_sym)
       end
 
       def normalize_options(options)

data/lib/mihari/commands/config.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require "colorize"
+
 module Mihari
   module Commands
     module Config
@@ -8,11 +12,13 @@ module Mihari
           def init_config
             filename = options["filename"]
 
-            if File.exist?(filename) && !(yes? "#{filename} exists. Do you want to overwrite it? (y/n)")
+            warning = "#{filename} exists. Do you want to overwrite it? (y/n)"
+            if File.exist?(filename) && !(yes? warning)
               return
             end
 
             Mihari::Config.initialize_yaml filename
+            puts "The config file is initialized as #{filename}.".colorize(:blue)
           end
         end
       end

data/lib/mihari/commands/dnstwister.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module DNSTwister

data/lib/mihari/commands/json.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module JSON
@@ -18,6 +20,10 @@ module Mihari
               tags = json["tags"] || []
 
               basic = Analyzers::Basic.new(title: title, description: description, artifacts: artifacts, source: "json", tags: tags)
+
+              basic.ignore_old_artifacts = options["ignore_old_artifacts"] || false
+              basic.ignore_threshold = options["ignore_threshold"] || 0
+
               basic.run
             end
           end

data/lib/mihari/commands/onyphe.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module Onyphe

data/lib/mihari/commands/passive_dns.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module PassiveDNS

data/lib/mihari/commands/securitytrails.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module SecurityTrails

data/lib/mihari/commands/shodan.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module Shodan

data/lib/mihari/commands/spyse.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module Spyse

data/lib/mihari/commands/urlscan.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module Urlscan

data/lib/mihari/commands/virustotal.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module VirusTotal

data/lib/mihari/commands/web.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module Web

data/lib/mihari/commands/zoomeye.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Commands
     module ZoomEye

data/lib/mihari/config.rb

@@ -58,7 +58,7 @@ module Mihari
 
       def initialize_yaml(filename)
         keys = new.instance_variables.map do |key|
-          key.to_s[1..-1]
+          key.to_s[1..]
         end
 
         config = keys.map do |key|

data/lib/mihari/models/artifact.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require "active_record"
+require "active_record/filter"
+require "active_support/core_ext/integer/time"
+require "active_support/core_ext/numeric/time"
 
 class ArtifactValidator < ActiveModel::Validator
   def validate(record)
@@ -20,8 +23,16 @@ module Mihari
       self.data_type = TypeChecker.type(data)
     end
 
-    def unique?
-      self.class.find_by(data: data).nil?
+    def unique?(ignore_old_artifacts: false, ignore_threshold: 0)
+      artifact = self.class.where(data: data).order(created_at: :desc).first
+      return true if artifact.nil?
+
+      return false unless ignore_old_artifacts
+
+      days_before = (-ignore_threshold).days.from_now
+      # if an artifact is created before {ignore_threshold} days, ignore it
+      #                           within {ignore_threshold} days, do not ignore it
+      artifact.created_at < days_before
     end
   end
 end

data/lib/mihari/status.rb

@@ -3,7 +3,7 @@
 module Mihari
   class Status
     def check
-      statuses.transform_values { |value| convert(**value) }
+      statuses
     end
 
     def self.check
@@ -12,14 +12,6 @@ module Mihari
 
     private
 
-    def convert(is_configured:, values:, type:)
-      {
-        is_configured: is_configured,
-        values: values,
-        type: type
-      }
-    end
-
     def statuses
       (Mihari.analyzers + Mihari.emitters).map do |klass|
         name = klass.to_s.split("::").last.to_s

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "2.1.0"
+  VERSION = "2.2.0"
 end

data/lib/mihari/web/app.rb

@@ -1,16 +1,21 @@
-require "awrence"
-require "colorize"
+# frozen_string_literal: true
+
 require "launchy"
 require "rack"
-require "safe_shell"
+require "rack/handler/puma"
 require "sinatra"
-require "sinatra/json"
-require "sinatra/reloader"
+
+require "mihari/web/helpers/json"
+
+require "mihari/web/controllers/alerts_controller"
+require "mihari/web/controllers/artifacts_controller"
+require "mihari/web/controllers/command_controller"
+require "mihari/web/controllers/config_controller"
+require "mihari/web/controllers/sources_controller"
+require "mihari/web/controllers/tags_controller"
 
 module Mihari
   class App < Sinatra::Base
-    register Sinatra::Reloader
-
     set :root, File.dirname(__FILE__)
     set :public_folder, File.join(root, "public")
 
@@ -18,140 +23,18 @@ module Mihari
       send_file File.join(settings.public_folder, "index.html")
     end
 
-    get "/api/alerts" do
-      page = params["page"] || 1
-      page = page.to_i
-      limit = 10
-
-      artifact_data = params["artifact"]
-      description = params["description"]
-      source = params["source"]
-      tag_name = params["tag"]
-      title = params["title"]
-
-      from_at = params["from_at"] || params["fromAt"]
-      from_at = DateTime.parse(from_at) if from_at
-      to_at = params["to_at"] || params["toAt"]
-      to_at = DateTime.parse(to_at) if to_at
-
-      alerts = Mihari::Alert.search(
-        artifact_data: artifact_data,
-        description: description,
-        from_at: from_at,
-        limit: limit,
-        page: page,
-        source: source,
-        tag_name: tag_name,
-        title: title,
-        to_at: to_at
-      )
-      total = Mihari::Alert.count(
-        artifact_data: artifact_data,
-        description: description,
-        from_at: from_at,
-        source: source,
-        tag_name: tag_name,
-        title: title,
-        to_at: to_at
-      )
-
-      json = { alerts: alerts, total: total, current_page: page, page_size: limit }
-      json json.to_camelback_keys
-    end
-
-    delete "/api/alerts/:id" do
-      id = params["id"]
-      id = id.to_i
-
-      begin
-        alert = Mihari::Alert.find(id)
-        alert.destroy
-
-        status 204
-        body ""
-      rescue ActiveRecord::RecordNotFound
-        status 404
-
-        message = { message: "ID:#{id} is not found" }
-        json message
-      end
-    end
-
-    delete "/api/artifacts/:id" do
-      id = params["id"]
-      id = id.to_i
-
-      begin
-        alert = Mihari::Artifact.find(id)
-        alert.delete
-
-        status 204
-        body ""
-      rescue ActiveRecord::RecordNotFound
-        status 404
-
-        message = { message: "ID:#{id} is not found" }
-        json message
-      end
-    end
-
-    delete "/api/tags/:name" do
-      name = params["name"]
-
-      begin
-        Mihari::Tag.where(name: name).destroy_all
-
-        status 204
-        body ""
-      rescue ActiveRecord::RecordNotFound
-        status 404
-
-        message = { message: "Name:#{name} is not found" }
-        json message
-      end
-    end
-
-    get "/api/sources" do
-      tags = Mihari::Alert.distinct.pluck(:source)
-      json tags
-    end
-
-    get "/api/tags" do
-      tags = Mihari::Tag.distinct.pluck(:name)
-      json tags
-    end
-
-    get "/api/config" do
-      report = Status.check
-
-      json report.to_camelback_keys
-    end
-
-    post "/api/command" do
-      payload = JSON.parse(request.body.read)
-
-      command = payload["command"]
-      if command.nil?
-        status 400
-        return json( { message: "command is required" })
-      end
-
-      command = command.split
-
-      output = SafeShell.execute("mihari", *command)
-      success = $?.success?
-
-      json({ output: output, success: success })
-    end
+    use Mihari::Controllers::AlertsController
+    use Mihari::Controllers::ArtifactsController
+    use Mihari::Controllers::CommandController
+    use Mihari::Controllers::ConfigController
+    use Mihari::Controllers::SourcesController
+    use Mihari::Controllers::TagsController
 
     class << self
       def run!(port: 9292, host: "localhost")
-        url =  "http://#{host}:#{port}"
-
-        puts "The app will be available at #{url}.".colorize(:blue)
-        puts "(Press Ctrl+C to quit)".colorize(:blue)
+        url = "http://#{host}:#{port}"
 
-        Rack::Handler::WEBrick.run self, Port: port, Host: host do |server|
+        Rack::Handler::Puma.run self, Port: port, Host: host do |server|
           Launchy.open url
 
           [:INT, :TERM].each do |sig|