mihari 3.10.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4d53239fe8be60cdc674dc22c14f801147a71144649cec83f4fd3caaf2c99fa
-  data.tar.gz: 71b9b3746164b1f3eec9b98a807a6940e85a9184991efd7dd9abe5bd9b3f3578
+  metadata.gz: 1a42a8897840d2f2268f88c6734c1633642256ef08667c39f45f5d03ac13874f
+  data.tar.gz: 89bdd5aa38f833e3158464915ec46c05e8c0bb6f17d34fe397d1590c59a57d7c
 SHA512:
-  metadata.gz: 6bee10443bd55e0ac483e8495c508bcce95dc56073aeb99f6c1566f38fb73707140baa838c07212c39795017d109aef8e9478f560bc3d04a21cbb949a48c9b61
-  data.tar.gz: c35f757168a737c37dcbf9a54d0408eabdfbf807a7a57db77c59c10f9a5b803d6731c615f3360db93b05647ee5e6cdcd35518baf4a7a7efe8d73e198d7f2664e
+  metadata.gz: f62927fdb96eabffd99106bf03178e6eb573ee5e58b8fa2bb94b6b6b2ea898324fd65656bbb128aab67196ad03da959603dd73108c8840e883560d673c8afa5f
+  data.tar.gz: 2e752fa7ab93691ad6c38ec534af3bab14e27f2b66127dc00fc016f26554ea29fc2c743d2bdaef93d85e816d8a47e54d2d41bb6dafaabf711cfb2c833c3ff0ff

data/README.md

@@ -2,7 +2,6 @@
 
 [![Gem Version](https://badge.fury.io/rb/mihari.svg)](https://badge.fury.io/rb/mihari)
 [![Ruby CI](https://github.com/ninoseki/mihari/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/test.yml)
-[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/mihari)](https://hub.docker.com/r/ninoseki/mihari)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 
@@ -38,6 +37,7 @@ Mihari supports the following services by default.
 - [crt.sh](https://crt.sh/)
 - [DN Pedia](https://dnpedia.com/)
 - [dnstwister](https://dnstwister.report/)
+- [GreyNoise](https://www.greynoise.io/)
 - [Onyphe](https://onyphe.io)
 - [OTX](https://otx.alienvault.com/)
 - [PassiveTotal](https://community.riskiq.com/)

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.0.2-alpine3.13
+FROM ruby:3.0.3-alpine3.13
 
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
   && gem install pg mysql2 \

data/lib/mihari/analyzers/base.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "dry-initializer"
-require "parallel"
-
 module Mihari
   module Analyzers
     class Base
@@ -10,6 +7,7 @@ module Mihari
 
       include Mixins::AutonomousSystem
       include Mixins::Configurable
+      include Mixins::Database
       include Mixins::Retriable
 
       attr_accessor :ignore_old_artifacts, :ignore_threshold
@@ -52,10 +50,12 @@ module Mihari
       # @return [nil]
       #
       def run
-        set_enriched_artifacts
+        with_db_connection do
+          set_enriched_artifacts
 
-        Parallel.each(valid_emitters) do |emitter|
-          run_emitter emitter
+          Parallel.each(valid_emitters) do |emitter|
+            run_emitter emitter
+          end
         end
       end
 
@@ -111,7 +111,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def enriched_artifacts
-        @enriched_artifacts ||= unique_artifacts.map do |artifact|
+        @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
           artifact.enrich_all
           artifact
         end

data/lib/mihari/analyzers/binaryedge.rb

@@ -6,9 +6,8 @@ module Mihari
   module Analyzers
     class BinaryEdge < Base
       param :query
-      option :title, default: proc { "BinaryEdge search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
+      option :interval, default: proc { 0 }
 
       def artifacts
         results = search
@@ -17,9 +16,10 @@ module Mihari
         results.map do |result|
           events = result["events"] || []
           events.filter_map do |event|
-            event.dig "target", "ip"
+            data = event.dig("target", "ip")
+            data.nil? ? nil : Artifact.new(data: data, source: source, metadata: event)
           end
-        end.flatten.compact.uniq
+        end.flatten
       end
 
       private
@@ -55,6 +55,9 @@ module Mihari
 
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         responses
       end

data/lib/mihari/analyzers/censys.rb

@@ -6,9 +6,8 @@ module Mihari
   module Analyzers
     class Censys < Base
       param :query
-      option :title, default: proc { "Censys search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
+      option :interval, default: proc { 0 }
 
       def artifacts
         search
@@ -33,6 +32,9 @@ module Mihari
 
           cursor = response.result.links.next
           break if cursor == ""
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
 
         artifacts.flatten.uniq(&:data)
@@ -72,6 +74,7 @@ module Mihari
         Artifact.new(
           data: hit.ip,
           source: source,
+          metadata: hit.metadata,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/circl.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "CIRCL passive DNS/SSL search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 

data/lib/mihari/analyzers/crtsh.rb

@@ -6,15 +6,17 @@ module Mihari
   module Analyzers
     class Crtsh < Base
       param :query
-      option :title, default: proc { "crt.sh search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
       option :exclude_expired, default: proc { true }
 
       def artifacts
         results = search
-        name_values = results.filter_map { |result| result["name_value"] }
-        name_values.map(&:lines).flatten.uniq.map(&:chomp)
+        results.map do |result|
+          values = result["name_value"].to_s.lines.map(&:chomp)
+          values.map do |value|
+            Artifact.new(data: value, source: source, metadata: result)
+          end
+        end.flatten
       end
 
       private

data/lib/mihari/analyzers/dnpedia.rb

@@ -6,8 +6,7 @@ module Mihari
   module Analyzers
     class DNPedia < Base
       param :query
-      option :title, default: proc { "DNPedia domain search" }
-      option :description, default: proc { "query = #{query}" }
+
       option :tags, default: proc { [] }
 
       def artifacts
@@ -23,13 +22,14 @@ module Mihari
       #
       # Search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         res = api.search(query)
         rows = res["rows"] || []
         rows.map do |row|
-          [row["name"], row["zoneid"]].join(".")
+          data = [row["name"], row["zoneid"]].join(".")
+          Artifact.new(data: data, source: source, metadata: row)
         end
       end
     end

data/lib/mihari/analyzers/dnstwister.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require "dnstwister"
-require "resolv"
-require "parallel"
 
 module Mihari
   module Analyzers
@@ -10,8 +8,7 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "dnstwister domain search" }
-      option :description, default: proc { "query = #{query}" }
+
       option :tags, default: proc { [] }
 
       attr_reader :type

data/lib/mihari/analyzers/feed.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "mihari/feed/reader"
+require "mihari/feed/parser"
+
+module Mihari
+  module Analyzers
+    class Feed < Base
+      param :query
+
+      option :http_request_method, default: proc { "GET" }
+      option :http_request_headers, default: proc { {} }
+      option :http_request_payload, default: proc { {} }
+      option :http_request_payload_type, default: proc {}
+
+      option :selector, default: proc { "" }
+
+      def artifacts
+        Mihari::Feed::Parser.new(data).parse selector
+      end
+
+      private
+
+      def data
+        reader = Mihari::Feed::Reader.new(
+          query,
+          http_request_method: http_request_method,
+          http_request_headers: http_request_headers,
+          http_request_payload: http_request_payload,
+          http_request_payload_type: http_request_payload_type
+        )
+        reader.read
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/greynoise.rb

@@ -6,9 +6,6 @@ module Mihari
   module Analyzers
     class GreyNoise < Base
       param :query
-      option :title, default: proc { "GreyNoise search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       def artifacts
         res = Structs::GreyNoise::Response.from_dynamic!(search)
@@ -56,6 +53,7 @@ module Mihari
         Artifact.new(
           data: datum.ip,
           source: source,
+          metadata: datum.metadata_,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/onyphe.rb

@@ -7,9 +7,8 @@ module Mihari
   module Analyzers
     class Onyphe < Base
       param :query
-      option :title, default: proc { "Onyphe search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
+      option :interval, default: proc { 0 }
 
       def artifacts
         responses = search
@@ -59,6 +58,9 @@ module Mihari
 
           total = res.total
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         responses
       end
@@ -84,6 +86,7 @@ module Mihari
         Artifact.new(
           data: result.ip,
           source: source,
+          metadata: result.metadata,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/otx.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "OTX search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 

data/lib/mihari/analyzers/passivetotal.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "PassiveTotal search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -75,27 +72,29 @@ module Mihari
       #
       # Reverse whois search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def reverse_whois_search
         res = api.whois.search(query: query, field: "email")
         results = res["results"] || []
         results.map do |result|
-          result["domain"]
-        end.flatten.compact.uniq
+          data = result["domain"]
+          Artifact.new(data: data, source: source, metadata: result)
+        end.flatten
       end
 
       #
       # Passive SSL search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ssl_search
         res = api.ssl.history(query)
         results = res["results"] || []
         results.map do |result|
-          result["ipAddresses"]
-        end.flatten.compact.uniq
+          data = result["ipAddresses"]
+          Artifact.new(data: data, source: source, metadata: result)
+        end.flatten
       end
     end
   end

data/lib/mihari/analyzers/pulsedive.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "Pulsedive search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -47,7 +44,7 @@ module Mihari
       #
       # Search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
@@ -57,7 +54,12 @@ module Mihari
 
         properties = api.indicator.get_properties_by_id(iid)
         (properties["dns"] || []).filter_map do |property|
-          property["value"] if ["A", "PTR"].include?(property["name"])
+          if ["A", "PTR"].include?(property["name"])
+            nil
+          else
+            data = property["value"]
+            Artifact.new(data: data, source: source, metadata: property)
+          end
         end
       end
     end

data/lib/mihari/analyzers/rule.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "uuidtools"
-
 module Mihari
   module Analyzers
     ANALYZER_TO_CLASS = {
@@ -11,6 +9,7 @@ module Mihari
       "crtsh" => Crtsh,
       "dnpedia" => DNPedia,
       "dnstwister" => DNSTwister,
+      "feed" => Feed,
       "greynoise" => GreyNoise,
       "onyphe" => Onyphe,
       "otx" => OTX,
@@ -30,13 +29,14 @@ module Mihari
     }.freeze
 
     class Rule < Base
-      include Mihari::Mixins::DisallowedDataValue
+      include Mixins::DisallowedDataValue
+      include Mixins::Rule
 
       option :title
       option :description
       option :queries
 
-      option :id, default: proc {}
+      option :id, default: proc { "" }
       option :tags, default: proc { [] }
       option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
       option :disallowed_data_values, default: proc { [] }
@@ -46,7 +46,7 @@ module Mihari
       def initialize(**kwargs)
         super(**kwargs)
 
-        @source = id || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, title + description).to_s
+        @source = id
 
         validate_analyzer_configurations
       end
@@ -64,6 +64,12 @@ module Mihari
           klass = get_analyzer_class(analyzer_name)
 
           query = params[:query]
+
+          # set interval in the top level
+          options = params[:options] || {}
+          interval = options[:interval]
+          params[:interval] = interval if interval
+
           analyzer = klass.new(query, **params)
 
           # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>

data/lib/mihari/analyzers/securitytrails.rb

@@ -8,9 +8,6 @@ module Mihari
       include Mixins::Refang
 
       param :query
-      option :title, default: proc { "SecurityTrails search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
 
       attr_reader :type
 
@@ -47,7 +44,7 @@ module Mihari
       #
       # IP/domain/mail search
       #
-      # @return [Array<String>]
+      # @return [Array<String>, Array<Mihari::Artifact>]
       #
       def search
         case type
@@ -78,12 +75,15 @@ module Mihari
       #
       # IP search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ip_search
         result = api.domains.search(filter: { ipv4: query })
         records = result["records"] || []
-        records.filter_map { |record| record["hostname"] }.uniq
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, source: source, metadata: record)
+        end
       end
 
       #
@@ -94,7 +94,10 @@ module Mihari
       def mail_search
         result = api.domains.search(filter: { whois_email: query })
         records = result["records"] || []
-        records.filter_map { |record| record["hostname"] }.uniq
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, source: source, metadata: record)
+        end
       end
     end
   end

data/lib/mihari/analyzers/shodan.rb

@@ -6,9 +6,8 @@ module Mihari
   module Analyzers
     class Shodan < Base
       param :query
-      option :title, default: proc { "Shodan search" }
-      option :description, default: proc { "query = #{query}" }
-      option :tags, default: proc { [] }
+
+      option :interval, default: proc { 0 }
 
       def artifacts
         results = search
@@ -18,7 +17,7 @@ module Mihari
         results.map do |result|
           matches = result.matches || []
           matches.map { |match| build_artifact match }
-        end.flatten.compact.uniq(&:data)
+        end.flatten.uniq(&:data)
       end
 
       private
@@ -58,10 +57,14 @@ module Mihari
         responses = []
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
+
           break unless res
 
           responses << res
           break if res["total"].to_i <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         rescue JSON::ParserError
           # ignore JSON::ParserError
           # ref. https://github.com/ninoseki/mihari/issues/197
@@ -81,14 +84,18 @@ module Mihari
         as = nil
         as = AutonomousSystem.new(asn: normalize_asn(match.asn)) unless match.asn.nil?
 
-        geolocation = Geolocation.new(
-          country: match.location.country_name,
-          country_code: match.location.country_code
-        )
+        geolocation = nil
+        if !match.location.country_name.nil? && !match.location.country_code.nil?
+          geolocation = Geolocation.new(
+            country: match.location.country_name,
+            country_code: match.location.country_code
+          )
+        end
 
         Artifact.new(
           data: match.ip_str,
           source: source,
+          metadata: match.metadata,
           autonomous_system: as,
           geolocation: geolocation
         )

data/lib/mihari/analyzers/spyse.rb

@@ -1,16 +1,13 @@
 # frozen_string_literal: true
 
 require "spyse"
-require "json"
 
 module Mihari
   module Analyzers
     class Spyse < Base
       param :query
-      option :title, default: proc { "Spyse search" }
-      option :description, default: proc { "query = #{query}" }
+
       option :type, default: proc { "domain" }
-      option :tags, default: proc { [] }
 
       def artifacts
         search || []
@@ -42,33 +39,35 @@ module Mihari
       #
       # Domain search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def domain_search
         res = api.domain.search(search_params, limit: 100)
         items = res.dig("data", "items") || []
         items.map do |item|
-          item["name"]
-        end.uniq.compact
+          data = item["name"]
+          Artifact.new(data: data, source: source, metadata: item)
+        end
       end
 
       #
       # IP search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def ip_search
         res = api.ip.search(search_params, limit: 100)
         items = res.dig("data", "items") || []
         items.map do |item|
-          item["ip"]
-        end.uniq.compact
+          data = item["ip"]
+          Artifact.new(data: data, source: source, metadata: item)
+        end
       end
 
       #
       # IP/domain search
       #
-      # @return [Array<String>]
+      # @return [Array<Mihari::Artifact>]
       #
       def search
         case type