mihari 3.9.2 → 3.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c12a4ec4b0c1eee79deba3e5f3a511edcea4a13d7d72dc8142dbd85821095f55
-  data.tar.gz: 4a1e388e55efff4b715caf2261990e6122afd4eb26aca0874a33d3025a16ba5b
+  metadata.gz: 5a17c14dffe68b2f82316858615371036996a3a2b6d31d3c6a5c431294c38634
+  data.tar.gz: 97afaf2f8b20985b0908cc1fe5778fe75ab7d0e1bf13b990a7b4165c92843604
 SHA512:
-  metadata.gz: 8ead20235892bf77e222eff200f25539ffdd68cd21d14c5000d9196e518b9d2da2b461a2fe5d3dae4bc4bd899fb210f5f75b6330afad61826bc84dcfe40af23f
-  data.tar.gz: 4b5679794091a79adb426ef34c5396819c702d67ad84791dec1430eba0f007d1dcab5f7defc53b5dab06e8a0e174963c777fb05606f79a5f16c4377f07847f75
+  metadata.gz: 418d7f278536e245196723d2ebbe0d99934b0db3278e8f8178470241c67f25e05d0aacf7bab2b766ce69d0589a79e87eca554a91fe4cfae5cfa12cd52d97fb61
+  data.tar.gz: 9e90193273ee40c9956a138c2c73b713eb610a3ebdc1c9def73a6d354c4124a97e6842e438004956b3edc3206dfee8bae5771a86e964b59a2a7065a1f910891f

data/README.md

@@ -2,7 +2,6 @@
 
 [![Gem Version](https://badge.fury.io/rb/mihari.svg)](https://badge.fury.io/rb/mihari)
 [![Ruby CI](https://github.com/ninoseki/mihari/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/test.yml)
-[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/mihari)](https://hub.docker.com/r/ninoseki/mihari)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 
@@ -38,6 +37,7 @@ Mihari supports the following services by default.
 - [crt.sh](https://crt.sh/)
 - [DN Pedia](https://dnpedia.com/)
 - [dnstwister](https://dnstwister.report/)
+- [GreyNoise](https://www.greynoise.io/)
 - [Onyphe](https://onyphe.io)
 - [OTX](https://otx.alienvault.com/)
 - [PassiveTotal](https://community.riskiq.com/)

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.0.2-alpine3.13
+FROM ruby:3.0.3-alpine3.13
 
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
   && gem install pg mysql2 \

data/lib/mihari/analyzers/base.rb

@@ -111,7 +111,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def enriched_artifacts
-        @enriched_artifacts ||= unique_artifacts.map do |artifact|
+        @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
           artifact.enrich_all
           artifact
         end

data/lib/mihari/analyzers/binaryedge.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         results = search
         return [] unless results || results.empty?
@@ -55,6 +57,9 @@ module Mihari
 
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         responses
       end

data/lib/mihari/analyzers/censys.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         search
       end
@@ -33,6 +35,9 @@ module Mihari
 
           cursor = response.result.links.next
           break if cursor == ""
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
 
         artifacts.flatten.uniq(&:data)

data/lib/mihari/analyzers/feed.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "mihari/feed/reader"
+require "mihari/feed/parser"
+
+module Mihari
+  module Analyzers
+    class Feed < Base
+      param :query
+      option :title, default: proc { "Feed" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
+
+      option :http_request_method, default: proc { "GET" }
+      option :http_request_headers, default: proc { {} }
+      option :http_request_payload, default: proc { {} }
+      option :http_request_payload_type, default: proc {}
+
+      option :selector, default: proc { "" }
+
+      def artifacts
+        Mihari::Feed::Parser.new(data).parse selector
+      end
+
+      private
+
+      def data
+        reader = Mihari::Feed::Reader.new(
+          query,
+          http_request_method: http_request_method,
+          http_request_headers: http_request_headers,
+          http_request_payload: http_request_payload,
+          http_request_payload_type: http_request_payload_type
+        )
+        reader.read
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/greynoise.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "greynoise"
+
+module Mihari
+  module Analyzers
+    class GreyNoise < Base
+      param :query
+      option :title, default: proc { "GreyNoise search" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
+
+      def artifacts
+        res = Structs::GreyNoise::Response.from_dynamic!(search)
+        res.data.map do |datum|
+          build_artifact datum
+        end
+      end
+
+      private
+
+      PAGE_SIZE = 10_000
+
+      def configuration_keys
+        %w[greynoise_api_key]
+      end
+
+      def api
+        @api ||= ::GreyNoise::API.new(key: Mihari.config.greynoise_api_key)
+      end
+
+      #
+      # Search
+      #
+      # @return [Hash]
+      #
+      def search
+        api.experimental.gnql(query, size: PAGE_SIZE)
+      end
+
+      #
+      # Build an artifact from a GreyNoise search API response
+      #
+      # @param [Structs::GreyNoise::Datum] datum
+      #
+      # @return [Artifact]
+      #
+      def build_artifact(datum)
+        as = AutonomousSystem.new(asn: normalize_asn(datum.metadata.asn))
+
+        geolocation = Geolocation.new(
+          country: datum.metadata.country,
+          country_code: datum.metadata.country_code
+        )
+
+        Artifact.new(
+          data: datum.ip,
+          source: source,
+          autonomous_system: as,
+          geolocation: geolocation
+        )
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/onyphe.rb

@@ -11,6 +11,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         responses = search
         return [] unless responses
@@ -59,6 +61,9 @@ module Mihari
 
           total = res.total
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         responses
       end

data/lib/mihari/analyzers/rule.rb

@@ -11,6 +11,8 @@ module Mihari
       "crtsh" => Crtsh,
       "dnpedia" => DNPedia,
       "dnstwister" => DNSTwister,
+      "feed" => Feed,
+      "greynoise" => GreyNoise,
       "onyphe" => Onyphe,
       "otx" => OTX,
       "passivetotal" => PassiveTotal,
@@ -63,6 +65,12 @@ module Mihari
           klass = get_analyzer_class(analyzer_name)
 
           query = params[:query]
+
+          # set interval in the top level
+          options = params[:options] || {}
+          interval = options[:interval]
+          params[:interval] = interval
+
           analyzer = klass.new(query, **params)
 
           # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>

data/lib/mihari/analyzers/shodan.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         results = search
         return [] unless results || results.empty?
@@ -58,10 +60,14 @@ module Mihari
         responses = []
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
+
           break unless res
 
           responses << res
           break if res["total"].to_i <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         rescue JSON::ParserError
           # ignore JSON::ParserError
           # ref. https://github.com/ninoseki/mihari/issues/197
@@ -78,11 +84,16 @@ module Mihari
       # @return [Artifact]
       #
       def build_artifact(match)
-        as = AutonomousSystem.new(asn: normalize_asn(match.asn))
-        geolocation = Geolocation.new(
-          country: match.location.country_name,
-          country_code: match.location.country_code
-        )
+        as = nil
+        as = AutonomousSystem.new(asn: normalize_asn(match.asn)) unless match.asn.nil?
+
+        geolocation = nil
+        if !match.location.country_name.nil? && !match.location.country_code.nil?
+          geolocation = Geolocation.new(
+            country: match.location.country_name,
+            country_code: match.location.country_code
+          )
+        end
 
         Artifact.new(
           data: match.ip_str,

data/lib/mihari/analyzers/urlscan.rb

@@ -2,8 +2,6 @@
 
 require "urlscan"
 
-SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
-
 module Mihari
   module Analyzers
     class Urlscan < Base
@@ -12,7 +10,11 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
       option :allowed_data_types, default: proc { SUPPORTED_DATA_TYPES }
-      option :use_similarity, default: proc { false }
+
+      option :interval, default: proc { 0 }
+
+      SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
+      SIZE = 1000
 
       def initialize(*args, **kwargs)
         super
@@ -21,16 +23,15 @@ module Mihari
       end
 
       def artifacts
-        result = search
-        return [] unless result
-
-        results = result["results"] || []
+        responses = search
+        results = responses.map(&:results).flatten
 
         allowed_data_types.map do |type|
-          results.filter_map do |match|
-            match.dig "page", type
+          results.filter_map do |result|
+            page = result.page
+            page.send(type.to_sym)
           end.uniq
-        end.flatten
+        end.flatten.compact
       end
 
       private
@@ -43,15 +44,38 @@ module Mihari
         @api ||= ::UrlScan::API.new(Mihari.config.urlscan_api_key)
       end
 
+      #
+      # Search with search_after option
+      #
+      # @return [Structs::Urlscan::Response]
+      #
+      def search_with_search_after(search_after: nil)
+        res = api.search(query, size: SIZE, search_after: search_after)
+        Structs::Urlscan::Response.from_dynamic! res
+      end
+
       #
       # Search
       #
-      # @return [Array<Hash>]
+      # @return [Array<Structs::Urlscan::Response>]
       #
       def search
-        return api.pro.similar(query) if use_similarity
+        responses = []
+
+        search_after = nil
+        loop do
+          res = search_with_search_after(search_after: search_after)
+          responses << res
+
+          break if res.results.length < SIZE
+
+          search_after = res.results.last.sort.join(",")
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
+        end
 
-        api.search(query, size: 10_000)
+        responses
       end
 
       #

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def initialize(*args, **kwargs)
         super
 
@@ -54,6 +56,9 @@ module Mihari
           break if response.meta.cursor.nil?
 
           cursor = response.meta.cursor
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
 
         responses

data/lib/mihari/analyzers/zoomeye.rb

@@ -11,6 +11,8 @@ module Mihari
       option :tags, default: proc { [] }
       option :type, default: proc { "host" }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         case type
         when "host"
@@ -87,6 +89,9 @@ module Mihari
           total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         convert_responses responses.compact
       end
@@ -119,6 +124,9 @@ module Mihari
           total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         convert_responses responses.compact
       end

data/lib/mihari/cli/analyzer.rb

@@ -6,6 +6,8 @@ require "mihari/commands/circl"
 require "mihari/commands/crtsh"
 require "mihari/commands/dnpedia"
 require "mihari/commands/dnstwister"
+require "mihari/commands/feed"
+require "mihari/commands/greynoise"
 require "mihari/commands/onyphe"
 require "mihari/commands/otx"
 require "mihari/commands/passivetotal"
@@ -25,6 +27,7 @@ module Mihari
     class Analyzer < Base
       class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not."
       class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not."
+      class_option :interval, type: :numeric, default: 0, desc: "Seconds of the interval while calling API in a row."
       class_option :config, type: :string, desc: "Path to the config file"
 
       include Mihari::Commands::BinaryEdge
@@ -33,6 +36,8 @@ module Mihari
       include Mihari::Commands::Crtsh
       include Mihari::Commands::DNPedia
       include Mihari::Commands::DNSTwister
+      include Mihari::Commands::Feed
+      include Mihari::Commands::GreyNoise
       include Mihari::Commands::JSON
       include Mihari::Commands::Onyphe
       include Mihari::Commands::OTX

data/lib/mihari/commands/feed.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Feed
+      def self.included(thor)
+        thor.class_eval do
+          desc "feed [URL]", "ingest feed"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          method_option :http_request_method, type: :string, desc: "HTTP request method"
+          method_option :http_request_headers, type: :hash, desc: "HTTP request headers"
+          method_option :http_request_payload_type, type: :string, desc: "HTTP request payload type"
+          method_option :http_request_payload, type: :hash, desc: "HTTP request payload"
+          method_option :selector, type: :string, desc: "jr selector", required: true
+          def feed(query)
+            with_error_handling do
+              run_analyzer Analyzers::Feed, query: query, options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/greynoise.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module GreyNoise
+      def self.included(thor)
+        thor.class_eval do
+          desc "greynoise [QUERY]", "GreyNoise search"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          def greynoise(query)
+            with_error_handling do
+              run_analyzer Analyzers::GreyNoise, query: query, options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/commands/urlscan.rb

@@ -9,8 +9,7 @@ module Mihari
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
-          method_option :target_type, type: :string, default: "url", desc: "target type to fetch from search results (target type should be 'url', 'domain' or 'ip')"
-          method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
+          method_option :allowed_data_types, type: :array, default: ["url", "ip", "domain"], desc: "types to fetch from search results ('url', 'domain' or 'ip')"
           def urlscan(query)
             with_error_handling do
               run_analyzer Analyzers::Urlscan, query: query, options: options

data/lib/mihari/database.rb

@@ -2,7 +2,7 @@
 
 require "active_record"
 
-class InitialSchema < ActiveRecord::Migration[6.1]
+class InitialSchema < ActiveRecord::Migration[7.0]
   def change
     create_table :tags, if_not_exists: true do |t|
       t.string :name, null: false
@@ -32,13 +32,13 @@ class InitialSchema < ActiveRecord::Migration[6.1]
   end
 end
 
-class AddeSourceToArtifactSchema < ActiveRecord::Migration[6.1]
+class AddeSourceToArtifactSchema < ActiveRecord::Migration[7.0]
   def change
     add_column :artifacts, :source, :string, if_not_exists: true
   end
 end
 
-class EnrichmentsSchema < ActiveRecord::Migration[6.1]
+class EnrichmentsSchema < ActiveRecord::Migration[7.0]
   def change
     create_table :autonomous_systems, if_not_exists: true do |t|
       t.integer :asn, null: false
@@ -74,7 +74,7 @@ class EnrichmentsSchema < ActiveRecord::Migration[6.1]
   end
 end
 
-class EnrichmentCreatedAtSchema < ActiveRecord::Migration[6.1]
+class EnrichmentCreatedAtSchema < ActiveRecord::Migration[7.0]
   def change
     # Add created_at column because now it is able to enrich an atrifact after the creation
     add_column :autonomous_systems, :created_at, :datetime, if_not_exists: true

data/lib/mihari/enrichers/ipinfo.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "http"
 require "json"
 require "memist"

data/lib/mihari/errors.rb

@@ -8,4 +8,8 @@ module Mihari
   class RetryableError < Error; end
 
   class FileNotFoundError < Error; end
+
+  class HttpError < Error; end
+
+  class FeedParseError < Error; end
 end

data/lib/mihari/feed/parser.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "jr/cli/core_ext"
+
+module Mihari
+  module Feed
+    class Parser
+      attr_reader :data
+
+      #
+      # @param [Array<Hash>, Array<Array<String>>] data
+      #
+      def initialize(data)
+        @data = data
+      end
+
+      #
+      # Parse data by selector
+      #
+      # @param [String] selector
+      #
+      # @return [Array<String>]
+      #
+      def parse(selector)
+        parsed = data.instance_eval(selector)
+
+        raise FeedParseError unless parsed.is_a?(Array) || parsed.is_a?(Enumerator)
+        raise FeedParseError unless parsed.all?(String)
+
+        parsed.to_a
+      end
+    end
+  end
+end