mihari 5.4.2 → 5.4.3

data/lib/mihari/clients/securitytrails.rb

@@ -15,6 +15,50 @@ module Mihari
         super(base_url, headers: headers)
       end
 
+      #
+      # Domain search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def domain_search(query)
+        records = get_all_dns_history(query, type: "a")
+        records.map do |record|
+          (record["values"] || []).map { |value| value["ip"] }
+        end.flatten.compact.uniq
+      end
+
+      #
+      # IP search
+      #
+      # @param [String] query
+      #
+      # @return [Array<Mihari::Artifact>]
+      #
+      def ip_search(query)
+        records = search_by_ip(query)
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, metadata: record)
+        end
+      end
+
+      #
+      # Mail search
+      #
+      # @param [String] query
+      #
+      # @return [Array<String>]
+      #
+      def mail_search(query)
+        records = search_by_mail(query)
+        records.filter_map do |record|
+          data = record["hostname"]
+          Artifact.new(data: data, metadata: record)
+        end
+      end
+
       #
       # @param [String] mail
       #

data/lib/mihari/clients/shodan.rb

@@ -3,6 +3,8 @@
 module Mihari
   module Clients
     class Shodan < Base
+      PAGE_SIZE = 100
+
       # @return [String]
       attr_reader :api_key
 
@@ -10,11 +12,12 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://api.shodan.io", api_key:, headers: {})
+      def initialize(base_url = "https://api.shodan.io", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
 
         @api_key = api_key
       end
@@ -36,6 +39,31 @@ module Mihari
         res = get("/shodan/host/search", params: params)
         Structs::Shodan::Result.from_dynamic! JSON.parse(res.body.to_s)
       end
+
+      #
+      # @param [String] query
+      # @param [Boolean] minify
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::Shodan::Result>]
+      #
+      def search_with_pagination(query, minify: true, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = search(query, page: page, minify: minify)
+
+            y.yield res
+
+            break if res.total <= page * PAGE_SIZE
+
+            sleep_interval
+          rescue JSON::ParserError
+            # ignore JSON::ParserError
+            # ref. https://github.com/ninoseki/mihari/issues/197
+            next
+          end
+        end
+      end
     end
   end
 end

data/lib/mihari/clients/urlscan.rb

@@ -7,26 +7,52 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Interval, nil] interval
       #
-      def initialize(base_url = "https://urlscan.io", api_key:, headers: {})
+      def initialize(base_url = "https://urlscan.io", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
         headers["api-key"] = api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
       # @param [String] q
-      # @param [Integer] size
+      # @param [Integer, nil] size
       # @param [String, nil] search_after
       #
-      # @return [Hash]
+      # @return [Structs::Urlscan::Response]
       #
-      def search(q, size: 100, search_after: nil)
+      def search(q, size: nil, search_after: nil)
         params = { q: q, size: size, search_after: search_after }.compact
         res = get("/api/v1/search/", params: params)
-        JSON.parse res.body.to_s
+        Structs::Urlscan::Response.from_dynamic! JSON.parse(res.body.to_s)
+      end
+
+      #
+      # @param [String] q
+      # @param [Integer, nil] size
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::Urlscan::Response>]
+      #
+      def search_with_pagination(q, size: nil, pagination_limit: Mihari.config.pagination_limit)
+        search_after = nil
+
+        Enumerator.new do |y|
+          pagination_limit.times do
+            res = search(q, size: size, search_after: search_after)
+
+            y.yield res
+
+            break unless res.has_more
+
+            search_after = res.results.last.sort.join(",")
+
+            sleep_interval
+          end
+        end
       end
     end
   end

data/lib/mihari/clients/virustotal.rb

@@ -7,13 +7,14 @@ module Mihari
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {})
+      def initialize(base_url = "https://www.virustotal.com", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") if api_key.nil?
 
         headers["x-apikey"] = api_key
 
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
@@ -38,11 +39,35 @@ module Mihari
       # @param [String] query
       # @param [String, nil] cursor
       #
-      # @return [Hash]
+      # @return [Structs::VirusTotalIntelligence::Response]
       #
       def intel_search(query, cursor: nil)
         params = { query: query, cursor: cursor }.compact
-        _get("/api/v3/intelligence/search", params: params)
+        res = _get("/api/v3/intelligence/search", params: params)
+        Structs::VirusTotalIntelligence::Response.from_dynamic! res
+      end
+
+      #
+      # @param [String] query
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Structs::VirusTotalIntelligence::Response>]
+      #
+      def intel_search_with_pagination(query, pagination_limit: Mihari.config.pagination_limit)
+        cursor = nil
+
+        Enumerator.new do |y|
+          pagination_limit.times do
+            res = intel_search(query, cursor: cursor)
+
+            y.yield res
+
+            cursor = res.meta.cursor
+            break if cursor.nil?
+
+            sleep_interval
+          end
+        end
       end
 
       private

data/lib/mihari/clients/zoomeye.rb

@@ -3,18 +3,21 @@
 module Mihari
   module Clients
     class ZoomEye < Base
+      PAGE_SIZE = 10
+
       attr_reader :api_key
 
       #
       # @param [String] base_url
       # @param [String, nil] api_key
       # @param [Hash] headers
+      # @param [Integer, nil] interval
       #
-      def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {})
+      def initialize(base_url = "https://api.zoomeye.org", api_key:, headers: {}, interval: nil)
         raise(ArgumentError, "'api_key' argument is required") unless api_key
 
         headers["api-key"] = api_key
-        super(base_url, headers: headers)
+        super(base_url, headers: headers, interval: interval)
       end
 
       #
@@ -36,6 +39,30 @@ module Mihari
         _get("/host/search", params: params)
       end
 
+      #
+      # @param [String] query
+      # @param [String, nil] facets
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Hash>]
+      #
+      def host_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = host_search(query, facets: facets, page: page)
+
+            break if res.nil?
+
+            y.yield res
+
+            total = res["total"].to_i
+            break if total <= page * PAGE_SIZE
+
+            sleep_interval
+          end
+        end
+      end
+
       #
       # Search the Web technologies
       #
@@ -55,6 +82,30 @@ module Mihari
         _get("/web/search", params: params)
       end
 
+      #
+      # @param [String] query
+      # @param [String, nil] facets
+      # @param [Integer] pagination_limit
+      #
+      # @return [Enumerable<Hash>]
+      #
+      def web_search_with_pagination(query, facets: nil, pagination_limit: Mihari.config.pagination_limit)
+        Enumerator.new do |y|
+          (1..pagination_limit).each do |page|
+            res = web_search(query, facets: facets, page: page)
+
+            break if res.nil?
+
+            y.yield res
+
+            total = res["total"].to_i
+            break if total <= page * PAGE_SIZE
+
+            sleep_interval
+          end
+        end
+      end
+
       private
 
       #

data/lib/mihari/commands/web.rb

@@ -12,7 +12,7 @@ module Mihari
             method_option :threads, type: :string, default: "0:5", desc: "min:max threads to use"
             method_option :verbose, type: :boolean, default: true, desc: "Report each request"
             method_option :worker_timeout, type: :numeric, default: 60, desc: "Worker timeout value (in seconds)"
-            method_option :hide_config_values, type: :boolean, default: false,
+            method_option :hide_config_values, type: :boolean, default: true,
               desc: "Whether to hide config values or not"
             method_option :open, type: :boolean, default: true, desc: "Whether to open the app in browser or not"
             method_option :rack_env, type: :string, default: "production", desc: "Rack environment"

data/lib/mihari/config.rb

@@ -141,7 +141,7 @@ module Mihari
 
       @sentry_dsn = ENV.fetch("SENTRY_DSN", nil)
 
-      @hide_config_values = ENV.fetch("HIDE_CONFIG_VALUES", false)
+      @hide_config_values = ENV.fetch("HIDE_CONFIG_VALUES", true)
 
       @retry_times = ENV.fetch("RETRY_TIMES", 3).to_i
       @retry_interval = ENV.fetch("RETRY_INTERVAL", 5).to_i

data/lib/mihari/structs/censys.rb

@@ -18,7 +18,7 @@ module Mihari
         #
         # @return [Mihari::AutonomousSystem]
         #
-        def to_as
+        def as
           Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
         end
 
@@ -58,7 +58,7 @@ module Mihari
         #
         # @return [Mihari::Geolocation] <description>
         #
-        def to_geolocation
+        def geolocation
           # sometimes Censys overlooks country
           # then set geolocation as nil
           return nil if country.nil?
@@ -98,7 +98,7 @@ module Mihari
         #
         # @return [Mihari::Port]
         #
-        def to_port
+        def _port
           Port.new(port: port)
         end
 
@@ -162,20 +162,20 @@ module Mihari
         #
         # @return [Array<Mihari::Port>]
         #
-        def to_ports
-          services.map(&:to_port)
+        def ports
+          services.map(&:_port)
         end
 
         #
         # @return [Mihari::Artifact]
         #
-        def to_artifact
+        def artifact
           Artifact.new(
             data: ip,
             metadata: metadata,
-            autonomous_system: autonomous_system.to_as,
-            geolocation: location.to_geolocation,
-            ports: to_ports
+            autonomous_system: autonomous_system.as,
+            geolocation: location.geolocation,
+            ports: ports
           )
         end
 
@@ -269,8 +269,8 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
-          hits.map(&:to_artifact)
+        def artifacts
+          hits.map(&:artifact)
         end
 
         class << self

data/lib/mihari/structs/greynoise.rb

@@ -34,14 +34,14 @@ module Mihari
         #
         # @return [Mihari::AutonomousSystem]
         #
-        def to_as
+        def as
           Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
         end
 
         #
         # @return [Mihari::Geolocation]
         #
-        def to_geolocation
+        def geolocation
           Mihari::Geolocation.new(
             country: country,
             country_code: country_code
@@ -94,12 +94,12 @@ module Mihari
         #
         # @return [Mihari::Artifact]
         #
-        def to_artifact
+        def artifact
           Mihari::Artifact.new(
             data: ip,
             metadata: metadata_,
-            autonomous_system: metadata.to_as,
-            geolocation: metadata.to_geolocation
+            autonomous_system: metadata.as,
+            geolocation: metadata.geolocation
           )
         end
 
@@ -126,6 +126,7 @@ module Mihari
         attribute :data, Types.Array(Datum)
         attribute :message, Types::String
         attribute :query, Types::String
+        attribute :scroll, Types::String.optional
 
         #
         # @return [Boolean]
@@ -162,11 +163,18 @@ module Mihari
           attributes[:query]
         end
 
+        #
+        # @return [String, nil]
+        #
+        def scroll
+          attributes[:scroll]
+        end
+
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
-          data.map { |datum| datum.to_artifact }
+        def artifacts
+          data.map(&:artifact)
         end
 
         class << self
@@ -182,7 +190,8 @@ module Mihari
               count: d.fetch("count"),
               data: d.fetch("data").map { |x| Datum.from_dynamic!(x) },
               message: d.fetch("message"),
-              query: d.fetch("query")
+              query: d.fetch("query"),
+              scroll: d["scroll"]
             )
           end
         end

data/lib/mihari/structs/onyphe.rb

@@ -42,19 +42,19 @@ module Mihari
         #
         # @return [Mihari::Artifact]
         #
-        def to_artifact
+        def artifact
           Mihari::Artifact.new(
             data: ip,
             metadata: metadata,
-            autonomous_system: to_as,
-            geolocation: to_geolocation
+            autonomous_system: as,
+            geolocation: geolocation
           )
         end
 
         #
         # @return [Mihari::Geolocation, nil]
         #
-        def to_geolocation
+        def geolocation
           return nil if country_code.nil?
 
           Mihari::Geolocation.new(
@@ -66,7 +66,7 @@ module Mihari
         #
         # @return [Mihari::AutonomousSystem]
         #
-        def to_as
+        def as
           Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
         end
 
@@ -150,8 +150,8 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
-          results.map(&:to_artifact)
+        def artifacts
+          results.map(&:artifact)
         end
 
         class << self

data/lib/mihari/structs/shodan.rb

@@ -24,7 +24,7 @@ module Mihari
         #
         # @return [Mihari::Geolocation, nil]
         #
-        def to_geolocation
+        def geolocation
           return nil if country_name.nil? && country_code.nil?
 
           Mihari::Geolocation.new(
@@ -105,7 +105,7 @@ module Mihari
         #
         # @return [Mihari::AutonomousSystem, nil]
         #
-        def to_asn
+        def _asn
           return nil if asn.nil?
 
           Mihari::AutonomousSystem.new(asn: normalize_asn(asn))
@@ -194,7 +194,7 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
+        def artifacts
           matches.map do |match|
             metadata = collect_metadata_by_ip(match.ip_str)
             ports = collect_ports_by_ip(match.ip_str).map do |port|
@@ -207,8 +207,8 @@ module Mihari
             Mihari::Artifact.new(
               data: match.ip_str,
               metadata: metadata,
-              autonomous_system: match.to_asn,
-              geolocation: match.location.to_geolocation,
+              autonomous_system: match._asn,
+              geolocation: match.location.geolocation,
               ports: ports,
               reverse_dns_names: reverse_dns_names
             )

data/lib/mihari/structs/urlscan.rb

@@ -83,7 +83,7 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
+        def artifacts
           values = [page.url, page.domain, page.ip].compact
           values.map do |value|
             Mihari::Artifact.new(data: value, metadata: metadata)
@@ -129,8 +129,8 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
-          results.map(&:to_artifacts).flatten
+        def artifacts
+          results.map(&:artifacts).flatten
         end
 
         class << self

data/lib/mihari/structs/virustotal_intelligence.rb

@@ -81,7 +81,7 @@ module Mihari
         #
         # @return [Mihari::Artifact]
         #
-        def to_artifact
+        def artifact
           Artifact.new(data: value, metadata: metadata)
         end
 
@@ -155,8 +155,8 @@ module Mihari
         #
         # @return [Array<Mihari::Artifact>]
         #
-        def to_artifacts
-          data.map(&:to_artifact)
+        def artifacts
+          data.map(&:artifact)
         end
 
         class << self

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "5.4.2"
+  VERSION = "5.4.3"
 end

data/mihari.gemspec

@@ -47,28 +47,28 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rb-fsevent", "~> 0.11"
   spec.add_development_dependency "rerun", "~> 0.14"
   spec.add_development_dependency "rspec", "~> 3.12"
-  spec.add_development_dependency "simplecov-lcov", "~> 0.8.0"
+  spec.add_development_dependency "simplecov-lcov", "~> 0.8"
   spec.add_development_dependency "standard", "~> 1.31"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.2"
-  spec.add_development_dependency "webmock", "~> 3.18"
+  spec.add_development_dependency "webmock", "~> 3.19"
 
   unless ci_env?
     spec.add_development_dependency "lefthook", "~> 1.4"
     spec.add_development_dependency "solargraph", "~> 0.49"
   end
 
-  spec.add_dependency "activerecord", "7.0.7"
+  spec.add_dependency "activerecord", "7.0.7.2"
   spec.add_dependency "addressable", "2.8.5"
   spec.add_dependency "awrence", "2.0.1"
   spec.add_dependency "dotenv", "2.8.1"
   spec.add_dependency "dry-container", "0.11.0"
   spec.add_dependency "dry-files", "1.0.1"
-  spec.add_dependency "dry-schema", "1.13.2"
+  spec.add_dependency "dry-schema", "1.13.3"
   spec.add_dependency "dry-struct", "1.6.0"
   spec.add_dependency "dry-validation", "1.10.0"
   spec.add_dependency "email_address", "0.2.4"
-  spec.add_dependency "grape", "1.7.0"
+  spec.add_dependency "grape", "1.8.0"
   spec.add_dependency "grape-entity", "1.0.0"
   spec.add_dependency "grape-swagger", "1.6.1"
   spec.add_dependency "grape-swagger-entity", "0.5.2"
@@ -86,9 +86,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rack-cors", "2.0.1"
   spec.add_dependency "rackup", "2.1.0"
   spec.add_dependency "semantic_logger", "4.14.0"
-  spec.add_dependency "sentry-ruby", "5.10.0"
+  spec.add_dependency "sentry-ruby", "5.11.0"
   spec.add_dependency "slack-notifier", "2.4.0"
-  spec.add_dependency "sqlite3", "1.6.3"
+  spec.add_dependency "sqlite3", "1.6.4"
   spec.add_dependency "thor", "1.2.2"
   spec.add_dependency "uuidtools", "2.2.0"
   spec.add_dependency "whois", "5.1.0"