mihari 3.10.1 → 3.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0fcea779559f641125a388f63f0aa2d39979842f9b66f116194c36fe6410c56c
-  data.tar.gz: ceefe0fa86e5d3c8066bab202f7dc1dfb2b21017554a2b63c5a18f567e45aadb
+  metadata.gz: 2c8fda839e1bb1ec4733b7da17e20e3b47bd5795d5aca25371d09e5a0fa9a575
+  data.tar.gz: a4551c61c625bd08167608051750bf1785a31ad5215e95e857753fc5fed31d00
 SHA512:
-  metadata.gz: 834c4c815ddf9c14b9047c9a8d9ed8ce56e3f115c16b797c4ebf40205529f7c94fe7ddda8f9c7f5cc0675c65123ac0957d28a0bfe51108d2492efc230f509dd8
-  data.tar.gz: 03ed05a672e17bfce72706de94aa2a339053f9b6545907c1e71cf60987be047e1af6bc6d4e95a16f6b6bb371de2ff3b65602ae4c3c9eb2eab3f70c2ace0735cf
+  metadata.gz: 81a1ca91b65b03f98bb3504a85c8113b2cf189a4911109e113da1058fb9a1bd61c8807e6f3bec72484786b89e68d0027782d38013676bd6ae78aa57434315972
+  data.tar.gz: b42b1658310a7ee87940fbebb6507edca2187d6be92957a149f9234af8bf32ba06e702c401d321b553901dc56094964e78ff602549734f51abbf24b4b7a1fa4d

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.0.2-alpine3.13
+FROM ruby:3.0.3-alpine3.13
 
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev mysql-client mysql-dev \
   && gem install pg mysql2 \

data/lib/mihari/analyzers/binaryedge.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         results = search
         return [] unless results || results.empty?
@@ -55,6 +57,9 @@ module Mihari
 
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         responses
       end

data/lib/mihari/analyzers/censys.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         search
       end
@@ -33,6 +35,9 @@ module Mihari
 
           cursor = response.result.links.next
           break if cursor == ""
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
 
         artifacts.flatten.uniq(&:data)

data/lib/mihari/analyzers/onyphe.rb

@@ -11,6 +11,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         responses = search
         return [] unless responses
@@ -59,6 +61,9 @@ module Mihari
 
           total = res.total
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         responses
       end

data/lib/mihari/analyzers/rule.rb

@@ -64,6 +64,12 @@ module Mihari
           klass = get_analyzer_class(analyzer_name)
 
           query = params[:query]
+
+          # set interval in the top level
+          options = params[:options] || {}
+          interval = options[:interval]
+          params[:interval] = interval
+
           analyzer = klass.new(query, **params)
 
           # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>

data/lib/mihari/analyzers/shodan.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         results = search
         return [] unless results || results.empty?
@@ -63,6 +65,9 @@ module Mihari
 
           responses << res
           break if res["total"].to_i <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         rescue JSON::ParserError
           # ignore JSON::ParserError
           # ref. https://github.com/ninoseki/mihari/issues/197

data/lib/mihari/analyzers/urlscan.rb

@@ -2,8 +2,6 @@
 
 require "urlscan"
 
-SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
-
 module Mihari
   module Analyzers
     class Urlscan < Base
@@ -12,7 +10,11 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
       option :allowed_data_types, default: proc { SUPPORTED_DATA_TYPES }
-      option :use_similarity, default: proc { false }
+
+      option :interval, default: proc { 0 }
+
+      SUPPORTED_DATA_TYPES = %w[url domain ip].freeze
+      SIZE = 1000
 
       def initialize(*args, **kwargs)
         super
@@ -21,16 +23,15 @@ module Mihari
       end
 
       def artifacts
-        result = search
-        return [] unless result
-
-        results = result["results"] || []
+        responses = search
+        results = responses.map(&:results).flatten
 
         allowed_data_types.map do |type|
-          results.filter_map do |match|
-            match.dig "page", type
+          results.filter_map do |result|
+            page = result.page
+            page.send(type.to_sym)
           end.uniq
-        end.flatten
+        end.flatten.compact
       end
 
       private
@@ -43,15 +44,38 @@ module Mihari
         @api ||= ::UrlScan::API.new(Mihari.config.urlscan_api_key)
       end
 
+      #
+      # Search with search_after option
+      #
+      # @return [Structs::Urlscan::Response]
+      #
+      def search_with_search_after(search_after: nil)
+        res = api.search(query, size: SIZE, search_after: search_after)
+        Structs::Urlscan::Response.from_dynamic! res
+      end
+
       #
       # Search
       #
-      # @return [Array<Hash>]
+      # @return [Array<Structs::Urlscan::Response>]
       #
       def search
-        return api.pro.similar(query) if use_similarity
+        responses = []
+
+        search_after = nil
+        loop do
+          res = search_with_search_after(search_after: search_after)
+          responses << res
+
+          break if res.results.length < SIZE
+
+          search_after = res.results.last.sort.join(",")
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
+        end
 
-        api.search(query, size: 10_000)
+        responses
       end
 
       #

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -10,6 +10,8 @@ module Mihari
       option :description, default: proc { "query = #{query}" }
       option :tags, default: proc { [] }
 
+      option :interval, default: proc { 0 }
+
       def initialize(*args, **kwargs)
         super
 
@@ -54,6 +56,9 @@ module Mihari
           break if response.meta.cursor.nil?
 
           cursor = response.meta.cursor
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
 
         responses

data/lib/mihari/analyzers/zoomeye.rb

@@ -11,6 +11,8 @@ module Mihari
       option :tags, default: proc { [] }
       option :type, default: proc { "host" }
 
+      option :interval, default: proc { 0 }
+
       def artifacts
         case type
         when "host"
@@ -87,6 +89,9 @@ module Mihari
           total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         convert_responses responses.compact
       end
@@ -119,6 +124,9 @@ module Mihari
           total = res["total"].to_i
           responses << res
           break if total <= page * PAGE_SIZE
+
+          # sleep #{interval} seconds to avoid the rate limitation (if it is set)
+          sleep interval
         end
         convert_responses responses.compact
       end

data/lib/mihari/cli/analyzer.rb

@@ -26,6 +26,7 @@ module Mihari
     class Analyzer < Base
       class_option :ignore_old_artifacts, type: :boolean, default: false, desc: "Whether to ignore old artifacts from checking or not."
       class_option :ignore_threshold, type: :numeric, default: 0, desc: "Number of days to define whether an artifact is old or not."
+      class_option :interval, type: :numeric, default: 0, desc: "Seconds of the interval while calling API in a row."
       class_option :config, type: :string, desc: "Path to the config file"
 
       include Mihari::Commands::BinaryEdge

data/lib/mihari/commands/urlscan.rb

@@ -9,8 +9,7 @@ module Mihari
           method_option :title, type: :string, desc: "title"
           method_option :description, type: :string, desc: "description"
           method_option :tags, type: :array, desc: "tags"
-          method_option :target_type, type: :string, default: "url", desc: "target type to fetch from search results (target type should be 'url', 'domain' or 'ip')"
-          method_option :use_similarity, type: :boolean, default: false, desc: "use similarity API or not"
+          method_option :allowed_data_types, type: :array, default: ["url", "ip", "domain"], desc: "types to fetch from search results ('url', 'domain' or 'ip')"
           def urlscan(query)
             with_error_handling do
               run_analyzer Analyzers::Urlscan, query: query, options: options

data/lib/mihari/schemas/rule.rb

@@ -7,33 +7,41 @@ require "mihari/schemas/macros"
 
 module Mihari
   module Schemas
+    AnalyzerOptions = Dry::Schema.Params do
+      optional(:interval).value(:integer)
+    end
+
     Analyzer = Dry::Schema.Params do
       required(:analyzer).value(Types::AnalyzerTypes)
       required(:query).value(:string)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Spyse = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("spyse"))
       required(:query).value(:string)
       required(:type).value(Types::String.enum("ip", "domain"))
+      optional(:options).hash(AnalyzerOptions)
     end
 
     ZoomEye = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("zoomeye"))
       required(:query).value(:string)
       required(:type).value(Types::String.enum("host", "web"))
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Crtsh = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("crtsh"))
       required(:query).value(:string)
       optional(:exclude_expired).value(:bool).default(true)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Urlscan = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("urlscan"))
       required(:query).value(:string)
-      optional(:use_similarity).value(:bool).default(true)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Rule = Dry::Schema.Params do

data/lib/mihari/structs/urlscan.rb

@@ -0,0 +1,51 @@
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module Urlscan
+      class Page < Dry::Struct
+        attribute :domain, Types::String.optional
+        attribute :ip, Types::String.optional
+        attribute :url, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            domain: d["domain"],
+            ip: d["ip"],
+            url: d.fetch("url")
+          )
+        end
+      end
+
+      class Result < Dry::Struct
+        attribute :page, Page
+        attribute :id, Types::String
+        attribute :sort, Types.Array(Types::String | Types::Integer)
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            page: Page.from_dynamic!(d.fetch("page")),
+            id: d.fetch("_id"),
+            sort: d.fetch("sort")
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :results, Types.Array(Result)
+        attribute :has_more, Types::Bool
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            results: d.fetch("results").map { |x| Result.from_dynamic!(x) },
+            has_more: d.fetch("has_more")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/types.rb

@@ -8,11 +8,14 @@ module Mihari
     Nil = Strict::Nil
     Hash = Strict::Hash
     String = Strict::String
+    Bool = Strict::Bool
     Double = Strict::Float | Strict::Integer
     DateTime = Strict::DateTime
 
     DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
 
+    UrlscanDataTypes = Types::String.enum("ip", "domain", "url")
+
     AnalyzerTypes = Types::String.enum(
       "binaryedge",
       "censys",

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.10.1"
+  VERSION = "3.11.0"
 end

data/lib/mihari.rb

@@ -117,6 +117,7 @@ require "mihari/structs/greynoise"
 require "mihari/structs/ipinfo"
 require "mihari/structs/onyphe"
 require "mihari/structs/shodan"
+require "mihari/structs/urlscan"
 require "mihari/structs/virustotal_intelligence"
 
 # Schemas

data/mihari.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 2.2"
   spec.add_development_dependency "coveralls_reborn", "~> 0.23"
-  spec.add_development_dependency "fakefs", "~> 1.3"
+  spec.add_development_dependency "fakefs", "~> 1.4"
   spec.add_development_dependency "mysql2", "~> 0.5"
   spec.add_development_dependency "overcommit", "~> 0.58"
   spec.add_development_dependency "pg", "~> 1.2"
@@ -36,8 +36,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rb-fsevent", "~> 0.11"
   spec.add_development_dependency "rerun", "~> 0.13"
   spec.add_development_dependency "rspec", "~> 3.10"
-  spec.add_development_dependency "standard", "~> 1.3"
-  spec.add_development_dependency "steep", "~> 0.46"
+  spec.add_development_dependency "standard", "~> 1.5"
+  spec.add_development_dependency "steep", "~> 0.47"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.0"
   spec.add_development_dependency "webmock", "~> 3.14"
@@ -94,7 +94,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "sqlite3", "~> 1.4"
   spec.add_dependency "thor", "~> 1.1"
   spec.add_dependency "thread_safe", "~> 0.3"
-  spec.add_dependency "urlscan", "~> 0.7"
+  spec.add_dependency "urlscan", "~> 0.8"
   spec.add_dependency "uuidtools", "~> 2.2"
   spec.add_dependency "virustotalx", "~> 1.2"
   spec.add_dependency "whois", "~> 5.0"

data/sig/lib/mihari/analyzers/binaryedge.rbs

@@ -7,6 +7,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private

data/sig/lib/mihari/analyzers/censys.rbs

@@ -6,6 +6,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private

data/sig/lib/mihari/analyzers/onyphe.rbs

@@ -6,6 +6,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private

data/sig/lib/mihari/analyzers/shodan.rbs

@@ -6,6 +6,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private

data/sig/lib/mihari/analyzers/urlscan.rbs

@@ -8,7 +8,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
       attr_reader allowed_data_types: Array[String]
-      attr_reader use_similarity: bool
+
+      attr_reader interval: ::Integer
 
       def initialize: (*untyped args, **untyped kwargs) -> void
 
@@ -20,7 +21,9 @@ module Mihari
 
       def api: () -> untyped
 
-      def search: () -> Array[Hash[(String | Symbol), untyped]]
+      def search_with_search_after: (search_after: String?) -> Array[Hash[(String | Symbol), untyped]]
+
+      def search: () -> Array[Mihari::Structs::Urlscan::Response]
 
       def valid_alllowed_data_types?: () -> bool
     end

data/sig/lib/mihari/analyzers/virustotal_intelligence.rbs

@@ -6,6 +6,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
 
+      attr_reader interval: ::Integer
+
       def initialize: (*untyped args, **untyped kwargs) -> void
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])

data/sig/lib/mihari/analyzers/zoomeye.rbs

@@ -7,6 +7,8 @@ module Mihari
       attr_reader tags: Array[String]
       attr_reader type: String
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private

data/sig/lib/mihari/structs/greynoise.rbs

@@ -6,14 +6,14 @@ module Mihari
         attr_reader country_code: String
         attr_reader asn: String
 
-        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> untyped
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) ->  Mihari::Structs::GreyNoise::Metadata
       end
 
       class Datum < Dry::Struct
         attr_reader ip: String
         attr_reader metadata: Mihari::Structs::GreyNoise::Metadata
 
-        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> untyped
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::GreyNoise::Datum
       end
 
       class Response < Dry::Struct
@@ -23,7 +23,7 @@ module Mihari
         attr_reader message: String
         attr_reader query: String
 
-        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> untyped
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::GreyNoise::Response
       end
     end
   end

data/sig/lib/mihari/structs/urlscan.rbs

@@ -0,0 +1,28 @@
+module Mihari
+  module Structs
+    module Urlscan
+      class Page < Dry::Struct
+        attr_reader domain: String?
+        attr_reader ip: String?
+        attr_reader url: String
+
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Urlscan::Page
+      end
+
+      class Result < Dry::Struct
+        attr_reader page: Mihari::Structs::Urlscan::Page
+        attr_reader id: String
+        attr_reader sort: Array[Integer | String]
+
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Urlscan::Result
+      end
+
+      class Response < Dry::Struct
+        attr_reader results: Array[Mihari::Structs::Urlscan::Result]
+        attr_reader has_more: Boolean
+
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Urlscan::Response
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 3.10.1
+  version: 3.11.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-28 00:00:00.000000000 Z
+date: 2021-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
@@ -170,28 +170,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: steep
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.46'
+        version: '0.47'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.46'
+        version: '0.47'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -968,14 +968,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: uuidtools
   requirement: !ruby/object:Gem::Requirement
@@ -1177,6 +1177,7 @@ files:
 - lib/mihari/structs/ipinfo.rb
 - lib/mihari/structs/onyphe.rb
 - lib/mihari/structs/shodan.rb
+- lib/mihari/structs/urlscan.rb
 - lib/mihari/structs/virustotal_intelligence.rb
 - lib/mihari/templates/rule.yml.erb
 - lib/mihari/type_checker.rb
@@ -1351,6 +1352,7 @@ files:
 - sig/lib/mihari/structs/ipinfo.rbs
 - sig/lib/mihari/structs/onyphe.rbs
 - sig/lib/mihari/structs/shodan.rbs
+- sig/lib/mihari/structs/urlscan.rbs
 - sig/lib/mihari/structs/virustotal_intelligence.rbs
 - sig/lib/mihari/type_checker.rbs
 - sig/lib/mihari/types.rbs