mihari 3.9.2 → 3.12.0

data/lib/mihari.rb

@@ -37,6 +37,8 @@ module Mihari
   setting :censys_secret, default: ENV["CENSYS_SECRET"]
   setting :circl_passive_password, default: ENV["CIRCL_PASSIVE_PASSWORD"]
   setting :circl_passive_username, default: ENV["CIRCL_PASSIVE_USERNAME"]
+  setting :database, default: ENV["DATABASE"] || "mihari.db"
+  setting :greynoise_api_key, default: ENV["GREYNOISE_API_KEY"]
   setting :ipinfo_api_key, default: ENV["IPINFO_API_KEY"]
   setting :misp_api_endpoint, default: ENV["MISP_API_ENDPOINT"]
   setting :misp_api_key, default: ENV["MISP_API_KEY"]
@@ -54,10 +56,9 @@ module Mihari
   setting :thehive_api_key, default: ENV["THEHIVE_API_KEY"]
   setting :urlscan_api_key, default: ENV["URLSCAN_API_KEY"]
   setting :virustotal_api_key, default: ENV["VIRUSTOTAL_API_KEY"]
-  setting :zoomeye_api_key, default: ENV["ZOOMEYE_API_KEY"]
   setting :webhook_url, default: ENV["WEBHOOK_URL"]
   setting :webhook_use_json_body, constructor: ->(value = ENV["WEBHOOK_USE_JSON_BODY"]) { truthy?(value) }
-  setting :database, default: ENV["DATABASE"] || "mihari.db"
+  setting :zoomeye_api_key, default: ENV["ZOOMEYE_API_KEY"]
 
   class << self
     include Mem
@@ -112,9 +113,11 @@ require "mihari/types"
 # Structs
 require "mihari/structs/alert"
 require "mihari/structs/censys"
+require "mihari/structs/greynoise"
 require "mihari/structs/ipinfo"
 require "mihari/structs/onyphe"
 require "mihari/structs/shodan"
+require "mihari/structs/urlscan"
 require "mihari/structs/virustotal_intelligence"
 
 # Schemas
@@ -147,6 +150,8 @@ require "mihari/analyzers/circl"
 require "mihari/analyzers/crtsh"
 require "mihari/analyzers/dnpedia"
 require "mihari/analyzers/dnstwister"
+require "mihari/analyzers/feed"
+require "mihari/analyzers/greynoise"
 require "mihari/analyzers/onyphe"
 require "mihari/analyzers/otx"
 require "mihari/analyzers/passivetotal"

data/mihari.gemspec

@@ -25,9 +25,9 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.2"
+  spec.add_development_dependency "bundler", "~> 2.3"
   spec.add_development_dependency "coveralls_reborn", "~> 0.23"
-  spec.add_development_dependency "fakefs", "~> 1.3"
+  spec.add_development_dependency "fakefs", "~> 1.4"
   spec.add_development_dependency "mysql2", "~> 0.5"
   spec.add_development_dependency "overcommit", "~> 0.58"
   spec.add_development_dependency "pg", "~> 1.2"
@@ -36,14 +36,13 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rb-fsevent", "~> 0.11"
   spec.add_development_dependency "rerun", "~> 0.13"
   spec.add_development_dependency "rspec", "~> 3.10"
-  spec.add_development_dependency "standard", "~> 1.3"
-  spec.add_development_dependency "steep", "~> 0.46"
+  spec.add_development_dependency "standard", "~> 1.5"
+  spec.add_development_dependency "steep", "~> 0.47"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.0"
   spec.add_development_dependency "webmock", "~> 3.14"
 
-  spec.add_dependency "activerecord", "~> 6.1"
-  spec.add_dependency "activerecord-filter", "~> 6.1"
+  spec.add_dependency "activerecord", "7.0.0"
   spec.add_dependency "addressable", "~> 2.8"
   spec.add_dependency "awrence", "~> 1.2"
   spec.add_dependency "binaryedge", "~> 0.1"
@@ -54,19 +53,21 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dnpedia", "~> 0.1"
   spec.add_dependency "dnstwister", "~> 0.1"
   spec.add_dependency "dotenv", "~> 2.7"
-  spec.add_dependency "dry-configurable", "~> 0.13"
-  spec.add_dependency "dry-container", "~> 0.9"
-  spec.add_dependency "dry-files", "~> 0.1"
-  spec.add_dependency "dry-initializer", "~> 3.0"
-  spec.add_dependency "dry-struct", "~> 1.4"
-  spec.add_dependency "dry-validation", "~> 1.7"
+  spec.add_dependency "dry-configurable", "0.13.0"
+  spec.add_dependency "dry-container", "0.9.0"
+  spec.add_dependency "dry-files", "0.1.0"
+  spec.add_dependency "dry-initializer", "3.0.4"
+  spec.add_dependency "dry-struct", "1.4.0"
+  spec.add_dependency "dry-validation", "1.7.0"
   spec.add_dependency "email_address", "~> 0.2"
-  spec.add_dependency "grape", "~> 1.6"
-  spec.add_dependency "grape-entity", "~> 0.10"
-  spec.add_dependency "grape-swagger", "~> 1.4"
-  spec.add_dependency "grape-swagger-entity", "~> 0.5"
+  spec.add_dependency "grape", "1.6.2"
+  spec.add_dependency "grape-entity", "0.10.1"
+  spec.add_dependency "grape-swagger", "1.4.2"
+  spec.add_dependency "grape-swagger-entity", "0.5.1"
+  spec.add_dependency "greynoise", "~> 0.1"
   spec.add_dependency "hachi", "~> 1.0"
   spec.add_dependency "http", "~> 5.0"
+  spec.add_dependency "jr-cli", "~> 0.5"
   spec.add_dependency "launchy", "~> 2.5"
   spec.add_dependency "mem", "~> 0.1"
   spec.add_dependency "memist", "~> 2.0"
@@ -81,9 +82,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "plissken", "~> 1.4"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "pulsedive", "~> 0.1"
-  spec.add_dependency "puma", "~> 5.5"
-  spec.add_dependency "rack", "~> 2.2"
-  spec.add_dependency "rack-contrib", "~> 2.3"
+  spec.add_dependency "puma", "5.5.2"
+  spec.add_dependency "rack", "2.2.3"
+  spec.add_dependency "rack-contrib", "2.3.0"
   spec.add_dependency "rack-cors", "~> 1.1"
   spec.add_dependency "safe_shell", "~> 1.1"
   spec.add_dependency "securitytrails", "~> 1.0"
@@ -91,9 +92,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "slack-notifier", "~> 2.4"
   spec.add_dependency "spysex", "~> 0.2"
   spec.add_dependency "sqlite3", "~> 1.4"
-  spec.add_dependency "thor", "~> 1.1"
+  spec.add_dependency "thor", "1.1.0"
   spec.add_dependency "thread_safe", "~> 0.3"
-  spec.add_dependency "urlscan", "~> 0.7"
+  spec.add_dependency "urlscan", "~> 0.8"
   spec.add_dependency "uuidtools", "~> 2.2"
   spec.add_dependency "virustotalx", "~> 1.2"
   spec.add_dependency "whois", "~> 5.0"

data/sig/lib/mihari/analyzers/binaryedge.rbs

@@ -7,6 +7,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private

data/sig/lib/mihari/analyzers/censys.rbs

@@ -6,6 +6,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private

data/sig/lib/mihari/analyzers/feed.rbs

@@ -0,0 +1,23 @@
+module Mihari
+  module Analyzers
+    class Feed < Base
+      attr_reader query: String
+      attr_reader title: String
+      attr_reader description: String
+      attr_reader tags: Array[String]
+
+      attr_reader http_request_headers: Hash[(String | Symbol), untyped]
+      attr_reader http_request_method: String
+      attr_reader http_request_payload_type: String?
+      attr_reader http_request_payload: Hash[(String | Symbol), untyped]
+
+      attr_reader selector: String
+
+      def artifacts: () ->  (Array[String] | Array[Mihari::Artifact])
+
+      private
+
+      def data: () -> Array[Hash]
+    end
+  end
+end

data/sig/lib/mihari/analyzers/onyphe.rbs

@@ -6,6 +6,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private

data/sig/lib/mihari/analyzers/shodan.rbs

@@ -6,6 +6,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private

data/sig/lib/mihari/analyzers/urlscan.rbs

@@ -8,7 +8,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
       attr_reader allowed_data_types: Array[String]
-      attr_reader use_similarity: bool
+
+      attr_reader interval: ::Integer
 
       def initialize: (*untyped args, **untyped kwargs) -> void
 
@@ -20,7 +21,9 @@ module Mihari
 
       def api: () -> untyped
 
-      def search: () -> Array[Hash[(String | Symbol), untyped]]
+      def search_with_search_after: (search_after: String?) -> Array[Hash[(String | Symbol), untyped]]
+
+      def search: () -> Array[Mihari::Structs::Urlscan::Response]
 
       def valid_alllowed_data_types?: () -> bool
     end

data/sig/lib/mihari/analyzers/virustotal_intelligence.rbs

@@ -6,6 +6,8 @@ module Mihari
       attr_reader description: String
       attr_reader tags: Array[String]
 
+      attr_reader interval: ::Integer
+
       def initialize: (*untyped args, **untyped kwargs) -> void
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])

data/sig/lib/mihari/analyzers/zoomeye.rbs

@@ -7,6 +7,8 @@ module Mihari
       attr_reader tags: Array[String]
       attr_reader type: String
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private

data/sig/lib/mihari/cli/analyzer.rbs

@@ -13,6 +13,10 @@ module Mihari
 
       include Mihari::Commands::DNSTwister
 
+      include Mihari::Commands::Feed
+
+      include Mihari::Commands::GreyNoise
+
       include Mihari::Commands::JSON
 
       include Mihari::Commands::Onyphe

data/sig/lib/mihari/commands/feed.rbs

@@ -0,0 +1,7 @@
+module Mihari
+  module Commands
+    module Feed
+      def self.included: (untyped thor) -> untyped
+    end
+  end
+end

data/sig/lib/mihari/feed/parser.rbs

@@ -0,0 +1,11 @@
+module Mihari
+  module Feed
+    class Parser
+      attr_reader data: Array[Hash] | Array[Array[String]]
+
+      def initialize: (Array[Hash] | Array[Array[String]] data) -> void
+
+      def parse: (String selector) -> Array[String]
+    end
+  end
+end

data/sig/lib/mihari/feed/reader.rbs

@@ -0,0 +1,56 @@
+module Mihari
+  class FeedReader
+    attr_reader uri: URI
+
+    attr_reader http_request_headers: Hash[(String | Symbol), untyped]
+
+    attr_reader http_request_method: String
+
+    attr_reader http_request_payload_type: String?
+
+    attr_reader http_request_payload: Hash[(String | Symbol), untyped]
+
+    def initialize: (
+      String url,
+      ?http_request_headers: Hash[(String | Symbol), untyped]] http_request_headers,
+      ?http_request_method: ::String http_request_method,
+      ?http_request_payload_type: String?  http_request_payload_type,
+      ?http_request_payload: Hash[(String | Symbol), untyped] http_request_payload
+    ) -> void
+
+    def read: () -> Array[Hash]
+
+    def get: () -> Array[Hash]
+
+    def post: () -> Array[Hash]
+
+    #
+    # Convert text as JSON
+    #
+    # @param [String] text
+    #
+    # @return [Array<Hash>]
+    #
+    def convert_as_json: (String text) -> Array[Hash]
+
+    #
+    # Convert text as CSV
+    #
+    # @param [String] text
+    #
+    # @return [Array<Hash>]
+    #
+    def convert_as_csv: (String text) -> Array[Hash]
+
+    def https_options: () -> ({ use_ssl: ::TrueClass } | ::Hash[untyped, untyped])
+
+    #
+    # Make a HTTP request
+    #
+    # @param [Net::HTTPRequest] req
+    #
+    # @return [Array<Hash>]
+    #
+    def request: (Net::HTTPRequest req) -> Array[Hash]
+  end
+end

data/sig/lib/mihari/structs/alert.rbs

@@ -13,7 +13,7 @@ module Mihari
         attr_reader dns_record: String?
         attr_reader reverse_dns_name: String?
 
-        def has_valid_artifact_filters: () -> bool
+        def valid_artifact_filters?: () -> bool
       end
 
       class SearchFilterWithPagination

data/sig/lib/mihari/structs/greynoise.rbs

@@ -0,0 +1,30 @@
+module Mihari
+  module Structs
+    module GreyNoise
+      class Metadata < Dry::Struct
+        attr_reader country: String
+        attr_reader country_code: String
+        attr_reader asn: String
+
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) ->  Mihari::Structs::GreyNoise::Metadata
+      end
+
+      class Datum < Dry::Struct
+        attr_reader ip: String
+        attr_reader metadata: Mihari::Structs::GreyNoise::Metadata
+
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::GreyNoise::Datum
+      end
+
+      class Response < Dry::Struct
+        attr_reader complete: Boolean
+        attr_reader count: Integer
+        attr_reader data: Array[Mihari::Structs::GreyNoise::Danum]
+        attr_reader message: String
+        attr_reader query: String
+
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::GreyNoise::Response
+      end
+    end
+  end
+end

data/sig/lib/mihari/structs/shodan.rbs

@@ -2,14 +2,14 @@ module Mihari
   module Structs
     module Shodan
       class Location
-        attr_reader country_code: String
-        attr_reader country_name: String
+        attr_reader country_code: String?
+        attr_reader country_name: String?
 
         def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Shodan::Location
       end
 
       class Match
-        attr_reader asn: String
+        attr_reader asn: String?
         attr_reader hostnames: Array[String]
         attr_reader location: Mihari::Structs::Shodan::Location
         attr_reader domains: Array[String]

data/sig/lib/mihari/structs/urlscan.rbs

@@ -0,0 +1,28 @@
+module Mihari
+  module Structs
+    module Urlscan
+      class Page < Dry::Struct
+        attr_reader domain: String?
+        attr_reader ip: String?
+        attr_reader url: String
+
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Urlscan::Page
+      end
+
+      class Result < Dry::Struct
+        attr_reader page: Mihari::Structs::Urlscan::Page
+        attr_reader id: String
+        attr_reader sort: Array[Integer | String]
+
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Urlscan::Result
+      end
+
+      class Response < Dry::Struct
+        attr_reader results: Array[Mihari::Structs::Urlscan::Result]
+        attr_reader has_more: Boolean
+
+        def self.from_dynamic!: (Hash[(String | Symbol), untyped] d) -> Mihari::Structs::Urlscan::Response
+      end
+    end
+  end
+end

data/sig/lib/mihari/types.rbs

@@ -13,5 +13,9 @@ module Mihari
     DataTypes: Array[String]
 
     AnalyzerTypes: Array[String]
+
+    FeedHttpRequestMethods: Array[String]
+
+    FeedHttpRequestPayloadTypes: Array[String]
   end
 end