mihari 3.10.0 → 4.0.0

data/lib/mihari.rb

@@ -1,22 +1,52 @@
 # frozen_string_literal: true
 
-require "awrence"
-require "colorize"
+require "active_support"
+
+require "active_support/core_ext/hash"
+require "active_support/core_ext/integer/time"
+require "active_support/core_ext/numeric/time"
+
+require "active_record"
+
+# dry-rb
 require "dry/configurable"
 require "dry/files"
-require "mem"
+require "dry/initializer"
+require "dry/schema"
+require "dry/struct"
+require "dry/types"
+require "dry/validation"
+
+# standard & utility libs
+require "addressable/uri"
+require "awrence"
+require "colorize"
+require "email_address"
+require "ipaddr"
+require "json"
+require "memist"
+require "net/http"
+require "net/https"
+require "net/ping"
 require "plissken"
+require "public_suffix"
+require "resolv"
+require "uuidtools"
 require "yaml"
+require "parallel"
 
 # Load .env
 require "dotenv/load"
 
+# Mihari
+require "mihari/version"
+require "mihari/errors"
+
 # Mixins
 require "mihari/mixins/autonomous_system"
 require "mihari/mixins/configurable"
-require "mihari/mixins/configuration"
+require "mihari/mixins/database"
 require "mihari/mixins/disallowed_data_value"
-require "mihari/mixins/hash"
 require "mihari/mixins/refang"
 require "mihari/mixins/retriable"
 require "mihari/mixins/rule"
@@ -30,7 +60,6 @@ end
 
 module Mihari
   extend Dry::Configurable
-  extend Mixins::Configuration
 
   setting :binaryedge_api_key, default: ENV["BINARYEDGE_API_KEY"]
   setting :censys_id, default: ENV["CENSYS_ID"]
@@ -61,7 +90,7 @@ module Mihari
   setting :zoomeye_api_key, default: ENV["ZOOMEYE_API_KEY"]
 
   class << self
-    include Mem
+    include Memist::Memoizable
 
     def emitters
       []
@@ -77,30 +106,9 @@ module Mihari
       []
     end
     memoize :enrichers
-
-    #
-    # Load configuration from YAML file
-    #
-    # @param [String] path Path to YAML file
-    #
-    # @return [nil]
-    #
-    def load_config_from_yaml(path)
-      config = load_config(path)
-
-      # validate loaded yaml data
-      validate_config config
-
-      config.each do |key, value|
-        Mihari.config.send("#{key.downcase}=".to_sym, value)
-      end
-    end
   end
 end
 
-require "mihari/version"
-require "mihari/errors"
-
 require "mihari/database"
 require "mihari/type_checker"
 
@@ -116,12 +124,15 @@ require "mihari/structs/censys"
 require "mihari/structs/greynoise"
 require "mihari/structs/ipinfo"
 require "mihari/structs/onyphe"
+require "mihari/structs/rule"
 require "mihari/structs/shodan"
+require "mihari/structs/urlscan"
 require "mihari/structs/virustotal_intelligence"
 
 # Schemas
+require "mihari/schemas/macros"
+
 require "mihari/schemas/analyzer"
-require "mihari/schemas/configuration"
 require "mihari/schemas/rule"
 
 # Enrichers
@@ -135,6 +146,7 @@ require "mihari/models/autonomous_system"
 require "mihari/models/dns"
 require "mihari/models/geolocation"
 require "mihari/models/reverse_dns"
+require "mihari/models/rule"
 require "mihari/models/tag"
 require "mihari/models/tagging"
 require "mihari/models/whois"
@@ -149,6 +161,7 @@ require "mihari/analyzers/circl"
 require "mihari/analyzers/crtsh"
 require "mihari/analyzers/dnpedia"
 require "mihari/analyzers/dnstwister"
+require "mihari/analyzers/feed"
 require "mihari/analyzers/greynoise"
 require "mihari/analyzers/onyphe"
 require "mihari/analyzers/otx"
@@ -184,9 +197,4 @@ require "mihari/status"
 require "mihari/web/app"
 
 # CLIs
-require "mihari/cli/base"
-
-require "mihari/cli/analyzer"
-require "mihari/cli/init"
-
 require "mihari/cli/main"

data/mihari.gemspec

@@ -25,9 +25,9 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.2"
+  spec.add_development_dependency "bundler", "~> 2.3"
   spec.add_development_dependency "coveralls_reborn", "~> 0.23"
-  spec.add_development_dependency "fakefs", "~> 1.3"
+  spec.add_development_dependency "fakefs", "~> 1.4"
   spec.add_development_dependency "mysql2", "~> 0.5"
   spec.add_development_dependency "overcommit", "~> 0.58"
   spec.add_development_dependency "pg", "~> 1.2"
@@ -36,40 +36,38 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rb-fsevent", "~> 0.11"
   spec.add_development_dependency "rerun", "~> 0.13"
   spec.add_development_dependency "rspec", "~> 3.10"
-  spec.add_development_dependency "standard", "~> 1.3"
-  spec.add_development_dependency "steep", "~> 0.46"
+  spec.add_development_dependency "standard", "~> 1.5"
+  spec.add_development_dependency "steep", "~> 0.47"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.0"
   spec.add_development_dependency "webmock", "~> 3.14"
 
-  spec.add_dependency "activerecord", "~> 6.1"
-  spec.add_dependency "activerecord-filter", "~> 6.1"
+  spec.add_dependency "activerecord", "7.0.1"
   spec.add_dependency "addressable", "~> 2.8"
   spec.add_dependency "awrence", "~> 1.2"
   spec.add_dependency "binaryedge", "~> 0.1"
   spec.add_dependency "censysx", "~> 0.1"
   spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "crtsh-rb", "~> 0.3"
-  spec.add_dependency "cymbal", "~> 2.0"
   spec.add_dependency "dnpedia", "~> 0.1"
   spec.add_dependency "dnstwister", "~> 0.1"
   spec.add_dependency "dotenv", "~> 2.7"
-  spec.add_dependency "dry-configurable", "~> 0.13"
-  spec.add_dependency "dry-container", "~> 0.9"
-  spec.add_dependency "dry-files", "~> 0.1"
-  spec.add_dependency "dry-initializer", "~> 3.0"
-  spec.add_dependency "dry-struct", "~> 1.4"
-  spec.add_dependency "dry-validation", "~> 1.7"
+  spec.add_dependency "dry-configurable", "0.14.0"
+  spec.add_dependency "dry-container", "0.9.0"
+  spec.add_dependency "dry-files", "0.1.0"
+  spec.add_dependency "dry-initializer", "3.1.1"
+  spec.add_dependency "dry-struct", "1.4.0"
+  spec.add_dependency "dry-validation", "1.7.0"
   spec.add_dependency "email_address", "~> 0.2"
-  spec.add_dependency "grape", "~> 1.6"
-  spec.add_dependency "grape-entity", "~> 0.10"
-  spec.add_dependency "grape-swagger", "~> 1.4"
-  spec.add_dependency "grape-swagger-entity", "~> 0.5"
+  spec.add_dependency "grape", "1.6.2"
+  spec.add_dependency "grape-entity", "0.10.1"
+  spec.add_dependency "grape-swagger", "1.4.2"
+  spec.add_dependency "grape-swagger-entity", "0.5.1"
   spec.add_dependency "greynoise", "~> 0.1"
   spec.add_dependency "hachi", "~> 1.0"
   spec.add_dependency "http", "~> 5.0"
+  spec.add_dependency "jr-cli", "~> 0.5"
   spec.add_dependency "launchy", "~> 2.5"
-  spec.add_dependency "mem", "~> 0.1"
   spec.add_dependency "memist", "~> 2.0"
   spec.add_dependency "misp", "~> 0.1"
   spec.add_dependency "net-ping", "~> 2.0"
@@ -82,9 +80,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "plissken", "~> 1.4"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "pulsedive", "~> 0.1"
-  spec.add_dependency "puma", "~> 5.5"
-  spec.add_dependency "rack", "~> 2.2"
-  spec.add_dependency "rack-contrib", "~> 2.3"
+  spec.add_dependency "puma", "5.5.2"
+  spec.add_dependency "rack", "2.2.3"
+  spec.add_dependency "rack-contrib", "2.3.0"
   spec.add_dependency "rack-cors", "~> 1.1"
   spec.add_dependency "safe_shell", "~> 1.1"
   spec.add_dependency "securitytrails", "~> 1.0"
@@ -92,9 +90,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "slack-notifier", "~> 2.4"
   spec.add_dependency "spysex", "~> 0.2"
   spec.add_dependency "sqlite3", "~> 1.4"
-  spec.add_dependency "thor", "~> 1.1"
+  spec.add_dependency "thor", "1.1.0"
   spec.add_dependency "thread_safe", "~> 0.3"
-  spec.add_dependency "urlscan", "~> 0.7"
+  spec.add_dependency "urlscan", "~> 0.8"
   spec.add_dependency "uuidtools", "~> 2.2"
   spec.add_dependency "virustotalx", "~> 1.2"
   spec.add_dependency "whois", "~> 5.0"

data/sig/lib/mihari/analyzers/binaryedge.rbs

@@ -3,9 +3,8 @@ module Mihari
     class BinaryEdge < Base
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
+      attr_reader interval: ::Integer
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 

data/sig/lib/mihari/analyzers/censys.rbs

@@ -2,9 +2,8 @@ module Mihari
   module Analyzers
     class Censys < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
+      attr_reader interval: ::Integer
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 

data/sig/lib/mihari/analyzers/circl.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void

data/sig/lib/mihari/analyzers/crtsh.rbs

@@ -2,9 +2,7 @@ module Mihari
   module Analyzers
     class Crtsh < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader exclude_expired: bool
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])

data/sig/lib/mihari/analyzers/dnpedia.rbs

@@ -2,9 +2,6 @@ module Mihari
   module Analyzers
     class DNPedia < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
@@ -12,7 +9,7 @@ module Mihari
 
       def api: () -> untyped
 
-      def search: () -> Array[String]
+      def search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/dnstwister.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void

data/sig/lib/mihari/analyzers/feed.rbs

@@ -0,0 +1,20 @@
+module Mihari
+  module Analyzers
+    class Feed < Base
+      attr_reader query: String
+
+      attr_reader http_request_headers: Hash[(String | Symbol), untyped]
+      attr_reader http_request_method: String
+      attr_reader http_request_payload_type: String?
+      attr_reader http_request_payload: Hash[(String | Symbol), untyped]
+
+      attr_reader selector: String
+
+      def artifacts: () ->  (Array[String] | Array[Mihari::Artifact])
+
+      private
+
+      def data: () -> Array[Hash]
+    end
+  end
+end

data/sig/lib/mihari/analyzers/onyphe.rbs

@@ -2,9 +2,8 @@ module Mihari
   module Analyzers
     class Onyphe < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
+      attr_reader interval: ::Integer
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 

data/sig/lib/mihari/analyzers/otx.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void

data/sig/lib/mihari/analyzers/passivetotal.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void
@@ -25,9 +23,9 @@ module Mihari
 
       def passive_dns_search: () -> Array[String]
 
-      def reverse_whois_search: () -> Array[String]
+      def reverse_whois_search: () -> Array[Mihari::Artifact]
 
-      def ssl_search: () -> Array[String]
+      def ssl_search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/pulsedive.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void
@@ -21,7 +19,7 @@ module Mihari
 
       def valid_type?: () -> bool
 
-      def search: () -> Array[String]
+      def search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/securitytrails.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void
@@ -25,9 +23,9 @@ module Mihari
 
       def domain_search: () -> Array[String]
 
-      def ip_search: () -> Array[String]
+      def ip_search: () -> Array[Mihari::Artifact]
 
-      def mail_search: () -> Array[String]
+      def mail_search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/shodan.rbs

@@ -2,9 +2,8 @@ module Mihari
   module Analyzers
     class Shodan < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
+      attr_reader interval: ::Integer
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 

data/sig/lib/mihari/analyzers/spyse.rbs

@@ -2,9 +2,7 @@ module Mihari
   module Analyzers
     class Spyse < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
@@ -19,11 +17,11 @@ module Mihari
 
       def valid_type?: () -> bool
 
-      def domain_search: () -> Array[String]
+      def domain_search: () -> Array[Mihari::Artifact]
 
-      def ip_search: () -> Array[String]
+      def ip_search: () -> Array[Mihari::Artifact]
 
-      def search: () -> Array[String]
+      def search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/urlscan.rbs

@@ -4,11 +4,10 @@ module Mihari
   module Analyzers
     class Urlscan < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader allowed_data_types: Array[String]
-      attr_reader use_similarity: bool
+
+      attr_reader interval: ::Integer
 
       def initialize: (*untyped args, **untyped kwargs) -> void
 
@@ -20,7 +19,9 @@ module Mihari
 
       def api: () -> untyped
 
-      def search: () -> Array[Hash[(String | Symbol), untyped]]
+      def search_with_search_after: (search_after: String?) -> Array[Hash[(String | Symbol), untyped]]
+
+      def search: () -> Array[Mihari::Structs::Urlscan::Response]
 
       def valid_alllowed_data_types?: () -> bool
     end

data/sig/lib/mihari/analyzers/virustotal.rbs

@@ -2,9 +2,7 @@ module Mihari
   module Analyzers
     class VirusTotal < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       include Mixins::Refang
@@ -21,11 +19,11 @@ module Mihari
 
       def valid_type?: () -> bool
 
-      def search: () -> Array[String]
+      def search: () -> Array[Mihari::Artifact]
 
-      def domain_search: () -> Array[String]
+      def domain_search: () -> Array[Mihari::Artifact]
 
-      def ip_search: () -> Array[String]
+      def ip_search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/virustotal_intelligence.rbs

@@ -2,9 +2,8 @@ module Mihari
   module Analyzers
     class VirusTotalIntelligence < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
+      attr_reader interval: ::Integer
 
       def initialize: (*untyped args, **untyped kwargs) -> void
 

data/sig/lib/mihari/analyzers/zoomeye.rbs

@@ -2,11 +2,11 @@ module Mihari
   module Analyzers
     class ZoomEye < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
+      attr_reader interval: ::Integer
+
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
       private
@@ -19,7 +19,7 @@ module Mihari
 
       def api: () -> untyped
 
-      def convert_responses: (Array[Hash[(String | Symbol), untyped]] responses) -> Array[String]
+      def convert_responses: (Array[Hash[(String | Symbol), untyped]] responses) -> Array[Mihari::Artifact]
 
       def _host_search: (String query, ?page: ::Integer page) -> (Hash[(String | Symbol), untyped] | nil)
 

data/sig/lib/mihari/commands/init.rbs

@@ -1,8 +1,6 @@
 module Mihari
   module Commands
     module Initialization
-      include Mixins::Configuration
-
       include Mixins::Rule
 
       def self.included: (untyped thor) -> untyped

data/sig/lib/mihari/commands/validator.rbs

@@ -3,8 +3,6 @@ module Mihari
     module Validator
       include Mixins::Rule
 
-      include Mixins::Configuration
-
       def self.included: (untyped thor) -> untyped
     end
   end

data/sig/lib/mihari/emitters/slack.rbs

@@ -3,7 +3,6 @@ module Mihari
     class Attachment
       attr_reader data: String
       attr_reader data_type: String
-      # include Mem
 
       # extend Dry::Initializer
 

data/sig/lib/mihari/feed/parser.rbs

@@ -0,0 +1,11 @@
+module Mihari
+  module Feed
+    class Parser
+      attr_reader data: Array[Hash] | Array[Array[String]]
+
+      def initialize: (Array[Hash] | Array[Array[String]] data) -> void
+
+      def parse: (String selector) -> Array[String]
+    end
+  end
+end

data/sig/lib/mihari/feed/reader.rbs

@@ -0,0 +1,56 @@
+module Mihari
+  class FeedReader
+    attr_reader uri: URI
+
+    attr_reader http_request_headers: Hash[(String | Symbol), untyped]
+
+    attr_reader http_request_method: String
+
+    attr_reader http_request_payload_type: String?
+
+    attr_reader http_request_payload: Hash[(String | Symbol), untyped]
+
+    def initialize: (
+      String uri,
+      ?http_request_headers: Hash[(String | Symbol), untyped]] http_request_headers,
+      ?http_request_method: ::String http_request_method,
+      ?http_request_payload_type: String?  http_request_payload_type,
+      ?http_request_payload: Hash[(String | Symbol), untyped] http_request_payload
+    ) -> void
+
+    def read: () -> Array[Hash]
+
+    def get: () -> Array[Hash]
+
+    def post: () -> Array[Hash]
+
+    #
+    # Convert text as JSON
+    #
+    # @param [String] text
+    #
+    # @return [Array<Hash>]
+    #
+    def convert_as_json: (String text) -> Array[Hash]
+
+    #
+    # Convert text as CSV
+    #
+    # @param [String] text
+    #
+    # @return [Array<Hash>]
+    #
+    def convert_as_csv: (String text) -> Array[Hash]
+
+    def https_options: () -> ({ use_ssl: ::TrueClass } | ::Hash[untyped, untyped])
+
+    #
+    # Make a HTTP request
+    #
+    # @param [Net::HTTPRequest] req
+    #
+    # @return [Array<Hash>]
+    #
+    def request: (Net::HTTPRequest req) -> Array[Hash]
+  end
+end

data/sig/lib/mihari/mixins/disallowed_data_value.rbs

@@ -1,8 +1,6 @@
 module Mihari
   module Mixins
     module DisallowedDataValue
-      # include Mem
-
       #
       # Normalize a value as a disallowed data value
       #