mihari 3.12.0 → 4.1.2

data/lib/mihari.rb

@@ -1,22 +1,52 @@
 # frozen_string_literal: true
 
-require "awrence"
-require "colorize"
+require "active_support"
+
+require "active_support/core_ext/hash"
+require "active_support/core_ext/integer/time"
+require "active_support/core_ext/numeric/time"
+
+require "active_record"
+
+# dry-rb
 require "dry/configurable"
 require "dry/files"
-require "mem"
+require "dry/initializer"
+require "dry/schema"
+require "dry/struct"
+require "dry/types"
+require "dry/validation"
+
+# standard & utility libs
+require "addressable/uri"
+require "awrence"
+require "colorize"
+require "email_address"
+require "ipaddr"
+require "json"
+require "memist"
+require "net/http"
+require "net/https"
+require "net/ping"
 require "plissken"
+require "public_suffix"
+require "resolv"
+require "uuidtools"
 require "yaml"
+require "parallel"
 
 # Load .env
 require "dotenv/load"
 
+# Mihari
+require "mihari/version"
+require "mihari/errors"
+
 # Mixins
 require "mihari/mixins/autonomous_system"
 require "mihari/mixins/configurable"
-require "mihari/mixins/configuration"
+require "mihari/mixins/database"
 require "mihari/mixins/disallowed_data_value"
-require "mihari/mixins/hash"
 require "mihari/mixins/refang"
 require "mihari/mixins/retriable"
 require "mihari/mixins/rule"
@@ -30,7 +60,6 @@ end
 
 module Mihari
   extend Dry::Configurable
-  extend Mixins::Configuration
 
   setting :binaryedge_api_key, default: ENV["BINARYEDGE_API_KEY"]
   setting :censys_id, default: ENV["CENSYS_ID"]
@@ -61,7 +90,7 @@ module Mihari
   setting :zoomeye_api_key, default: ENV["ZOOMEYE_API_KEY"]
 
   class << self
-    include Mem
+    include Memist::Memoizable
 
     def emitters
       []
@@ -77,30 +106,9 @@ module Mihari
       []
     end
     memoize :enrichers
-
-    #
-    # Load configuration from YAML file
-    #
-    # @param [String] path Path to YAML file
-    #
-    # @return [nil]
-    #
-    def load_config_from_yaml(path)
-      config = load_config(path)
-
-      # validate loaded yaml data
-      validate_config config
-
-      config.each do |key, value|
-        Mihari.config.send("#{key.downcase}=".to_sym, value)
-      end
-    end
   end
 end
 
-require "mihari/version"
-require "mihari/errors"
-
 require "mihari/database"
 require "mihari/type_checker"
 
@@ -116,13 +124,15 @@ require "mihari/structs/censys"
 require "mihari/structs/greynoise"
 require "mihari/structs/ipinfo"
 require "mihari/structs/onyphe"
+require "mihari/structs/rule"
 require "mihari/structs/shodan"
 require "mihari/structs/urlscan"
 require "mihari/structs/virustotal_intelligence"
 
 # Schemas
+require "mihari/schemas/macros"
+
 require "mihari/schemas/analyzer"
-require "mihari/schemas/configuration"
 require "mihari/schemas/rule"
 
 # Enrichers
@@ -136,13 +146,13 @@ require "mihari/models/autonomous_system"
 require "mihari/models/dns"
 require "mihari/models/geolocation"
 require "mihari/models/reverse_dns"
+require "mihari/models/rule"
 require "mihari/models/tag"
 require "mihari/models/tagging"
 require "mihari/models/whois"
 
 # Analyzers
 require "mihari/analyzers/base"
-require "mihari/analyzers/basic"
 
 require "mihari/analyzers/binaryedge"
 require "mihari/analyzers/censys"
@@ -186,9 +196,4 @@ require "mihari/status"
 require "mihari/web/app"
 
 # CLIs
-require "mihari/cli/base"
-
-require "mihari/cli/analyzer"
-require "mihari/cli/init"
-
 require "mihari/cli/main"

data/mihari.gemspec

@@ -30,33 +30,32 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "fakefs", "~> 1.4"
   spec.add_development_dependency "mysql2", "~> 0.5"
   spec.add_development_dependency "overcommit", "~> 0.58"
-  spec.add_development_dependency "pg", "~> 1.2"
+  spec.add_development_dependency "pg", "~> 1.3"
   spec.add_development_dependency "rack-test", "~> 1.1"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rb-fsevent", "~> 0.11"
   spec.add_development_dependency "rerun", "~> 0.13"
-  spec.add_development_dependency "rspec", "~> 3.10"
-  spec.add_development_dependency "standard", "~> 1.5"
+  spec.add_development_dependency "rspec", "~> 3.11"
+  spec.add_development_dependency "standard", "~> 1.7"
   spec.add_development_dependency "steep", "~> 0.47"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 6.0"
   spec.add_development_dependency "webmock", "~> 3.14"
 
-  spec.add_dependency "activerecord", "7.0.0"
+  spec.add_dependency "activerecord", "7.0.2.2"
   spec.add_dependency "addressable", "~> 2.8"
-  spec.add_dependency "awrence", "~> 1.2"
+  spec.add_dependency "awrence", "~> 2.0"
   spec.add_dependency "binaryedge", "~> 0.1"
   spec.add_dependency "censysx", "~> 0.1"
   spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "crtsh-rb", "~> 0.3"
-  spec.add_dependency "cymbal", "~> 2.0"
   spec.add_dependency "dnpedia", "~> 0.1"
   spec.add_dependency "dnstwister", "~> 0.1"
   spec.add_dependency "dotenv", "~> 2.7"
-  spec.add_dependency "dry-configurable", "0.13.0"
+  spec.add_dependency "dry-configurable", "0.14.0"
   spec.add_dependency "dry-container", "0.9.0"
   spec.add_dependency "dry-files", "0.1.0"
-  spec.add_dependency "dry-initializer", "3.0.4"
+  spec.add_dependency "dry-initializer", "3.1.1"
   spec.add_dependency "dry-struct", "1.4.0"
   spec.add_dependency "dry-validation", "1.7.0"
   spec.add_dependency "email_address", "~> 0.2"
@@ -69,20 +68,19 @@ Gem::Specification.new do |spec|
   spec.add_dependency "http", "~> 5.0"
   spec.add_dependency "jr-cli", "~> 0.5"
   spec.add_dependency "launchy", "~> 2.5"
-  spec.add_dependency "mem", "~> 0.1"
   spec.add_dependency "memist", "~> 2.0"
   spec.add_dependency "misp", "~> 0.1"
   spec.add_dependency "net-ping", "~> 2.0"
-  spec.add_dependency "normalize_country", "0.3"
+  spec.add_dependency "normalize_country", "~> 0.3"
   spec.add_dependency "onyphe", "~> 2.0"
   spec.add_dependency "otx_ruby", "~> 0.9"
   spec.add_dependency "parallel", "~> 1.21"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
-  spec.add_dependency "plissken", "~> 1.4"
+  spec.add_dependency "plissken", "~> 2.0"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "pulsedive", "~> 0.1"
-  spec.add_dependency "puma", "5.5.2"
+  spec.add_dependency "puma", "5.6.2"
   spec.add_dependency "rack", "2.2.3"
   spec.add_dependency "rack-contrib", "2.3.0"
   spec.add_dependency "rack-cors", "~> 1.1"
@@ -92,7 +90,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "slack-notifier", "~> 2.4"
   spec.add_dependency "spysex", "~> 0.2"
   spec.add_dependency "sqlite3", "~> 1.4"
-  spec.add_dependency "thor", "1.1.0"
+  spec.add_dependency "thor", "1.2.1"
   spec.add_dependency "thread_safe", "~> 0.3"
   spec.add_dependency "urlscan", "~> 0.8"
   spec.add_dependency "uuidtools", "~> 2.2"

data/sig/lib/mihari/analyzers/binaryedge.rbs

@@ -3,9 +3,6 @@ module Mihari
     class BinaryEdge < Base
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
 
       attr_reader interval: ::Integer
 

data/sig/lib/mihari/analyzers/censys.rbs

@@ -2,9 +2,6 @@ module Mihari
   module Analyzers
     class Censys < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
 
       attr_reader interval: ::Integer
 

data/sig/lib/mihari/analyzers/circl.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void

data/sig/lib/mihari/analyzers/crtsh.rbs

@@ -2,9 +2,7 @@ module Mihari
   module Analyzers
     class Crtsh < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader exclude_expired: bool
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])

data/sig/lib/mihari/analyzers/dnpedia.rbs

@@ -2,9 +2,6 @@ module Mihari
   module Analyzers
     class DNPedia < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
 
@@ -12,7 +9,7 @@ module Mihari
 
       def api: () -> untyped
 
-      def search: () -> Array[String]
+      def search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/dnstwister.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void

data/sig/lib/mihari/analyzers/feed.rbs

@@ -2,9 +2,6 @@ module Mihari
   module Analyzers
     class Feed < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
 
       attr_reader http_request_headers: Hash[(String | Symbol), untyped]
       attr_reader http_request_method: String

data/sig/lib/mihari/analyzers/onyphe.rbs

@@ -2,9 +2,6 @@ module Mihari
   module Analyzers
     class Onyphe < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
 
       attr_reader interval: ::Integer
 

data/sig/lib/mihari/analyzers/otx.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void

data/sig/lib/mihari/analyzers/passivetotal.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void
@@ -25,9 +23,9 @@ module Mihari
 
       def passive_dns_search: () -> Array[String]
 
-      def reverse_whois_search: () -> Array[String]
+      def reverse_whois_search: () -> Array[Mihari::Artifact]
 
-      def ssl_search: () -> Array[String]
+      def ssl_search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/pulsedive.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void
@@ -21,7 +19,7 @@ module Mihari
 
       def valid_type?: () -> bool
 
-      def search: () -> Array[String]
+      def search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/securitytrails.rbs

@@ -4,9 +4,7 @@ module Mihari
       include Mixins::Refang
 
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def initialize: (*untyped args, **untyped kwargs) -> void
@@ -25,9 +23,9 @@ module Mihari
 
       def domain_search: () -> Array[String]
 
-      def ip_search: () -> Array[String]
+      def ip_search: () -> Array[Mihari::Artifact]
 
-      def mail_search: () -> Array[String]
+      def mail_search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/shodan.rbs

@@ -2,9 +2,6 @@ module Mihari
   module Analyzers
     class Shodan < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
 
       attr_reader interval: ::Integer
 
@@ -22,6 +19,8 @@ module Mihari
 
       def search: () -> Array[Hash[(String | Symbol), untyped]]
 
+      def collect_metadata_by_ip: (Array[Mihari::Structs::Shodan::Match] matches, ip: String) -> Array[Hash[(String | Symbol), untyped]]
+
       #
       # Build an artifact from a Shodan search API response
       #

data/sig/lib/mihari/analyzers/spyse.rbs

@@ -2,9 +2,7 @@ module Mihari
   module Analyzers
     class Spyse < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       def artifacts: () -> (Array[String] | Array[Mihari::Artifact])
@@ -19,11 +17,11 @@ module Mihari
 
       def valid_type?: () -> bool
 
-      def domain_search: () -> Array[String]
+      def domain_search: () -> Array[Mihari::Artifact]
 
-      def ip_search: () -> Array[String]
+      def ip_search: () -> Array[Mihari::Artifact]
 
-      def search: () -> Array[String]
+      def search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/urlscan.rbs

@@ -4,9 +4,7 @@ module Mihari
   module Analyzers
     class Urlscan < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader allowed_data_types: Array[String]
 
       attr_reader interval: ::Integer

data/sig/lib/mihari/analyzers/virustotal.rbs

@@ -2,9 +2,7 @@ module Mihari
   module Analyzers
     class VirusTotal < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       include Mixins::Refang
@@ -21,11 +19,11 @@ module Mihari
 
       def valid_type?: () -> bool
 
-      def search: () -> Array[String]
+      def search: () -> Array[Mihari::Artifact]
 
-      def domain_search: () -> Array[String]
+      def domain_search: () -> Array[Mihari::Artifact]
 
-      def ip_search: () -> Array[String]
+      def ip_search: () -> Array[Mihari::Artifact]
     end
   end
 end

data/sig/lib/mihari/analyzers/virustotal_intelligence.rbs

@@ -2,9 +2,6 @@ module Mihari
   module Analyzers
     class VirusTotalIntelligence < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
 
       attr_reader interval: ::Integer
 

data/sig/lib/mihari/analyzers/zoomeye.rbs

@@ -2,9 +2,7 @@ module Mihari
   module Analyzers
     class ZoomEye < Base
       attr_reader query: String
-      attr_reader title: String
-      attr_reader description: String
-      attr_reader tags: Array[String]
+
       attr_reader type: String
 
       attr_reader interval: ::Integer
@@ -21,7 +19,7 @@ module Mihari
 
       def api: () -> untyped
 
-      def convert_responses: (Array[Hash[(String | Symbol), untyped]] responses) -> Array[String]
+      def convert_responses: (Array[Hash[(String | Symbol), untyped]] responses) -> Array[Mihari::Artifact]
 
       def _host_search: (String query, ?page: ::Integer page) -> (Hash[(String | Symbol), untyped] | nil)
 

data/sig/lib/mihari/commands/init.rbs

@@ -1,8 +1,6 @@
 module Mihari
   module Commands
     module Initialization
-      include Mixins::Configuration
-
       include Mixins::Rule
 
       def self.included: (untyped thor) -> untyped

data/sig/lib/mihari/commands/validator.rbs

@@ -3,8 +3,6 @@ module Mihari
     module Validator
       include Mixins::Rule
 
-      include Mixins::Configuration
-
       def self.included: (untyped thor) -> untyped
     end
   end

data/sig/lib/mihari/emitters/slack.rbs

@@ -3,7 +3,6 @@ module Mihari
     class Attachment
       attr_reader data: String
       attr_reader data_type: String
-      # include Mem
 
       # extend Dry::Initializer
 

data/sig/lib/mihari/feed/reader.rbs

@@ -11,7 +11,7 @@ module Mihari
     attr_reader http_request_payload: Hash[(String | Symbol), untyped]
 
     def initialize: (
-      String url,
+      String uri,
       ?http_request_headers: Hash[(String | Symbol), untyped]] http_request_headers,
       ?http_request_method: ::String http_request_method,
       ?http_request_payload_type: String?  http_request_payload_type,

data/sig/lib/mihari/mixins/disallowed_data_value.rbs

@@ -1,8 +1,6 @@
 module Mihari
   module Mixins
     module DisallowedDataValue
-      # include Mem
-
       #
       # Normalize a value as a disallowed data value
       #

data/sig/lib/mihari/mixins/rule.rbs

@@ -4,20 +4,11 @@ module Mihari
       #
       # Load rule into hash
       #
-      # @param [String] path Path to YAML file or YAML string
+      # @param [String] path_or_id
       #
-      # @return [Hash]
+      # @return [Mihari::Structs::Rule::Rule]
       #
-      def load_rule: (String path) -> Hash
-
-      #
-      # Validate rule schema and return a normalized rule
-      #
-      # @param [Hash] rule
-      #
-      # @return [Hash]
-      #
-      def validate_rule: (Hash rule) -> Hash
+      def load_rule: (String path_or_id) -> Mihari::Structs::Rule::Rule
 
       #
       # Returns a template for rule
@@ -26,6 +17,8 @@ module Mihari
       #
       def rule_template: () -> String
 
+      def validate_rule!: (Mihari::Structs::Rule::Rule rule) -> nil
+
       #
       # Create (blank) rule file
       #

data/sig/lib/mihari/models/alert.rbs

@@ -7,7 +7,7 @@ module Mihari
     attr_accessor artifacts(): Array[Mihari::Artifact]
     attr_accessor Tags(): Array[Mihari::Tag]
 
-    def self.search: (Mihari::Structs::Alert::SearchFilterWithPagination filter) -> Array[Hash[(String | Symbol), untyped]]
+    def self.search: (Mihari::Structs::Alert::SearchFilterWithPagination filter) -> Array[Mihari::Alert]
 
     def self.count: (Mihari::Structs::Alert::SearchFilter filter) -> Integer
 

data/sig/lib/mihari/models/artifact.rbs

@@ -7,6 +7,8 @@ module Mihari
     attr_accessor data (): String
     attr_accessor data_type (): String
 
+    attr_accessor metadata(): Hash?
+
     attr_accessor autonomous_system(): Mihari::AutonomousSystem
     attr_accessor geolocation (): Mihari::Geolocation
     attr_accessor whois_record(): Mihari::WhoisRecord

data/sig/lib/mihari/models/rule.rbs

@@ -0,0 +1,14 @@
+module Mihari
+  class Rule < ActiveRecord::Base
+    attr_accessor id(): String
+    attr_accessor title(): String
+    attr_accessor description(): String
+    attr_accessor data(): Hash
+
+    def to_h: () -> Hash
+
+    def self.search: (Mihari::Structs::Rule::SearchFilterWithPagination filter) -> Array[Mihari::Rule]
+
+    def self.count: (Mihari::Structs::Rule::SearchFilter filter) -> Integer
+  end
+end