mihari 7.3.1 → 7.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed9b24457d01edc4d1643e6c31c654ad7c13bf71fc849b10bb02276abf45852c
-  data.tar.gz: 1be36d2083e0b0209ad16475a8f20eb4d2ee9bfcb37eae43fa76091207526def
+  metadata.gz: 9ec4774493a408eb666c7a33e671c977f7c400356758aab00ba776b36910bc42
+  data.tar.gz: bf0e0269c1e12d73b064d06ebf41e10686caeb66aaf70ec39f4e3ce7843bc51a
 SHA512:
-  metadata.gz: 4482938e33386e24054cb215f78f065e3190ba32269872aea6a9f543745a2c71777bb609120dcaa3872dba9b6ebd307651f80239342d6ac9427db205f03c80e0
-  data.tar.gz: 5723cbca9f18c519fc4cf91775d751f417a85161add0c0c6d499fd03075c7985c1cb73ef68e24b4913b1d8d3a9652a0e4e1025a32b83c987d3e9e22886923e38
+  metadata.gz: 6dff8b5b3bcd3098bb90e84f1d026325ca5a24d2cffb30761e2c243dfbc81bfef8093bd885ef7e9b07a2338ef2ca1cba10686fd7612a6261205c08f0b9258a15
+  data.tar.gz: 6958e9d9e344b98c29209ce4d5501ded9b8584367cf5556c99e351913e0fe8800d829ccedb53704951a7b29eb5b73950e071ff2460e95ad067aff81dbef2c81d

data/.gitignore

@@ -63,3 +63,11 @@ lib/mihari/web/public/
 # Rules & Alerts
 rules/
 alerts/
+
+# Python Environments
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/

data/.rubocop.yml

@@ -1,5 +1,3 @@
-Style/HashSyntax:
-  EnforcedShorthandSyntax: either
 Style/StringLiterals:
   EnforcedStyle: double_quotes
 Metrics/BlockLength:

data/.shadowenv.d/.gitignore

@@ -0,0 +1,2 @@
+/.*
+!/.gitignore

data/.shadowenv.d/000_unset_all.lisp

@@ -0,0 +1,39 @@
+(env/set "APP_ENV" ())
+(env/set "BINARYEDGE_API_KEY" ())
+(env/set "CENSYS_ID" ())
+(env/set "CENSYS_SECRET" ())
+(env/set "CIRCL_PASSIVE_PASSWORD" ())
+(env/set "CIRCL_PASSIVE_USERNAME" ())
+(env/set "DATABASE_URL" ())
+(env/set "FOFA_API_KEY" ())
+(env/set "FOFA_EMAIL" ())
+(env/set "GREYNOISE_API_KEY" ())
+(env/set "HIDE_CONFIG_VALUES" ())
+(env/set "HUNTERHOW_API_KEY" ())
+(env/set "IGNORE_ERROR" ())
+(env/set "MISP_API_KEY" ())
+(env/set "MISP_URL" ())
+(env/set "ONYPHE_API_KEY" ())
+(env/set "OTX_API_KEY" ())
+(env/set "PAGINATION_INTERVAL" ())
+(env/set "PAGINATION_LIMIT" ())
+(env/set "PARALLEL" ())
+(env/set "PASSIVETOTAL_API_KEY" ())
+(env/set "PASSIVETOTAL_USERNAME" ())
+(env/set "PULSEDIVE_API_KEY" ())
+(env/set "RACK_ENV" ())
+(env/set "RETRY_EXPONENTIAL_BACKOFF" ())
+(env/set "RETRY_INTERVAL" ())
+(env/set "RETRY_TIMES" ())
+(env/set "SECURITYTRAILS_API_KEY" ())
+(env/set "SENTRY_DSN" ())
+(env/set "SENTRY_TRACE_SAMPLE_RATE" ())
+(env/set "SHODAN_API_KEY" ())
+(env/set "SIDEKIQ_REDIS_URL" ())
+(env/set "SLACK_CHANNEL" ())
+(env/set "SLACK_WEBHOOK_URL" ())
+(env/set "THEHIVE_API_KEY" ())
+(env/set "THEHIVE_URL" ())
+(env/set "URLSCAN_API_KEY" ())
+(env/set "VIRUSTOTAL_API_KEY" ())
+(env/set "ZOOMEYE_API_KEY" ())

data/README.md

@@ -6,14 +6,6 @@
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 
----
-
-<p align="center">
-  <img src="https://github.com/ninoseki/mihari/raw/master/images/logo.png"/>
-</p>
-
----
-
 A query aggregator for OSINT based threat hunting.
 
 Mihari can aggregate multiple searches across multiple services in a single rule & persist findings in a database.
@@ -40,6 +32,8 @@ Mihari supports the following services by default.
 
 See [documentation](https://ninoseki.github.io/mihari/) for more details.
 
+You can also refer to [JSAC2024 workshop materials](https://ninoseki.github.io/jsac_mihari_workshop/) to learn how Mihari works through some exercises.
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/mihari/analyzers/base.rb

@@ -14,7 +14,7 @@ module Mihari
       # @param [Hash, nil] options
       #
       def initialize(query, options: nil)
-        super(options: options)
+        super(options:)
 
         @query = query
       end
@@ -44,7 +44,7 @@ module Mihari
       # @return [Boolean]
       #
       def parallel?
-        options[:parallel] || Mihari.config.parallel
+        options[:parallel] || Mihari.config.analyzer_parallelism
       end
 
       # @return [Array<String>, Array<Mihari::Models::Artifact>]

data/lib/mihari/analyzers/binaryedge.rb

@@ -15,13 +15,13 @@ module Mihari
       # @param [String, nil] api_key
       #
       def initialize(query, options: nil, api_key: nil)
-        super(query, options: options)
+        super(query, options:)
 
         @api_key = api_key || Mihari.config.binaryedge_api_key
       end
 
       def artifacts
-        client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
+        client.search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
       end
 
       private
@@ -32,9 +32,9 @@ module Mihari
       #
       def client
         Clients::BinaryEdge.new(
-          api_key: api_key,
-          pagination_interval: pagination_interval,
-          timeout: timeout
+          api_key:,
+          pagination_interval:,
+          timeout:
         )
       end
     end

data/lib/mihari/analyzers/censys.rb

@@ -19,7 +19,7 @@ module Mihari
       # @param [String, nil] secret
       #
       def initialize(query, options: nil, id: nil, secret: nil)
-        super(query, options: options)
+        super(query, options:)
 
         @id = id || Mihari.config.censys_id
         @secret = secret || Mihari.config.censys_secret
@@ -29,7 +29,7 @@ module Mihari
       # @return [Array<Mihari::Models::Artifact>]
       #
       def artifacts
-        client.search_with_pagination(query, pagination_limit: pagination_limit).map do |res|
+        client.search_with_pagination(query, pagination_limit:).map do |res|
           res.result.artifacts
         end.flatten.uniq(&:data)
       end
@@ -48,10 +48,10 @@ module Mihari
       #
       def client
         Clients::Censys.new(
-          id: id,
-          secret: secret,
-          pagination_interval: pagination_interval,
-          timeout: timeout
+          id:,
+          secret:,
+          pagination_interval:,
+          timeout:
         )
       end
 

data/lib/mihari/analyzers/circl.rb

@@ -24,7 +24,7 @@ module Mihari
       # @param [String, nil] password
       #
       def initialize(query, options: nil, username: nil, password: nil)
-        super(refang(query), options: options)
+        super(refang(query), options:)
 
         @type = DataType.type(query)
 
@@ -50,7 +50,7 @@ module Mihari
       private
 
       def client
-        Clients::CIRCL.new(username: username, password: password, timeout: timeout)
+        Clients::CIRCL.new(username:, password:, timeout:)
       end
 
       def username?

data/lib/mihari/analyzers/crtsh.rb

@@ -19,7 +19,7 @@ module Mihari
       # @param [String, nil] match
       #
       def initialize(query, options: nil, exclude_expired: true, match: nil)
-        super(query, options: options)
+        super(query, options:)
 
         @exclude_expired = exclude_expired
         @match = match
@@ -27,7 +27,7 @@ module Mihari
 
       def artifacts
         exclude = exclude_expired ? "expired" : nil
-        client.search(query, exclude: exclude, match: match).map do |result|
+        client.search(query, exclude:, match:).map do |result|
           values = result["name_value"].to_s.lines.map(&:chomp).reject { |value| value.starts_with?("*.") }
           values.map { |value| Models::Artifact.new(data: value, metadata: result) }
         end.flatten
@@ -39,7 +39,7 @@ module Mihari
       # @return [Mihari::Clients::Crtsh]
       #
       def client
-        Mihari::Clients::Crtsh.new(timeout: timeout)
+        Mihari::Clients::Crtsh.new(timeout:)
       end
     end
   end

data/lib/mihari/analyzers/dnstwister.rb

@@ -16,7 +16,7 @@ module Mihari
       # @param [Hash, nil] options
       #
       def initialize(query, options: nil)
-        super(refang(query), options: options)
+        super(refang(query), options:)
 
         @type = DataType.type(query)
       end
@@ -40,7 +40,7 @@ module Mihari
       end
 
       def client
-        Clients::DNSTwister.new(timeout: timeout)
+        Clients::DNSTwister.new(timeout:)
       end
 
       #

data/lib/mihari/analyzers/feed.rb

@@ -28,30 +28,24 @@ module Mihari
       attr_reader :query
 
       #
-      # @param [String] query
+      # @param [String] url
       # @param [Hash, nil] options
-      # @param [String] method
-      # @param [Hash, nil] headers
-      # @param [Hash, nil] params
-      # @param [Hash, nil] json
-      # @param [form, nil] form
-      # @param [String] selector
+      # @param [Hash] params
       #
-      # @param [Object] url
-      def initialize(url, options: nil, method: "GET", headers: nil, params: nil, json: nil, form: nil, selector: "")
-        super(url, options: options)
-
-        @method = method
-        @headers = headers || {}
-        @params = params
-        @json = json
-        @form = form
-        @selector = selector
+      def initialize(url, options: nil, **params)
+        super(url, options:)
+
+        @method = params[:method] || "GET"
+        @headers = params[:headers] || {}
+        @params = params[:params]
+        @json = params[:json]
+        @form = params[:form]
+        @selector = params[:selector] || ""
       end
 
       def artifacts
         data = Services::FeedReader.call(
-          url, headers: headers, method: method, params: params, json: json, form: form, timeout: timeout
+          url, headers:, method:, params:, json:, form:, timeout:
         )
         Services::FeedParser.call(data, selector)
       end

data/lib/mihari/analyzers/fofa.rb

@@ -19,14 +19,14 @@ module Mihari
       # @param [String, nil] email
       #
       def initialize(query, options: nil, api_key: nil, email: nil)
-        super(query, options: options)
+        super(query, options:)
 
         @api_key = api_key || Mihari.config.fofa_api_key
         @email = email || Mihari.config.fofa_email
       end
 
       def artifacts
-        client.search_with_pagination(query, pagination_limit: pagination_limit).map do |res|
+        client.search_with_pagination(query, pagination_limit:).map do |res|
           (res.results || []).map { |result| result[1] }
         end.flatten.compact
       end
@@ -46,10 +46,10 @@ module Mihari
       #
       def client
         Clients::Fofa.new(
-          api_key: api_key,
-          email: email,
-          pagination_interval: pagination_interval,
-          timeout: timeout
+          api_key:,
+          email:,
+          pagination_interval:,
+          timeout:
         )
       end
     end

data/lib/mihari/analyzers/greynoise.rb

@@ -15,7 +15,7 @@ module Mihari
       # @param [String, nil] api_key
       #
       def initialize(query, options: nil, api_key: nil)
-        super(query, options: options)
+        super(query, options:)
 
         @api_key = api_key || Mihari.config.greynoise_api_key
       end
@@ -23,7 +23,7 @@ module Mihari
       def artifacts
         client.gnql_search_with_pagination(
           query,
-          pagination_limit: pagination_limit
+          pagination_limit:
         ).map(&:artifacts).flatten
       end
 
@@ -31,9 +31,9 @@ module Mihari
 
       def client
         Clients::GreyNoise.new(
-          api_key: api_key,
-          pagination_interval: pagination_interval,
-          timeout: timeout
+          api_key:,
+          pagination_interval:,
+          timeout:
         )
       end
     end

data/lib/mihari/analyzers/hunterhow.rb

@@ -23,7 +23,7 @@ module Mihari
       # @param [String, nil] api_key
       #
       def initialize(query, start_time: nil, end_time: nil, options: nil, api_key: nil)
-        super(query, options: options)
+        super(query, options:)
 
         @api_key = api_key || Mihari.config.hunterhow_api_key
 
@@ -48,9 +48,9 @@ module Mihari
 
       def client
         Clients::HunterHow.new(
-          api_key: api_key,
-          pagination_interval: pagination_interval,
-          timeout: timeout
+          api_key:,
+          pagination_interval:,
+          timeout:
         )
       end
     end

data/lib/mihari/analyzers/onyphe.rb

@@ -17,7 +17,7 @@ module Mihari
       # @param [String, nil] api_key
       #
       def initialize(query, options: nil, api_key: nil)
-        super(query, options: options)
+        super(query, options:)
 
         @api_key = api_key || Mihari.config.onyphe_api_key
       end
@@ -25,7 +25,7 @@ module Mihari
       def artifacts
         client.datascan_with_pagination(
           query,
-          pagination_limit: pagination_limit
+          pagination_limit:
         ).map(&:artifacts).flatten
       end
 
@@ -33,9 +33,9 @@ module Mihari
 
       def client
         Clients::Onyphe.new(
-          api_key: api_key,
-          pagination_interval: pagination_interval,
-          timeout: timeout
+          api_key:,
+          pagination_interval:,
+          timeout:
         )
       end
     end

data/lib/mihari/analyzers/otx.rb

@@ -20,7 +20,7 @@ module Mihari
       # @param [String, nil] api_key
       #
       def initialize(query, options: nil, api_key: nil)
-        super(refang(query), options: options)
+        super(refang(query), options:)
 
         @type = DataType.type(query)
 
@@ -41,7 +41,7 @@ module Mihari
       private
 
       def client
-        Mihari::Clients::OTX.new(api_key: api_key, timeout: timeout)
+        Mihari::Clients::OTX.new(api_key:, timeout:)
       end
 
       #

data/lib/mihari/analyzers/passivetotal.rb

@@ -24,7 +24,7 @@ module Mihari
       # @param [String, nil] username
       #
       def initialize(query, options: nil, api_key: nil, username: nil)
-        super(refang(query), options: options)
+        super(refang(query), options:)
 
         @type = DataType.type(query)
 
@@ -69,7 +69,7 @@ module Mihari
         res = client.reverse_whois_search(query)
         (res["results"] || []).map do |result|
           data = result["domain"]
-          Models::Artifact.new(data: data, metadata: result)
+          Models::Artifact.new(data:, metadata: result)
         end
       end
 
@@ -82,7 +82,7 @@ module Mihari
       end
 
       def client
-        Clients::PassiveTotal.new(username: username, api_key: api_key, timeout: timeout)
+        Clients::PassiveTotal.new(username:, api_key:, timeout:)
       end
 
       #

data/lib/mihari/analyzers/pulsedive.rb

@@ -20,7 +20,7 @@ module Mihari
       # @param [String, nil] api_key
       #
       def initialize(query, options: nil, api_key: nil)
-        super(refang(query), options: options)
+        super(refang(query), options:)
 
         @type = DataType.type(query)
 
@@ -38,7 +38,7 @@ module Mihari
             nil
           else
             data = property["value"]
-            Models::Artifact.new(data: data, metadata: property)
+            Models::Artifact.new(data:, metadata: property)
           end
         end
       end
@@ -46,7 +46,7 @@ module Mihari
       private
 
       def client
-        @client ||= Clients::PulseDive.new(api_key: api_key, timeout: timeout)
+        @client ||= Clients::PulseDive.new(api_key:, timeout:)
       end
 
       #

data/lib/mihari/analyzers/securitytrails.rb

@@ -23,7 +23,7 @@ module Mihari
       # @param [String, nil] api_key
       #
       def initialize(query, options: nil, api_key: nil)
-        super(refang(query), options: options)
+        super(refang(query), options:)
 
         @type = DataType.type(query)
 
@@ -66,7 +66,7 @@ module Mihari
         res = client.ip_search(query)
         (res["records"] || []).filter_map do |record|
           data = record["hostname"]
-          Models::Artifact.new(data: data, metadata: record)
+          Models::Artifact.new(data:, metadata: record)
         end
       end
 
@@ -74,12 +74,12 @@ module Mihari
         res = client.mail_search(query)
         (res["records"] || []).filter_map do |record|
           data = record["hostname"]
-          Models::Artifact.new(data: data, metadata: record)
+          Models::Artifact.new(data:, metadata: record)
         end
       end
 
       def client
-        Clients::SecurityTrails.new(api_key: api_key, timeout: timeout)
+        Clients::SecurityTrails.new(api_key:, timeout:)
       end
 
       #

data/lib/mihari/analyzers/shodan.rb

@@ -15,7 +15,7 @@ module Mihari
       # @param [String, nil] api_key
       #
       def initialize(query, options: nil, api_key: nil)
-        super(query, options: options)
+        super(query, options:)
 
         @api_key = api_key || Mihari.config.shodan_api_key
       end
@@ -23,7 +23,7 @@ module Mihari
       def artifacts
         client.search_with_pagination(
           query,
-          pagination_limit: pagination_limit
+          pagination_limit:
         ).map(&:artifacts).flatten.uniq(&:data)
       end
 
@@ -34,9 +34,9 @@ module Mihari
       #
       def client
         Clients::Shodan.new(
-          api_key: api_key,
-          pagination_interval: pagination_interval,
-          timeout: timeout
+          api_key:,
+          pagination_interval:,
+          timeout:
         )
       end
     end

data/lib/mihari/analyzers/urlscan.rb

@@ -21,7 +21,7 @@ module Mihari
       # @param [Array<String>] allowed_data_types
       #
       def initialize(query, options: nil, api_key: nil, allowed_data_types: SUPPORTED_DATA_TYPES)
-        super(query, options: options)
+        super(query, options:)
 
         @api_key = api_key || Mihari.config.urlscan_api_key
         @allowed_data_types = allowed_data_types
@@ -33,7 +33,7 @@ module Mihari
 
       def artifacts
         # @type [Array<Mihari::Models::Artifact>]
-        artifacts = client.search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
+        artifacts = client.search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
         artifacts.select { |artifact| allowed_data_types.include? artifact.data_type }
       end
 
@@ -41,9 +41,9 @@ module Mihari
 
       def client
         Clients::Urlscan.new(
-          api_key: api_key,
-          pagination_interval: pagination_interval,
-          timeout: timeout
+          api_key:,
+          pagination_interval:,
+          timeout:
         )
       end
 

data/lib/mihari/analyzers/virustotal.rb

@@ -20,7 +20,7 @@ module Mihari
       # @param [String, nil] api_key
       #
       def initialize(query, options: nil, api_key: nil)
-        super(refang(query), options: options)
+        super(refang(query), options:)
 
         @type = DataType.type(query)
 
@@ -50,7 +50,7 @@ module Mihari
       private
 
       def client
-        Clients::VirusTotal.new(api_key: api_key)
+        Clients::VirusTotal.new(api_key:)
       end
 
       #
@@ -73,7 +73,7 @@ module Mihari
         data = res["data"] || []
         data.filter_map do |item|
           data = item.dig("attributes", "ip_address")
-          data.nil? ? nil : Models::Artifact.new(data: data, metadata: item)
+          data.nil? ? nil : Models::Artifact.new(data:, metadata: item)
         end
       end
 
@@ -88,7 +88,7 @@ module Mihari
         data = res["data"] || []
         data.filter_map do |item|
           data = item.dig("attributes", "host_name")
-          Models::Artifact.new(data: data, metadata: item)
+          Models::Artifact.new(data:, metadata: item)
         end.uniq
       end
     end

data/lib/mihari/analyzers/virustotal_intelligence.rb

@@ -15,13 +15,13 @@ module Mihari
       # @param [String, nil] api_key
       #
       def initialize(query, options: nil, api_key: nil)
-        super(query, options: options)
+        super(query, options:)
 
         @api_key = api_key || Mihari.config.virustotal_api_key
       end
 
       def artifacts
-        client.intel_search_with_pagination(query, pagination_limit: pagination_limit).map(&:artifacts).flatten
+        client.intel_search_with_pagination(query, pagination_limit:).map(&:artifacts).flatten
       end
 
       class << self
@@ -56,9 +56,9 @@ module Mihari
       #
       def client
         Clients::VirusTotal.new(
-          api_key: api_key,
-          pagination_interval: pagination_interval,
-          timeout: timeout
+          api_key:,
+          pagination_interval:,
+          timeout:
         )
       end
     end

data/lib/mihari/analyzers/zoomeye.rb

@@ -19,7 +19,7 @@ module Mihari
       # @param [String] type
       #
       def initialize(query, options: nil, api_key: nil, type: "host")
-        super(query, options: options)
+        super(query, options:)
 
         @type = type
         @api_key = api_key || Mihari.config.zoomeye_api_key
@@ -53,9 +53,9 @@ module Mihari
 
       def client
         Clients::ZoomEye.new(
-          api_key: api_key,
-          pagination_interval: pagination_interval,
-          timeout: timeout
+          api_key:,
+          pagination_interval:,
+          timeout:
         )
       end
 
@@ -74,7 +74,7 @@ module Mihari
           if data.is_a?(Array)
             data.map { |d| Models::Artifact.new(data: d, metadata: match) }
           else
-            Models::Artifact.new(data: data, metadata: match)
+            Models::Artifact.new(data:, metadata: match)
           end
         end.flatten
       end