mihari 3.9.2 → 3.12.0

data/lib/mihari/feed/reader.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "csv"
+require "json"
+require "net/https"
+require "uri"
+
+module Mihari
+  module Feed
+    class Reader
+      attr_reader :uri, :http_request_headers, :http_request_method, :http_request_payload_type, :http_request_payload
+
+      def initialize(url, http_request_headers: {}, http_request_method: "GET", http_request_payload_type: nil, http_request_payload: {})
+        @uri = URI(url)
+        @http_request_headers = http_request_headers
+        @http_request_method = http_request_method
+        @http_request_payload_type = http_request_payload_type
+        @http_request_payload = http_request_payload
+      end
+
+      def read
+        return get if http_request_method == "GET"
+
+        post
+      end
+
+      def get
+        uri.query = URI.encode_www_form(http_request_payload)
+        get = Net::HTTP::Get.new(uri)
+
+        request(get)
+      end
+
+      def post
+        post = Net::HTTP::Post.new(uri)
+
+        case http_request_payload_type
+        when "application/json"
+          post.body = JSON.generate(http_request_payload)
+        when "application/x-www-form-urlencoded"
+          post.set_form_data(http_request_payload)
+        end
+
+        request(post)
+      end
+
+      #
+      # Convert text as JSON
+      #
+      # @param [String] text
+      #
+      # @return [Array<Hash>]
+      #
+      def convert_as_json(text)
+        data = JSON.parse(text, symbolize_names: true)
+        return data if data.is_a?(Array)
+
+        [data]
+      end
+
+      #
+      # Convert text as CSV
+      #
+      # @param [String] text
+      #
+      # @return [Array<Hash>]
+      #
+      def convert_as_csv(text)
+        text_without_comments = text.lines.reject { |line| line.start_with? "#" }.join("\n")
+
+        CSV.new(text_without_comments).to_a.reject(&:empty?)
+      end
+
+      def https_options
+        return { use_ssl: true } if uri.scheme == "https"
+
+        {}
+      end
+
+      #
+      # Make a HTTP request
+      #
+      # @param [Net::HTTPRequest] req
+      #
+      # @return [Array<Hash>]
+      #
+      def request(req)
+        Net::HTTP.start(uri.host, uri.port, https_options) do |http|
+          # set headers
+          http_request_headers.each do |k, v|
+            req[k] = v
+          end
+
+          response = http.request(req)
+
+          code = response.code.to_i
+          raise HttpError, "Unsupported response code returned: #{code}" if code != 200
+
+          body = response.body
+
+          content_type = response["Content-Type"].to_s
+          data = if content_type.include?("application/json")
+            convert_as_json(body)
+          else
+            convert_as_csv(body)
+          end
+
+          data
+        end
+      end
+    end
+  end
+end

data/lib/mihari/mixins/autonomous_system.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Mixins
     module AutonomousSystem

data/lib/mihari/mixins/configuration.rb

@@ -51,9 +51,9 @@ module Mihari
       # @return [String] A template for config
       #
       def config_template
-        config = Mihari.config.values.keys.map do |key|
+        config = Mihari.config.values.keys.to_h do |key|
           [key.to_s, nil]
-        end.to_h
+        end
 
         YAML.dump(config)
       end

data/lib/mihari/mixins/disallowed_data_value.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "mem"
 
 module Mihari

data/lib/mihari/mixins/rule.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "date"
 require "pathname"
 require "yaml"

data/lib/mihari/models/alert.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "active_record"
-require "active_record/filter"
 
 module Mihari
   class Alert < ActiveRecord::Base
@@ -55,7 +54,7 @@ module Mihari
         artifact = artifact.where(dns_records: { value: filter.dns_record }) if filter.dns_record
         artifact = artifact.where(reverse_dns_names: { name: filter.reverse_dns_name }) if filter.reverse_dns_name
         # get artifact ids if there is any valid filter for artifact
-        if filter.has_valid_artifact_filters
+        if filter.valid_artifact_filters?
           artifact_ids = artifact.pluck(:id)
           # set invalid ID if nothing is matched with the filters
           artifact_ids = [-1] if artifact_ids.empty?
@@ -70,10 +69,10 @@ module Mihari
         relation = relation.where(source: filter.source) if filter.source
         relation = relation.where(title: filter.title) if filter.title
 
-        relation = relation.filter(description: { like: "%#{filter.description}%" }) if filter.description
+        relation = relation.where("description LIKE ?", "%#{filter.description}%") if filter.description
 
-        relation = relation.filter(created_at: { gte: filter.from_at }) if filter.from_at
-        relation = relation.filter(created_at: { lte: filter.to_at }) if filter.to_at
+        relation = relation.where("alerts.created_at >= ?", filter.from_at) if filter.from_at
+        relation = relation.where("alerts.created_at <= ?", filter.to_at) if filter.to_at
 
         relation
       end

data/lib/mihari/models/artifact.rb

@@ -1,10 +1,9 @@
 # frozen_string_literal: true
 
 require "active_record"
-require "active_record/filter"
 require "active_support/core_ext/integer/time"
 require "active_support/core_ext/numeric/time"
-require "uri"
+require "addressable/uri"
 
 class ArtifactValidator < ActiveModel::Validator
   def validate(record)
@@ -119,7 +118,7 @@ module Mihari
     def normalize_as_domain(url_or_domain)
       return url_or_domain if data_type == "domain"
 
-      URI.parse(url_or_domain).host
+      Addressable::URI.parse(url_or_domain).host
     end
 
     def can_enrich_whois?

data/lib/mihari/models/dns.rb

@@ -37,7 +37,7 @@ module Mihari
         resources = Resolv::DNS.new.getresources(domain, resource_type)
         resource_name = resource_type.to_s.split("::").last
 
-        resources.map do |resource|
+        resources.filter_map do |resource|
           # A, AAAA
           if resource.respond_to?(:address)
             new(resource: resource_name, value: resource.address.to_s)
@@ -48,7 +48,7 @@ module Mihari
           elsif resource.respond_to?(:data)
             new(resource: resource_name, value: resource.data.to_s)
           end
-        end.compact
+        end
       end
     end
   end

data/lib/mihari/schemas/configuration.rb

@@ -13,6 +13,8 @@ module Mihari
       optional(:censys_secret).value(:string)
       optional(:circl_passive_password).value(:string)
       optional(:circl_passive_username).value(:string)
+      optional(:database).value(:string)
+      optional(:greynoise_api_key).value(:string)
       optional(:ipinfo_api_key).value(:string)
       optional(:misp_api_endpoint).value(:string)
       optional(:misp_api_key).value(:string)
@@ -30,10 +32,9 @@ module Mihari
       optional(:thehive_api_key).value(:string)
       optional(:urlscan_api_key).value(:string)
       optional(:virustotal_api_key).value(:string)
-      optional(:zoomeye_api_key).value(:string)
       optional(:webhook_url).value(:string)
       optional(:webhook_use_json_body).value(:bool)
-      optional(:database).value(:string)
+      optional(:zoomeye_api_key).value(:string)
     end
 
     class ConfigurationContract < Dry::Validation::Contract

data/lib/mihari/schemas/rule.rb

@@ -7,33 +7,52 @@ require "mihari/schemas/macros"
 
 module Mihari
   module Schemas
+    AnalyzerOptions = Dry::Schema.Params do
+      optional(:interval).value(:integer)
+    end
+
     Analyzer = Dry::Schema.Params do
       required(:analyzer).value(Types::AnalyzerTypes)
       required(:query).value(:string)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Spyse = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("spyse"))
       required(:query).value(:string)
       required(:type).value(Types::String.enum("ip", "domain"))
+      optional(:options).hash(AnalyzerOptions)
     end
 
     ZoomEye = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("zoomeye"))
       required(:query).value(:string)
       required(:type).value(Types::String.enum("host", "web"))
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Crtsh = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("crtsh"))
       required(:query).value(:string)
       optional(:exclude_expired).value(:bool).default(true)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Urlscan = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("urlscan"))
       required(:query).value(:string)
-      optional(:use_similarity).value(:bool).default(true)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
+    Feed = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("feed"))
+      required(:query).value(:string)
+      required(:http_request_method).value(Types::FeedHttpRequestMethods).default("GET")
+      required(:http_request_headers).value(:hash).default({})
+      required(:http_request_payload).value(:hash).default({})
+      required(:selector).value(:string)
+      optional(:http_request_payload_type).value(Types::FeedHttpRequestPayloadTypes)
+      optional(:options).hash(AnalyzerOptions)
     end
 
     Rule = Dry::Schema.Params do
@@ -47,7 +66,7 @@ module Mihari
       optional(:created_on).value(:date)
       optional(:updated_on).value(:date)
 
-      required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh }
+      required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh | Feed }
 
       optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
       optional(:disallowed_data_values).value(array[:string]).default([])

data/lib/mihari/status.rb

@@ -18,11 +18,11 @@ module Mihari
     # @return [Array<Hash>]
     #
     def statuses
-      (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).map do |klass|
+      (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).to_h do |klass|
         name = klass.to_s.split("::").last.to_s
 
         [name, build_status(klass)]
-      end.to_h.compact
+      end.compact
     end
 
     #

data/lib/mihari/structs/alert.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 
@@ -16,7 +18,7 @@ module Mihari
         attribute? :dns_record, Types::String.optional
         attribute? :reverse_dns_name, Types::String.optional
 
-        def has_valid_artifact_filters
+        def valid_artifact_filters?
           !(artifact_data || asn || dns_record || reverse_dns_name).nil?
         end
       end

data/lib/mihari/structs/censys.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/structs/greynoise.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module GreyNoise
+      class Metadata < Dry::Struct
+        attribute :country, Types::String
+        attribute :country_code, Types::String
+        attribute :asn, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            country: d.fetch("country"),
+            country_code: d.fetch("country_code"),
+            asn: d.fetch("asn")
+          )
+        end
+      end
+
+      class Datum < Dry::Struct
+        attribute :ip, Types::String
+        attribute :metadata, Metadata
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            ip: d.fetch("ip"),
+            metadata: Metadata.from_dynamic!(d.fetch("metadata"))
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :complete, Types::Bool
+        attribute :count, Types::Int
+        attribute :data, Types.Array(Datum)
+        attribute :message, Types::String
+        attribute :query, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            complete: d.fetch("complete"),
+            count: d.fetch("count"),
+            data: d.fetch("data").map { |x| Datum.from_dynamic!(x) },
+            message: d.fetch("message"),
+            query: d.fetch("query")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/ipinfo.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/structs/onyphe.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/structs/shodan.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 
@@ -5,20 +7,20 @@ module Mihari
   module Structs
     module Shodan
       class Location < Dry::Struct
-        attribute :country_code, Types::String
-        attribute :country_name, Types::String
+        attribute :country_code, Types::String.optional
+        attribute :country_name, Types::String.optional
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
-            country_code: d.fetch("country_code"),
-            country_name: d.fetch("country_name")
+            country_code: d["country_code"],
+            country_name: d["country_name"]
           )
         end
       end
 
       class Match < Dry::Struct
-        attribute :asn, Types::String
+        attribute :asn, Types::String.optional
         attribute :hostnames, Types.Array(Types::String)
         attribute :location, Location
         attribute :domains, Types.Array(Types::String)
@@ -27,7 +29,7 @@ module Mihari
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
-            asn: d.fetch("asn"),
+            asn: d["asn"],
             hostnames: d.fetch("hostnames"),
             location: Location.from_dynamic!(d.fetch("location")),
             domains: d.fetch("domains"),

data/lib/mihari/structs/urlscan.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "json"
+require "dry/struct"
+
+module Mihari
+  module Structs
+    module Urlscan
+      class Page < Dry::Struct
+        attribute :domain, Types::String.optional
+        attribute :ip, Types::String.optional
+        attribute :url, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            domain: d["domain"],
+            ip: d["ip"],
+            url: d.fetch("url")
+          )
+        end
+      end
+
+      class Result < Dry::Struct
+        attribute :page, Page
+        attribute :id, Types::String
+        attribute :sort, Types.Array(Types::String | Types::Integer)
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            page: Page.from_dynamic!(d.fetch("page")),
+            id: d.fetch("_id"),
+            sort: d.fetch("sort")
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :results, Types.Array(Result)
+        attribute :has_more, Types::Bool
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            results: d.fetch("results").map { |x| Result.from_dynamic!(x) },
+            has_more: d.fetch("has_more")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/virustotal_intelligence.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/types.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "dry/types"
 
 module Mihari
@@ -8,17 +10,22 @@ module Mihari
     Nil = Strict::Nil
     Hash = Strict::Hash
     String = Strict::String
+    Bool = Strict::Bool
     Double = Strict::Float | Strict::Integer
     DateTime = Strict::DateTime
 
     DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
 
+    UrlscanDataTypes = Types::String.enum("ip", "domain", "url")
+
     AnalyzerTypes = Types::String.enum(
       "binaryedge",
       "censys",
       "circl",
       "dnpedia",
       "dnstwister",
+      "feed",
+      "greynoise",
       "onyphe",
       "otx",
       "passivetotal",
@@ -32,5 +39,8 @@ module Mihari
       "vt_intel",
       "vt"
     )
+
+    FeedHttpRequestMethods = Types::String.enum("GET", "POST")
+    FeedHttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.9.2"
+  VERSION = "3.12.0"
 end

data/lib/mihari/web/api.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Entities
 require "mihari/web/entities/message"
 

data/lib/mihari/web/entities/artifact.rb

@@ -14,10 +14,10 @@ module Mihari
       expose :whois_record, using: Entities::WhoisRecord, documentation: { type: Entities::WhoisRecord, required: false }, as: :whoisRecord
 
       expose :reverse_dns_names, using: Entities::ReverseDnsName, documentation: { type: Entities::ReverseDnsName, is_array: true, required: false }, as: :reverseDnsNames do |status, _options|
-        status.reverse_dns_names.length > 0 ? status.reverse_dns_names : nil
+        status.reverse_dns_names.empty? ? nil : status.reverse_dns_names
       end
       expose :dns_records, using: Entities::DnsRecord, documentation: { type: Entities::DnsRecord, is_array: true, required: false }, as: :dnsRecords do |status, _options|
-        status.dns_records.length > 0 ? status.dns_records : nil
+        status.dns_records.empty? ? nil : status.dns_records
       end
     end
   end

data/lib/mihari/web/entities/whois.rb

@@ -9,7 +9,7 @@ module Mihari
       expose :expires_on, documentation: { type: Date, required: false }, as: :expiresOn
       expose :registrar, documentation: { type: Hash, required: false }
       expose :contacts, documentation: { type: Hash, is_array: true, required: true } do |whois_record, _options|
-        whois_record.contacts.map { |h| h.to_camelback_keys }
+        whois_record.contacts.map(&:to_camelback_keys)
       end
     end
   end

data/lib/mihari/web/public/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/static/favicon.ico"><title>Mihari</title><link href="/static/js/app.14008741.js" rel="preload" as="script"></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/static/js/app.14008741.js"></script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/static/favicon.ico"><title>Mihari</title><link href="/static/js/app.5dc97aae.js" rel="preload" as="script"></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/static/js/app.5dc97aae.js"></script></body></html>