mihari 3.10.0 → 4.0.0

data/lib/mihari/status.rb

@@ -18,11 +18,11 @@ module Mihari
     # @return [Array<Hash>]
     #
     def statuses
-      (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).map do |klass|
+      (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).to_h do |klass|
         name = klass.to_s.split("::").last.to_s
 
         [name, build_status(klass)]
-      end.to_h.compact
+      end.compact
     end
 
     #

data/lib/mihari/structs/alert.rb

@@ -1,5 +1,4 @@
-require "json"
-require "dry/struct"
+# frozen_string_literal: true
 
 module Mihari
   module Structs
@@ -16,7 +15,7 @@ module Mihari
         attribute? :dns_record, Types::String.optional
         attribute? :reverse_dns_name, Types::String.optional
 
-        def has_valid_artifact_filters
+        def valid_artifact_filters?
           !(artifact_data || asn || dns_record || reverse_dns_name).nil?
         end
       end

data/lib/mihari/structs/censys.rb

@@ -1,5 +1,4 @@
-require "json"
-require "dry/struct"
+# frozen_string_literal: true
 
 module Mihari
   module Structs
@@ -32,13 +31,15 @@ module Mihari
         attribute :ip, Types::String
         attribute :location, Location
         attribute :autonomous_system, AutonomousSystem
+        attribute :metadata, Types::Hash
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
             ip: d.fetch("ip"),
             location: Location.from_dynamic!(d.fetch("location")),
-            autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system"))
+            autonomous_system: AutonomousSystem.from_dynamic!(d.fetch("autonomous_system")),
+            metadata: d
           )
         end
       end

data/lib/mihari/structs/greynoise.rb

@@ -1,5 +1,4 @@
-require "json"
-require "dry/struct"
+# frozen_string_literal: true
 
 module Mihari
   module Structs
@@ -22,12 +21,14 @@ module Mihari
       class Datum < Dry::Struct
         attribute :ip, Types::String
         attribute :metadata, Metadata
+        attribute :metadata_, Types::Hash
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
             ip: d.fetch("ip"),
-            metadata: Metadata.from_dynamic!(d.fetch("metadata"))
+            metadata: Metadata.from_dynamic!(d.fetch("metadata")),
+            metadata_: d
           )
         end
       end

data/lib/mihari/structs/ipinfo.rb

@@ -1,5 +1,4 @@
-require "json"
-require "dry/struct"
+# frozen_string_literal: true
 
 module Mihari
   module Structs

data/lib/mihari/structs/onyphe.rb

@@ -1,5 +1,4 @@
-require "json"
-require "dry/struct"
+# frozen_string_literal: true
 
 module Mihari
   module Structs
@@ -8,6 +7,7 @@ module Mihari
         attribute :asn, Types::String
         attribute :country_code, Types::String.optional
         attribute :ip, Types::String
+        attribute :metadata, Types::Hash
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
@@ -15,7 +15,8 @@ module Mihari
             asn: d.fetch("asn"),
             ip: d.fetch("ip"),
             # Onyphe's country = 2-letter country code
-            country_code: d["country"]
+            country_code: d["country"],
+            metadata: d
           )
         end
       end
@@ -24,7 +25,7 @@ module Mihari
         attribute :count, Types::Int
         attribute :error, Types::Int
         attribute :max_page, Types::Int
-        attribute :page, Types::String
+        attribute :page, Types::Int
         attribute :results, Types.Array(Result)
         attribute :status, Types::String
         attribute :total, Types::Int
@@ -35,7 +36,7 @@ module Mihari
             count: d.fetch("count"),
             error: d.fetch("error"),
             max_page: d.fetch("max_page"),
-            page: d.fetch("page"),
+            page: d.fetch("page").to_i,
             results: d.fetch("results").map { |x| Result.from_dynamic!(x) },
             status: d.fetch("status"),
             total: d.fetch("total")

data/lib/mihari/structs/rule.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    module Rule
+      class SearchFilter < Dry::Struct
+        attribute? :description, Types::String.optional
+        attribute? :tag_name, Types::String.optional
+        attribute? :title, Types::String.optional
+        attribute? :from_at, Types::DateTime.optional
+        attribute? :to_at, Types::DateTime.optional
+      end
+
+      class SearchFilterWithPagination < SearchFilter
+        attribute? :page, Types::Int.default(1)
+        attribute? :limit, Types::Int.default(10)
+
+        def without_pagination
+          SearchFilter.new(
+            description: description,
+            from_at: from_at,
+            tag_name: tag_name,
+            title: title,
+            to_at: to_at
+          )
+        end
+      end
+
+      class Rule
+        attr_reader :data, :errors
+
+        def initialize(data)
+          @data = data.deep_symbolize_keys
+          @errors = nil
+          @no_method_error = nil
+
+          validate
+        end
+
+        #
+        # @return [Boolean]
+        #
+        def errors?
+          return false if @errors.nil?
+
+          !@errors.empty?
+        end
+
+        #
+        # @return [Array<String>]
+        #
+        def error_messages
+          return [] if @errors.nil?
+
+          @errors.messages.map do |message|
+            path = message.path.map(&:to_s).join
+            "#{path} #{message.text}"
+          end
+        end
+
+        def validate
+          begin
+            contract = Schemas::RuleContract.new
+            result = contract.call(data)
+          rescue NoMethodError => e
+            @no_method_error = e
+            return
+          end
+
+          @data = result.to_h
+          @errors = result.errors
+        end
+
+        def validate!
+          raise RuleValidationError, "Data should be a hash" unless data.is_a?(Hash)
+
+          raise RuleValidationError, error_messages.join("\n") if errors?
+
+          raise RuleValidationError, "Something wrong with queries" unless @no_method_error.nil?
+        end
+
+        def [](key)
+          data[key.to_sym]
+        end
+
+        #
+        # @return [String]
+        #
+        def id
+          @id ||= data[:id] || UUIDTools::UUID.md5_create(UUIDTools::UUID_URL_NAMESPACE, data.to_yaml).to_s
+        end
+
+        #
+        # @return [String]
+        #
+        def title
+          @title ||= data[:title]
+        end
+
+        #
+        # @return [String]
+        #
+        def description
+          @description ||= data[:description]
+        end
+
+        #
+        # @return [Mihari::Rule]
+        #
+        def to_model
+          Mihari::Rule.new(
+            id: id,
+            title: title,
+            description: description,
+            data: data
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/shodan.rb

@@ -1,18 +1,17 @@
-require "json"
-require "dry/struct"
+# frozen_string_literal: true
 
 module Mihari
   module Structs
     module Shodan
       class Location < Dry::Struct
-        attribute :country_code, Types::String
-        attribute :country_name, Types::String
+        attribute :country_code, Types::String.optional
+        attribute :country_name, Types::String.optional
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
           new(
-            country_code: d.fetch("country_code"),
-            country_name: d.fetch("country_name")
+            country_code: d["country_code"],
+            country_name: d["country_name"]
           )
         end
       end
@@ -23,6 +22,7 @@ module Mihari
         attribute :location, Location
         attribute :domains, Types.Array(Types::String)
         attribute :ip_str, Types::String
+        attribute :metadata, Types::Hash
 
         def self.from_dynamic!(d)
           d = Types::Hash[d]
@@ -31,7 +31,8 @@ module Mihari
             hostnames: d.fetch("hostnames"),
             location: Location.from_dynamic!(d.fetch("location")),
             domains: d.fetch("domains"),
-            ip_str: d.fetch("ip_str")
+            ip_str: d.fetch("ip_str"),
+            metadata: d
           )
         end
       end

data/lib/mihari/structs/urlscan.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Structs
+    module Urlscan
+      class Page < Dry::Struct
+        attribute :domain, Types::String.optional
+        attribute :ip, Types::String.optional
+        attribute :url, Types::String
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            domain: d["domain"],
+            ip: d["ip"],
+            url: d.fetch("url")
+          )
+        end
+      end
+
+      class Result < Dry::Struct
+        attribute :page, Page
+        attribute :id, Types::String
+        attribute :sort, Types.Array(Types::String | Types::Integer)
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            page: Page.from_dynamic!(d.fetch("page")),
+            id: d.fetch("_id"),
+            sort: d.fetch("sort")
+          )
+        end
+      end
+
+      class Response < Dry::Struct
+        attribute :results, Types.Array(Result)
+        attribute :has_more, Types::Bool
+
+        def self.from_dynamic!(d)
+          d = Types::Hash[d]
+          new(
+            results: d.fetch("results").map { |x| Result.from_dynamic!(x) },
+            has_more: d.fetch("has_more")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mihari/structs/virustotal_intelligence.rb

@@ -1,5 +1,4 @@
-require "json"
-require "dry/struct"
+# frozen_string_literal: true
 
 module Mihari
   module Structs
@@ -19,6 +18,7 @@ module Mihari
         attribute :type, Types::String
         attribute :id, Types::String
         attribute :context_attributes, ContextAttributes.optional
+        attribute :metadata, Types::Hash
 
         def value
           case type
@@ -42,7 +42,8 @@ module Mihari
           new(
             type: d.fetch("type"),
             id: d.fetch("id"),
-            context_attributes: context_attributes
+            context_attributes: context_attributes,
+            metadata: d
           )
         end
       end

data/lib/mihari/type_checker.rb

@@ -1,11 +1,5 @@
 # frozen_string_literal: true
 
-require "addressable/uri"
-require "dry-initializer"
-require "email_address"
-require "ipaddr"
-require "public_suffix"
-
 module Mihari
   class TypeChecker
     extend Dry::Initializer
@@ -16,6 +10,8 @@ module Mihari
       super
 
       raise ArgumentError if data.is_a?(Hash)
+
+      @data = data.to_s
     end
 
     # @return [Boolean]

data/lib/mihari/types.rb

@@ -1,4 +1,4 @@
-require "dry/types"
+# frozen_string_literal: true
 
 module Mihari
   module Types
@@ -8,17 +8,21 @@ module Mihari
     Nil = Strict::Nil
     Hash = Strict::Hash
     String = Strict::String
+    Bool = Strict::Bool
     Double = Strict::Float | Strict::Integer
     DateTime = Strict::DateTime
 
     DataTypes = Types::String.enum(*ALLOWED_DATA_TYPES)
 
+    UrlscanDataTypes = Types::String.enum("ip", "domain", "url")
+
     AnalyzerTypes = Types::String.enum(
       "binaryedge",
       "censys",
       "circl",
       "dnpedia",
       "dnstwister",
+      "feed",
       "greynoise",
       "onyphe",
       "otx",
@@ -33,5 +37,8 @@ module Mihari
       "vt_intel",
       "vt"
     )
+
+    FeedHttpRequestMethods = Types::String.enum("GET", "POST")
+    FeedHttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.10.0"
+  VERSION = "4.0.0"
 end

data/lib/mihari/web/api.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Entities
 require "mihari/web/entities/message"
 
@@ -16,12 +18,15 @@ require "mihari/web/entities/artifact"
 
 require "mihari/web/entities/alert"
 
+require "mihari/web/entities/rule"
+
 # Endpoints
 require "mihari/web/endpoints/alerts"
 require "mihari/web/endpoints/artifacts"
 require "mihari/web/endpoints/command"
 require "mihari/web/endpoints/configs"
 require "mihari/web/endpoints/ip_addresses"
+require "mihari/web/endpoints/rules"
 require "mihari/web/endpoints/sources"
 require "mihari/web/endpoints/tags"
 
@@ -35,6 +40,7 @@ module Mihari
     mount Endpoints::Command
     mount Endpoints::Configs
     mount Endpoints::IPAddresses
+    mount Endpoints::Rules
     mount Endpoints::Sources
     mount Endpoints::Tags
 

data/lib/mihari/web/app.rb

@@ -11,6 +11,8 @@ require "grape-entity"
 require "grape-swagger"
 require "grape-swagger-entity"
 
+require "mihari/web/middleware/connection_adapter"
+
 require "mihari/web/api"
 
 module Mihari
@@ -34,6 +36,8 @@ module Mihari
             end
           end
 
+          use Middleware::ConnectionAdapter
+
           run App.new
         end.to_app
       end
@@ -41,14 +45,8 @@ module Mihari
       def run!(port: 9292, host: "localhost", threads: "0:16", verbose: false)
         url = "http://#{host}:#{port}"
 
-        Rack::Handler::Puma.run(instance, Port: port, Host: host, Threads: threads, Verbose: verbose) do |server|
+        Rack::Handler::Puma.run(instance, Port: port, Host: host, Threads: threads, Verbose: verbose) do |_launcher|
           Launchy.open(url) if ENV["RACK_ENV"] != "development"
-
-          [:INT, :TERM].each do |sig|
-            trap(sig) do
-              server.shutdown
-            end
-          end
         end
       end
     end

data/lib/mihari/web/endpoints/alerts.rb

@@ -7,14 +7,17 @@ module Mihari
         desc "Search alerts", {
           is_array: true,
           success: Entities::Alert,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Search alerts"
         }
         params do
           optional :page, type: Integer
+
           optional :artifact, type: String
           optional :description, type: String
           optional :source, type: String
           optional :tag, type: String
+          optional :title, type: String
 
           optional :fromAt, type: DateTime
           optional :toAt, type: DateTime
@@ -38,7 +41,7 @@ module Mihari
           filter["tag_name"] = filter["tag"]
 
           # symbolize hash keys
-          filter = filter.to_h.transform_keys(&:to_sym)
+          filter = filter.to_h.symbolize_keys
 
           search_filter_with_pagenation = Structs::Alert::SearchFilterWithPagination.new(**filter)
           alerts = Mihari::Alert.search(search_filter_with_pagenation)
@@ -49,7 +52,8 @@ module Mihari
 
         desc "Delete an alert", {
           success: Entities::Message,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Delete an alert"
         }
         params do
           requires :id, type: Integer

data/lib/mihari/web/endpoints/artifacts.rb

@@ -6,7 +6,8 @@ module Mihari
       namespace :artifacts do
         desc "Get an artifact", {
           success: Entities::Artifact,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Get an artifact"
         }
         params do
           requires :id, type: Integer
@@ -38,7 +39,8 @@ module Mihari
 
         desc "Enrich an artifact", {
           success: Entities::Message,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Enrich an artifact"
         }
         params do
           requires :id, type: Integer
@@ -67,7 +69,8 @@ module Mihari
 
         desc "Delete an artifact", {
           success: Entities::Message,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Delete an artifact"
         }
         params do
           requires :id, type: Integer

data/lib/mihari/web/endpoints/command.rb

@@ -8,7 +8,8 @@ module Mihari
       namespace :command do
         desc "Run a command", {
           success: Entities::CommandResult,
-          failure: [{ code: 400, message: "Bad request", model: Entities::Message }]
+          failure: [{ code: 400, message: "Bad request", model: Entities::Message }],
+          summary: "Run a command"
         }
         params do
           requires :command, type: String, documentation: { param_type: "body" }

data/lib/mihari/web/endpoints/configs.rb

@@ -6,7 +6,8 @@ module Mihari
       namespace :configs do
         desc "Get configs", {
           is_array: true,
-          success: Entities::Config
+          success: Entities::Config,
+          summary: "Get configs"
         }
         get "/" do
           statuses = Status.check

data/lib/mihari/web/endpoints/ip_addresses.rb

@@ -6,7 +6,8 @@ module Mihari
       namespace :ip_addresses do
         desc "Get an IP address", {
           success: Entities::IPAddress,
-          failure: [{ code: 404, message: "Not found", model: Entities::Message }]
+          failure: [{ code: 404, message: "Not found", model: Entities::Message }],
+          summary: "Get an IP address"
         }
         params do
           requires :ip, type: String, regexp: /\A[0-9.]+\z/