mihari 4.2.0 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 853724da55a4225403595284e6437da9816d016d99e380d239507e1eecdf35cd
-  data.tar.gz: 35e5ef84d38c67eaa1670a5e6261aed5c629966921ca3b14fb2509c9e3993c45
+  metadata.gz: dc5b6ab7daa3030a0ef0778df2753247bde10b978be709102ecf1be60b672a9c
+  data.tar.gz: 1c1c283cf1e2989e9e94e0aa582567dd71b7900f5c5faa40a0ce3f1b327982a3
 SHA512:
-  metadata.gz: 50855245bf70579da57c26859d2ab491e5238f7200c1de50cc7f46cafcd94c57e411e0fdbffdd60ddb00df2c8783ed051c87be24963fe10c5459784f337bd8d3
-  data.tar.gz: 436e6c0f3ceacb5b72f4212e83832759320cce109310736d87f6d1bfb49ac61f32a25a4d44cf133d2d5794466dea72bd69244997ccbbc6525418bc9aef403d0f
+  metadata.gz: 918ee19022035b5f5e3db9a00318253803b1cc430b45676225af94861fe6dc6fe51343545d224e6094f09ad37dc003713fbbcb1777904b01205903e22d05bf23
+  data.tar.gz: c5ce99eb3d8e01b1b2a8ac51916afa172c1fecb55611a3b8aa76e686c72801beb11135ed1c8aed31e11b693b7467dbe513902f55e229c834bbfcb09460ba4202

data/.github/workflows/test.yml

@@ -1,9 +1,13 @@
 name: Ruby CI
 
-on: [pull_request]
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
 
     services:
@@ -39,7 +43,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.7, "3.0"]
+        ruby: [2.7, "3.0", 3.1]
 
     steps:
       - uses: actions/checkout@v3
@@ -65,3 +69,21 @@ jobs:
           DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
         run: |
           bundle exec rake
+
+      - name: Coveralls Parallel
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}
+          flag-name: run-${{ matrix.ruby-version }}
+          parallel: true
+
+  coverage:
+    name: Coverage
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}
+          parallel-finished: true

data/lib/mihari/analyzers/rule.rb

@@ -28,6 +28,15 @@ module Mihari
       "zoomeye" => ZoomEye
     }.freeze
 
+    EMITTER_TO_CLASS = {
+      "database" => Emitters::Database,
+      "http" => Emitters::HTTP,
+      "misp" => Emitters::MISP,
+      "slack" => Emitters::Slack,
+      "the_hive" => Emitters::TheHive,
+      "webhook" => Emitters::Webhook
+    }.freeze
+
     class Rule < Base
       include Mixins::DisallowedDataValue
       include Mixins::Rule
@@ -41,6 +50,8 @@ module Mihari
       option :allowed_data_types, default: proc { ALLOWED_DATA_TYPES }
       option :disallowed_data_values, default: proc { [] }
 
+      option :emitters, optional: true
+
       attr_reader :source
 
       def initialize(**kwargs)
@@ -48,6 +59,8 @@ module Mihari
 
         @source = id
 
+        @emitters = emitters || DEFAULT_EMITTERS
+
         validate_analyzer_configurations
       end
 
@@ -59,18 +72,20 @@ module Mihari
       def artifacts
         artifacts = []
 
-        queries.each do |params|
-          analyzer_name = params[:analyzer]
+        queries.each do |original_params|
+          parmas = original_params.deep_dup
+
+          analyzer_name = parmas[:analyzer]
           klass = get_analyzer_class(analyzer_name)
 
-          query = params[:query]
+          query = parmas[:query]
 
           # set interval in the top level
-          options = params[:options] || {}
+          options = parmas[:options] || {}
           interval = options[:interval]
-          params[:interval] = interval if interval
+          parmas[:interval] = interval if interval
 
-          analyzer = klass.new(query, **params)
+          analyzer = klass.new(query, **parmas)
 
           # Use #normalized_artifacts method to get atrifacts as Array<Mihari::Artifact>
           # So Mihari::Artifact object has "source" attribute (e.g. "Shodan")
@@ -120,6 +135,34 @@ module Mihari
 
       private
 
+      #
+      # Get emitter class
+      #
+      # @param [String] emitter_name
+      #
+      # @return [Class<Mihari::Emitters::Base>] emitter class
+      #
+      def get_emitter_class(emitter_name)
+        emitter = EMITTER_TO_CLASS[emitter_name]
+        return emitter if emitter
+
+        raise ArgumentError, "#{emitter_name} is not supported"
+      end
+
+      def valid_emitters
+        @valid_emitters ||= emitters.filter_map do |original_params|
+          params = original_params.deep_dup
+
+          name = params[:emitter]
+          params.delete(:emitter)
+
+          klass = get_emitter_class(name)
+          emitter = klass.new(**params)
+
+          emitter.valid? ? emitter : nil
+        end
+      end
+
       #
       # Get analyzer class
       #

data/lib/mihari/constants.rb

@@ -2,4 +2,6 @@
 
 module Mihari
   ALLOWED_DATA_TYPES = ["hash", "ip", "domain", "url", "mail"].freeze
+
+  DEFAULT_EMITTERS = ["database", "misp", "slack", "the_hive", "webhook"].map { |name| { emitter: name } }.freeze
 end

data/lib/mihari/emitters/base.rb

@@ -6,6 +6,9 @@ module Mihari
       include Mixins::Configurable
       include Mixins::Retriable
 
+      def initialize(*)
+      end
+
       class << self
         def inherited(child)
           super

data/lib/mihari/emitters/http.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "erb"
+
+module Mihari
+  module Emitters
+    class PayloadTemplate < ERB
+      def self.template
+        %{
+					{
+						"title": "<%= @title %>",
+						"description": "<%= @description %>",
+						"source": "<%= @source %>",
+						"artifacts": [
+							<% @artifacts.each_with_index do |artifact, idx| %>
+								"<%= artifact.data %>"
+								<%= ',' if idx < (@artifacts.length - 1) %>
+							<% end %>
+						],
+						"tags": [
+							<% @tags.each_with_index do |tag, idx| %>
+								"<%= tag %>"
+								<%= ',' if idx < (@tags.length - 1) %>
+							<% end %>
+						]
+					}
+				}
+      end
+
+      def initialize(title:, description:, artifacts:, source:, tags:, options: {})
+        @title = title
+        @description = description
+        @artifacts = artifacts
+        @source = source
+        @tags = tags
+
+        @template = options.fetch(:template, self.class.template)
+        super(@template)
+      end
+
+      def result
+        super(binding)
+      end
+    end
+
+    class HTTP < Base
+      # @return [Addressable::URI, nil]
+      attr_reader :uri
+
+      # @return [Hash]
+      attr_reader :http_request_headers
+
+      # @return [String]
+      attr_reader :http_request_method
+
+      # @return [String, nil]
+      attr_reader :template
+
+      def initialize(*args, **kwargs)
+        super(*args, **kwargs)
+
+        uri = kwargs[:url] || kwargs[:uri]
+        http_request_headers = kwargs[:http_request_headers] || {}
+        http_request_method = kwargs[:http_request_method] || "POST"
+        template = kwargs[:template]
+
+        @uri = Addressable::URI.parse(uri) if uri
+        @http_request_headers = http_request_headers
+        @http_request_method = http_request_method
+        @template = template
+      end
+
+      def emit(title:, description:, artifacts:, source:, tags:)
+        return if artifacts.empty?
+
+        res = nil
+
+        payload_ = payload_as_string(
+          title: title,
+          description: description,
+          artifacts: artifacts,
+          source: source,
+          tags: tags
+        )
+        payload = JSON.parse(payload_)
+
+        client = Mihari::HTTP.new(uri, headers: http_request_headers, payload: payload)
+
+        case http_request_method
+        when "GET"
+          res = client.get
+        when "POST"
+          res = client.post
+        end
+
+        res
+      end
+
+      def valid?
+        return false if uri.nil?
+
+        ["http", "https"].include? uri.scheme.downcase
+      end
+
+      private
+
+      def payload_as_string(title:, description:, artifacts:, source:, tags:)
+        @payload_as_string ||= [].tap do |out|
+          options = {}
+          unless template.nil?
+            options[:template] = File.read(template)
+          end
+
+          payload_template = PayloadTemplate.new(
+            title: title,
+            description: description,
+            artifacts: artifacts,
+            source: source,
+            tags: tags,
+            options: options
+          )
+          out << payload_template.result
+        end.first
+      end
+    end
+  end
+end

data/lib/mihari/emitters/webhook.rb

@@ -11,20 +11,11 @@ module Mihari
       def emit(title:, description:, artifacts:, source:, tags:)
         return if artifacts.empty?
 
-        uri = Addressable::URI.parse(Mihari.config.webhook_url)
-        data = {
-          title: title,
-          description: description,
-          artifacts: artifacts.map(&:data),
-          source: source,
-          tags: tags
-        }
+        headers = { 'content-type': "application/x-www-form-urlencoded" }
+        headers["content-type"] = "application/json" if use_json_body?
 
-        if use_json_body?
-          Net::HTTP.post(uri, data.to_json, "Content-Type" => "application/json")
-        else
-          Net::HTTP.post_form(uri, data)
-        end
+        emitter = Emitters::HTTP.new(uri: Mihari.config.webhook_url)
+        emitter.emit(title: title, description: description, artifacts: artifacts, source: source, tags: tags)
       end
 
       private
@@ -45,16 +36,16 @@ module Mihari
       #
       # Check whether a webhook URL is set or not
       #
-      # @return [<Type>] <description>
+      # @return [Boolean]
       #
       def webhook_url?
         !webhook_url.nil?
       end
 
       #
-      # Check whether to use JSON body or NOT
+      # Check whether to use JSON body or not
       #
-      # @return [<Type>] <description>
+      # @return [Boolean]
       #
       def use_json_body?
         @use_json_body ||= Mihari.config.webhook_use_json_body

data/lib/mihari/entities/rule.rb

@@ -8,6 +8,10 @@ module Mihari
       expose :interval, documentation: { type: Integer, required: false }
     end
 
+    class Emitter < Grape::Entity
+      expose :emitter, documentation: { type: String, required: true }
+    end
+
     class Rule < Grape::Entity
       expose :id, documentation: { type: String, required: true }
 
@@ -16,6 +20,7 @@ module Mihari
       expose :title, documentation: { type: String, required: true }
       expose :description, documentation: { type: String, required: true }
       expose :queries, using: Entities::Query, documentation: { type: Entities::Query, is_array: true, required: true }
+      expose :emitters, using: Entities::Emitter, documentation: { type: Entities::Emitter, is_array: true, required: false }
       expose :tags, documentation: { type: String, is_array: true }
       expose :allowed_data_types, documentation: { type: String, is_array: true }, as: :allowedDtaTypes
       expose :disallowed_data_values, documentation: { type: String, is_array: true }, as: :disallowedDataValues

data/lib/mihari/feed/reader.rb

@@ -1,25 +1,27 @@
 # frozen_string_literal: true
 
 require "csv"
+require "insensitive_hash"
 
 module Mihari
   module Feed
     class Reader
-      attr_reader :uri, :http_request_headers, :http_request_method, :http_request_payload_type, :http_request_payload
+      attr_reader :uri, :http_request_headers, :http_request_method, :http_request_payload
 
       def initialize(uri, http_request_headers: {}, http_request_method: "GET", http_request_payload_type: nil, http_request_payload: {})
         @uri = Addressable::URI.parse(uri)
-        @http_request_headers = http_request_headers
+        @http_request_headers = http_request_headers.insensitive
         @http_request_method = http_request_method
-        @http_request_payload_type = http_request_payload_type
         @http_request_payload = http_request_payload
+
+        http_request_headers["content-type"] = http_request_payload_type if http_request_payload_type
       end
 
       def read
         return read_file(uri.path) if uri.scheme == "file"
 
         res = nil
-        client = HTTP.new(uri, headers: http_request_headers, payload: http_request_payload, payload_type: http_request_payload_type)
+        client = HTTP.new(uri, headers: http_request_headers, payload: http_request_payload)
 
         res = client.get if http_request_method == "GET"
         res = client.post if http_request_method == "POST"

data/lib/mihari/http.rb

@@ -1,13 +1,14 @@
 # frozen_string_literal: true
 
+require "insensitive_hash"
+
 module Mihari
   class HTTP
-    attr_reader :uri, :headers, :payload_type, :payload
+    attr_reader :uri, :headers, :payload
 
-    def initialize(uri, headers: {}, payload_type: nil, payload: {})
-      @uri = Addressable::URI.parse(uri)
-      @headers = headers
-      @payload_type = payload_type
+    def initialize(uri, headers: {}, payload: {})
+      @uri = uri.is_a?(Addressable::URI) ? uri : Addressable::URI.parse(uri)
+      @headers = headers.insensitive
       @payload = payload
     end
 
@@ -31,12 +32,10 @@ module Mihari
     def post
       post = Net::HTTP::Post.new(uri)
 
-      case payload_type
+      case content_type
       when "application/json"
-        headers["content-type"] = "application/json" unless headers.key?("content-type")
         post.body = JSON.generate(payload)
       when "application/x-www-form-urlencoded"
-        headers["content-type"] = "application/x-www-form-urlencoded" unless headers.key?("content-type")
         post.set_form_data(payload)
       end
 
@@ -44,19 +43,23 @@ module Mihari
     end
 
     class << self
-      def get(uri, headers: {}, payload_type: nil, payload: {})
-        client = new(uri, headers: headers, payload_type: payload_type, payload: payload)
+      def get(uri, headers: {}, payload: {})
+        client = new(uri, headers: headers, payload: payload)
         client.get
       end
 
-      def post(uri, headers: {}, payload_type: nil, payload: {})
-        client = new(uri, headers: headers, payload_type: payload_type, payload: payload)
+      def post(uri, headers: {}, payload: {})
+        client = new(uri, headers: headers, payload: payload)
         client.post
       end
     end
 
     private
 
+    def content_type
+      headers["content-type"] || "application/json"
+    end
+
     #
     # Get options for HTTP request
     #
@@ -84,8 +87,10 @@ module Mihari
 
         res = http.request(req)
 
-        code = res.code.to_i
-        raise HttpError, "Unsupported response code returned: #{code}" if code != 200
+        unless res.is_a?(Net::HTTPSuccess)
+          code = res.code.to_i
+          raise HttpError, "Unsuccessful response code returned: #{code}"
+        end
 
         res
       end

data/lib/mihari/schemas/rule.rb

@@ -42,14 +42,26 @@ module Mihari
     Feed = Dry::Schema.Params do
       required(:analyzer).value(Types::String.enum("feed"))
       required(:query).value(:string)
-      required(:http_request_method).value(Types::FeedHttpRequestMethods).default("GET")
-      required(:http_request_headers).value(:hash).default({})
-      required(:http_request_payload).value(:hash).default({})
       required(:selector).value(:string)
-      optional(:http_request_payload_type).value(Types::FeedHttpRequestPayloadTypes)
+      optional(:http_request_method).value(Types::HttpRequestMethods).default("GET")
+      optional(:http_request_headers).value(:hash).default({})
+      optional(:http_request_payload).value(:hash).default({})
+      optional(:http_request_payload_type).value(Types::HttpRequestPayloadTypes)
       optional(:options).hash(AnalyzerOptions)
     end
 
+    Emitter = Dry::Schema.Params do
+      required(:emitter).value(Types::EmitterTypes)
+    end
+
+    HTTP = Dry::Schema.Params do
+      required(:emitter).value(Types::String.enum("http"))
+      required(:url).value(:string)
+      optional(:http_request_method).value(Types::HttpRequestMethods).default("POST")
+      optional(:http_request_headers).value(:hash).default({})
+      optional(:template).value(:string)
+    end
+
     Rule = Dry::Schema.Params do
       required(:title).value(:string)
       required(:description).value(:string)
@@ -63,11 +75,25 @@ module Mihari
 
       required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh | Feed }
 
+      optional(:emitters).value(:array).each { Emitter | HTTP }
+
       optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
       optional(:disallowed_data_values).value(array[:string]).default([])
 
       optional(:ignore_old_artifacts).value(:bool).default(false)
       optional(:ignore_threshold).value(:integer).default(0)
+
+      before(:key_coercer) do |result|
+        # it looks like that dry-schema v1.9.1 has an issue with setting an array of schemas as a default value
+        # e.g. optional(:emitters).value(:array).each { Emitter | HTTP }.default(DEFAULT_EMITTERS) does not work well
+        # so let's do a dirty hack...
+        h = result.to_h
+
+        emitters = h[:emitters]
+        h[:emitters] = emitters || DEFAULT_EMITTERS
+
+        h
+      end
     end
 
     class RuleContract < Dry::Validation::Contract

data/lib/mihari/structs/rule.rb

@@ -133,6 +133,7 @@ module Mihari
             queries: self[:queries],
             allowed_data_types: self[:allowed_data_types],
             disallowed_data_values: self[:disallowed_data_values],
+            emitters: self[:emitters],
             id: id
           )
           analyzer.ignore_old_artifacts = self[:ignore_old_artifacts]

data/lib/mihari/types.rb

@@ -22,7 +22,6 @@ module Mihari
       "circl",
       "dnpedia",
       "dnstwister",
-      "feed",
       "greynoise",
       "onyphe",
       "otx",
@@ -38,7 +37,16 @@ module Mihari
       "vt"
     )
 
-    FeedHttpRequestMethods = Types::String.enum("GET", "POST")
-    FeedHttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
+    HttpRequestMethods = Types::String.enum("GET", "POST")
+    HttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
+
+    EmitterTypes = Types::String.enum(
+      "database",
+      "http",
+      "misp",
+      "slack",
+      "the_hive",
+      "webhook"
+    )
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "4.2.0"
+  VERSION = "4.3.0"
 end

data/lib/mihari/web/public/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="viewport" content="width=device-width,initial-scale=1"/><link rel="icon" href="/static/favicon.ico"/><title>Mihari</title><script defer="defer" type="module" src="/static/js/chunk-vendors.15e84e22.js"></script><script defer="defer" type="module" src="/static/js/app.4818aedd.js"></script><link href="/static/css/chunk-vendors.3ed9b08e.css" rel="stylesheet"><link href="/static/css/app.43138058.css" rel="stylesheet"><script defer="defer" src="/static/js/chunk-vendors-legacy.c99e452e.js" nomodule></script><script defer="defer" src="/static/js/app-legacy.46b666f0.js" nomodule></script></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="viewport" content="width=device-width,initial-scale=1"/><link rel="icon" href="/static/favicon.ico"/><title>Mihari</title><script defer="defer" type="module" src="/static/js/chunk-vendors.c5525f1e.js"></script><script defer="defer" type="module" src="/static/js/app.e74e91d7.js"></script><link href="/static/css/chunk-vendors.c57bb3fd.css" rel="stylesheet"><link href="/static/css/app.0de4b715.css" rel="stylesheet"><script defer="defer" src="/static/js/chunk-vendors-legacy.41357cdf.js" nomodule></script><script defer="defer" src="/static/js/app-legacy.e451304b.js" nomodule></script></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>