mihari 3.11.0 → 3.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c8fda839e1bb1ec4733b7da17e20e3b47bd5795d5aca25371d09e5a0fa9a575
-  data.tar.gz: a4551c61c625bd08167608051750bf1785a31ad5215e95e857753fc5fed31d00
+  metadata.gz: 5a17c14dffe68b2f82316858615371036996a3a2b6d31d3c6a5c431294c38634
+  data.tar.gz: 97afaf2f8b20985b0908cc1fe5778fe75ab7d0e1bf13b990a7b4165c92843604
 SHA512:
-  metadata.gz: 81a1ca91b65b03f98bb3504a85c8113b2cf189a4911109e113da1058fb9a1bd61c8807e6f3bec72484786b89e68d0027782d38013676bd6ae78aa57434315972
-  data.tar.gz: b42b1658310a7ee87940fbebb6507edca2187d6be92957a149f9234af8bf32ba06e702c401d321b553901dc56094964e78ff602549734f51abbf24b4b7a1fa4d
+  metadata.gz: 418d7f278536e245196723d2ebbe0d99934b0db3278e8f8178470241c67f25e05d0aacf7bab2b766ce69d0589a79e87eca554a91fe4cfae5cfa12cd52d97fb61
+  data.tar.gz: 9e90193273ee40c9956a138c2c73b713eb610a3ebdc1c9def73a6d354c4124a97e6842e438004956b3edc3206dfee8bae5771a86e964b59a2a7065a1f910891f

data/README.md

@@ -2,7 +2,6 @@
 
 [![Gem Version](https://badge.fury.io/rb/mihari.svg)](https://badge.fury.io/rb/mihari)
 [![Ruby CI](https://github.com/ninoseki/mihari/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/test.yml)
-[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/mihari)](https://hub.docker.com/r/ninoseki/mihari)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 

data/lib/mihari/analyzers/base.rb

@@ -111,7 +111,7 @@ module Mihari
       # @return [Array<Mihari::Artifact>]
       #
       def enriched_artifacts
-        @enriched_artifacts ||= unique_artifacts.map do |artifact|
+        @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact|
           artifact.enrich_all
           artifact
         end

data/lib/mihari/analyzers/feed.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "mihari/feed/reader"
+require "mihari/feed/parser"
+
+module Mihari
+  module Analyzers
+    class Feed < Base
+      param :query
+      option :title, default: proc { "Feed" }
+      option :description, default: proc { "query = #{query}" }
+      option :tags, default: proc { [] }
+
+      option :http_request_method, default: proc { "GET" }
+      option :http_request_headers, default: proc { {} }
+      option :http_request_payload, default: proc { {} }
+      option :http_request_payload_type, default: proc {}
+
+      option :selector, default: proc { "" }
+
+      def artifacts
+        Mihari::Feed::Parser.new(data).parse selector
+      end
+
+      private
+
+      def data
+        reader = Mihari::Feed::Reader.new(
+          query,
+          http_request_method: http_request_method,
+          http_request_headers: http_request_headers,
+          http_request_payload: http_request_payload,
+          http_request_payload_type: http_request_payload_type
+        )
+        reader.read
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/rule.rb

@@ -11,6 +11,7 @@ module Mihari
       "crtsh" => Crtsh,
       "dnpedia" => DNPedia,
       "dnstwister" => DNSTwister,
+      "feed" => Feed,
       "greynoise" => GreyNoise,
       "onyphe" => Onyphe,
       "otx" => OTX,

data/lib/mihari/cli/analyzer.rb

@@ -6,6 +6,7 @@ require "mihari/commands/circl"
 require "mihari/commands/crtsh"
 require "mihari/commands/dnpedia"
 require "mihari/commands/dnstwister"
+require "mihari/commands/feed"
 require "mihari/commands/greynoise"
 require "mihari/commands/onyphe"
 require "mihari/commands/otx"
@@ -35,6 +36,7 @@ module Mihari
       include Mihari::Commands::Crtsh
       include Mihari::Commands::DNPedia
       include Mihari::Commands::DNSTwister
+      include Mihari::Commands::Feed
       include Mihari::Commands::GreyNoise
       include Mihari::Commands::JSON
       include Mihari::Commands::Onyphe

data/lib/mihari/commands/feed.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Commands
+    module Feed
+      def self.included(thor)
+        thor.class_eval do
+          desc "feed [URL]", "ingest feed"
+          method_option :title, type: :string, desc: "title"
+          method_option :description, type: :string, desc: "description"
+          method_option :tags, type: :array, desc: "tags"
+          method_option :http_request_method, type: :string, desc: "HTTP request method"
+          method_option :http_request_headers, type: :hash, desc: "HTTP request headers"
+          method_option :http_request_payload_type, type: :string, desc: "HTTP request payload type"
+          method_option :http_request_payload, type: :hash, desc: "HTTP request payload"
+          method_option :selector, type: :string, desc: "jr selector", required: true
+          def feed(query)
+            with_error_handling do
+              run_analyzer Analyzers::Feed, query: query, options: options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/database.rb

@@ -2,7 +2,7 @@
 
 require "active_record"
 
-class InitialSchema < ActiveRecord::Migration[6.1]
+class InitialSchema < ActiveRecord::Migration[7.0]
   def change
     create_table :tags, if_not_exists: true do |t|
       t.string :name, null: false
@@ -32,13 +32,13 @@ class InitialSchema < ActiveRecord::Migration[6.1]
   end
 end
 
-class AddeSourceToArtifactSchema < ActiveRecord::Migration[6.1]
+class AddeSourceToArtifactSchema < ActiveRecord::Migration[7.0]
   def change
     add_column :artifacts, :source, :string, if_not_exists: true
   end
 end
 
-class EnrichmentsSchema < ActiveRecord::Migration[6.1]
+class EnrichmentsSchema < ActiveRecord::Migration[7.0]
   def change
     create_table :autonomous_systems, if_not_exists: true do |t|
       t.integer :asn, null: false
@@ -74,7 +74,7 @@ class EnrichmentsSchema < ActiveRecord::Migration[6.1]
   end
 end
 
-class EnrichmentCreatedAtSchema < ActiveRecord::Migration[6.1]
+class EnrichmentCreatedAtSchema < ActiveRecord::Migration[7.0]
   def change
     # Add created_at column because now it is able to enrich an atrifact after the creation
     add_column :autonomous_systems, :created_at, :datetime, if_not_exists: true

data/lib/mihari/enrichers/ipinfo.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "http"
 require "json"
 require "memist"

data/lib/mihari/errors.rb

@@ -8,4 +8,8 @@ module Mihari
   class RetryableError < Error; end
 
   class FileNotFoundError < Error; end
+
+  class HttpError < Error; end
+
+  class FeedParseError < Error; end
 end

data/lib/mihari/feed/parser.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "jr/cli/core_ext"
+
+module Mihari
+  module Feed
+    class Parser
+      attr_reader :data
+
+      #
+      # @param [Array<Hash>, Array<Array<String>>] data
+      #
+      def initialize(data)
+        @data = data
+      end
+
+      #
+      # Parse data by selector
+      #
+      # @param [String] selector
+      #
+      # @return [Array<String>]
+      #
+      def parse(selector)
+        parsed = data.instance_eval(selector)
+
+        raise FeedParseError unless parsed.is_a?(Array) || parsed.is_a?(Enumerator)
+        raise FeedParseError unless parsed.all?(String)
+
+        parsed.to_a
+      end
+    end
+  end
+end

data/lib/mihari/feed/reader.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "csv"
+require "json"
+require "net/https"
+require "uri"
+
+module Mihari
+  module Feed
+    class Reader
+      attr_reader :uri, :http_request_headers, :http_request_method, :http_request_payload_type, :http_request_payload
+
+      def initialize(url, http_request_headers: {}, http_request_method: "GET", http_request_payload_type: nil, http_request_payload: {})
+        @uri = URI(url)
+        @http_request_headers = http_request_headers
+        @http_request_method = http_request_method
+        @http_request_payload_type = http_request_payload_type
+        @http_request_payload = http_request_payload
+      end
+
+      def read
+        return get if http_request_method == "GET"
+
+        post
+      end
+
+      def get
+        uri.query = URI.encode_www_form(http_request_payload)
+        get = Net::HTTP::Get.new(uri)
+
+        request(get)
+      end
+
+      def post
+        post = Net::HTTP::Post.new(uri)
+
+        case http_request_payload_type
+        when "application/json"
+          post.body = JSON.generate(http_request_payload)
+        when "application/x-www-form-urlencoded"
+          post.set_form_data(http_request_payload)
+        end
+
+        request(post)
+      end
+
+      #
+      # Convert text as JSON
+      #
+      # @param [String] text
+      #
+      # @return [Array<Hash>]
+      #
+      def convert_as_json(text)
+        data = JSON.parse(text, symbolize_names: true)
+        return data if data.is_a?(Array)
+
+        [data]
+      end
+
+      #
+      # Convert text as CSV
+      #
+      # @param [String] text
+      #
+      # @return [Array<Hash>]
+      #
+      def convert_as_csv(text)
+        text_without_comments = text.lines.reject { |line| line.start_with? "#" }.join("\n")
+
+        CSV.new(text_without_comments).to_a.reject(&:empty?)
+      end
+
+      def https_options
+        return { use_ssl: true } if uri.scheme == "https"
+
+        {}
+      end
+
+      #
+      # Make a HTTP request
+      #
+      # @param [Net::HTTPRequest] req
+      #
+      # @return [Array<Hash>]
+      #
+      def request(req)
+        Net::HTTP.start(uri.host, uri.port, https_options) do |http|
+          # set headers
+          http_request_headers.each do |k, v|
+            req[k] = v
+          end
+
+          response = http.request(req)
+
+          code = response.code.to_i
+          raise HttpError, "Unsupported response code returned: #{code}" if code != 200
+
+          body = response.body
+
+          content_type = response["Content-Type"].to_s
+          data = if content_type.include?("application/json")
+            convert_as_json(body)
+          else
+            convert_as_csv(body)
+          end
+
+          data
+        end
+      end
+    end
+  end
+end

data/lib/mihari/mixins/autonomous_system.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mihari
   module Mixins
     module AutonomousSystem

data/lib/mihari/mixins/configuration.rb

@@ -51,9 +51,9 @@ module Mihari
       # @return [String] A template for config
       #
       def config_template
-        config = Mihari.config.values.keys.map do |key|
+        config = Mihari.config.values.keys.to_h do |key|
           [key.to_s, nil]
-        end.to_h
+        end
 
         YAML.dump(config)
       end

data/lib/mihari/mixins/disallowed_data_value.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "mem"
 
 module Mihari

data/lib/mihari/mixins/rule.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "date"
 require "pathname"
 require "yaml"

data/lib/mihari/models/alert.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "active_record"
-require "active_record/filter"
 
 module Mihari
   class Alert < ActiveRecord::Base
@@ -55,7 +54,7 @@ module Mihari
         artifact = artifact.where(dns_records: { value: filter.dns_record }) if filter.dns_record
         artifact = artifact.where(reverse_dns_names: { name: filter.reverse_dns_name }) if filter.reverse_dns_name
         # get artifact ids if there is any valid filter for artifact
-        if filter.has_valid_artifact_filters
+        if filter.valid_artifact_filters?
           artifact_ids = artifact.pluck(:id)
           # set invalid ID if nothing is matched with the filters
           artifact_ids = [-1] if artifact_ids.empty?
@@ -70,10 +69,10 @@ module Mihari
         relation = relation.where(source: filter.source) if filter.source
         relation = relation.where(title: filter.title) if filter.title
 
-        relation = relation.filter(description: { like: "%#{filter.description}%" }) if filter.description
+        relation = relation.where("description LIKE ?", "%#{filter.description}%") if filter.description
 
-        relation = relation.filter(created_at: { gte: filter.from_at }) if filter.from_at
-        relation = relation.filter(created_at: { lte: filter.to_at }) if filter.to_at
+        relation = relation.where("alerts.created_at >= ?", filter.from_at) if filter.from_at
+        relation = relation.where("alerts.created_at <= ?", filter.to_at) if filter.to_at
 
         relation
       end

data/lib/mihari/models/artifact.rb

@@ -1,10 +1,9 @@
 # frozen_string_literal: true
 
 require "active_record"
-require "active_record/filter"
 require "active_support/core_ext/integer/time"
 require "active_support/core_ext/numeric/time"
-require "uri"
+require "addressable/uri"
 
 class ArtifactValidator < ActiveModel::Validator
   def validate(record)
@@ -119,7 +118,7 @@ module Mihari
     def normalize_as_domain(url_or_domain)
       return url_or_domain if data_type == "domain"
 
-      URI.parse(url_or_domain).host
+      Addressable::URI.parse(url_or_domain).host
     end
 
     def can_enrich_whois?

data/lib/mihari/models/dns.rb

@@ -37,7 +37,7 @@ module Mihari
         resources = Resolv::DNS.new.getresources(domain, resource_type)
         resource_name = resource_type.to_s.split("::").last
 
-        resources.map do |resource|
+        resources.filter_map do |resource|
           # A, AAAA
           if resource.respond_to?(:address)
             new(resource: resource_name, value: resource.address.to_s)
@@ -48,7 +48,7 @@ module Mihari
           elsif resource.respond_to?(:data)
             new(resource: resource_name, value: resource.data.to_s)
           end
-        end.compact
+        end
       end
     end
   end

data/lib/mihari/schemas/rule.rb

@@ -44,6 +44,17 @@ module Mihari
       optional(:options).hash(AnalyzerOptions)
     end
 
+    Feed = Dry::Schema.Params do
+      required(:analyzer).value(Types::String.enum("feed"))
+      required(:query).value(:string)
+      required(:http_request_method).value(Types::FeedHttpRequestMethods).default("GET")
+      required(:http_request_headers).value(:hash).default({})
+      required(:http_request_payload).value(:hash).default({})
+      required(:selector).value(:string)
+      optional(:http_request_payload_type).value(Types::FeedHttpRequestPayloadTypes)
+      optional(:options).hash(AnalyzerOptions)
+    end
+
     Rule = Dry::Schema.Params do
       required(:title).value(:string)
       required(:description).value(:string)
@@ -55,7 +66,7 @@ module Mihari
       optional(:created_on).value(:date)
       optional(:updated_on).value(:date)
 
-      required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh }
+      required(:queries).value(:array).each { Analyzer | Spyse | ZoomEye | Urlscan | Crtsh | Feed }
 
       optional(:allowed_data_types).value(array[Types::DataTypes]).default(ALLOWED_DATA_TYPES)
       optional(:disallowed_data_values).value(array[:string]).default([])

data/lib/mihari/status.rb

@@ -18,11 +18,11 @@ module Mihari
     # @return [Array<Hash>]
     #
     def statuses
-      (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).map do |klass|
+      (Mihari.analyzers + Mihari.emitters + Mihari.enrichers).to_h do |klass|
         name = klass.to_s.split("::").last.to_s
 
         [name, build_status(klass)]
-      end.to_h.compact
+      end.compact
     end
 
     #

data/lib/mihari/structs/alert.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 
@@ -16,7 +18,7 @@ module Mihari
         attribute? :dns_record, Types::String.optional
         attribute? :reverse_dns_name, Types::String.optional
 
-        def has_valid_artifact_filters
+        def valid_artifact_filters?
           !(artifact_data || asn || dns_record || reverse_dns_name).nil?
         end
       end

data/lib/mihari/structs/censys.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/structs/greynoise.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/structs/ipinfo.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/structs/onyphe.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/structs/shodan.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/structs/urlscan.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/structs/virustotal_intelligence.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "json"
 require "dry/struct"
 

data/lib/mihari/types.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "dry/types"
 
 module Mihari
@@ -22,6 +24,7 @@ module Mihari
       "circl",
       "dnpedia",
       "dnstwister",
+      "feed",
       "greynoise",
       "onyphe",
       "otx",
@@ -36,5 +39,8 @@ module Mihari
       "vt_intel",
       "vt"
     )
+
+    FeedHttpRequestMethods = Types::String.enum("GET", "POST")
+    FeedHttpRequestPayloadTypes = Types::String.enum("application/json", "application/x-www-form-urlencoded")
   end
 end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "3.11.0"
+  VERSION = "3.12.0"
 end

data/lib/mihari/web/api.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Entities
 require "mihari/web/entities/message"
 

data/lib/mihari/web/entities/artifact.rb

@@ -14,10 +14,10 @@ module Mihari
       expose :whois_record, using: Entities::WhoisRecord, documentation: { type: Entities::WhoisRecord, required: false }, as: :whoisRecord
 
       expose :reverse_dns_names, using: Entities::ReverseDnsName, documentation: { type: Entities::ReverseDnsName, is_array: true, required: false }, as: :reverseDnsNames do |status, _options|
-        status.reverse_dns_names.length > 0 ? status.reverse_dns_names : nil
+        status.reverse_dns_names.empty? ? nil : status.reverse_dns_names
       end
       expose :dns_records, using: Entities::DnsRecord, documentation: { type: Entities::DnsRecord, is_array: true, required: false }, as: :dnsRecords do |status, _options|
-        status.dns_records.length > 0 ? status.dns_records : nil
+        status.dns_records.empty? ? nil : status.dns_records
       end
     end
   end

data/lib/mihari/web/entities/whois.rb

@@ -9,7 +9,7 @@ module Mihari
       expose :expires_on, documentation: { type: Date, required: false }, as: :expiresOn
       expose :registrar, documentation: { type: Hash, required: false }
       expose :contacts, documentation: { type: Hash, is_array: true, required: true } do |whois_record, _options|
-        whois_record.contacts.map { |h| h.to_camelback_keys }
+        whois_record.contacts.map(&:to_camelback_keys)
       end
     end
   end

data/lib/mihari/web/public/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/static/favicon.ico"><title>Mihari</title><link href="/static/js/app.fbc19869.js" rel="preload" as="script"></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/static/js/app.fbc19869.js"></script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/static/favicon.ico"><title>Mihari</title><link href="/static/js/app.5dc97aae.js" rel="preload" as="script"></head><body><noscript><strong>We're sorry but Mihari doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/static/js/app.5dc97aae.js"></script></body></html>