mihari 0.17.4 → 1.1.1

data/lib/mihari/emitters/base.rb

@@ -16,7 +16,7 @@ module Mihari
       end
 
       def run(**params)
-        retry_on_error { emit(params) }
+        retry_on_error { emit(**params) }
       end
 
       def emit(*)

data/lib/mihari/emitters/database.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Mihari
+  module Emitters
+    class Database < Base
+      def valid?
+        true
+      end
+
+      def emit(title:, description:, artifacts:, source:, tags: [])
+        return if artifacts.empty?
+
+        tags = tags.map { |name| Tag.find_or_create_by(name: name) }.compact.uniq
+        taggings = tags.map { |tag| Tagging.new(tag_id: tag.id) }
+
+        alert = Alert.new(
+          title: title,
+          description: description,
+          artifacts: artifacts,
+          source: source,
+          taggings: taggings
+        )
+
+        alert.save
+        alert
+      end
+    end
+  end
+end

data/lib/mihari/emitters/misp.rb

@@ -6,6 +6,13 @@ require "net/ping"
 module Mihari
   module Emitters
     class MISP < Base
+      def initialize
+        ::MISP.configure do |config|
+          config.api_endpoint = Mihari.config.misp_api_endpoint
+          config.api_key = Mihari.config.misp_api_key
+        end
+      end
+
       # @return [true, false]
       def valid?
         api_endpoint? && api_key? && ping?
@@ -28,7 +35,7 @@ module Mihari
       private
 
       def config_keys
-        %w(MISP_API_ENDPOINT MISP_API_KEY)
+        %w(misp_api_endpoint misp_api_key)
       end
 
       def build_attribute(artifact)

data/lib/mihari/emitters/slack.rb

@@ -4,6 +4,8 @@ require "slack-notifier"
 require "digest/sha2"
 require "mem"
 
+require "mihari/slack_monkeypatch"
+
 module Mihari
   module Emitters
     class Attachment
@@ -123,7 +125,7 @@ module Mihari
         ].join("\n")
       end
 
-      def emit(title:, description:, artifacts:, tags: [])
+      def emit(title:, description:, artifacts:, tags: [], **_options)
         return if artifacts.empty?
 
         attachments = to_attachments(artifacts)
@@ -135,7 +137,7 @@ module Mihari
       private
 
       def config_keys
-        %w(SLACK_WEBHOOK_URL)
+        %w(slack_webhook_url)
       end
     end
   end

data/lib/mihari/emitters/stdout.rb

@@ -9,11 +9,12 @@ module Mihari
         true
       end
 
-      def emit(title:, description:, artifacts:, tags:)
+      def emit(title:, description:, artifacts:, source:, tags:)
         h = {
           title: title,
           description: description,
           artifacts: artifacts.map(&:data),
+          source: source,
           tags: tags
         }
         puts JSON.pretty_generate(h)

data/lib/mihari/emitters/the_hive.rb

@@ -1,42 +1,56 @@
 # frozen_string_literal: true
 
+require "hachi"
+require "net/ping"
+
 module Mihari
   module Emitters
     class TheHive < Base
       # @return [true, false]
       def valid?
-        the_hive.valid?
+        api_endpont? && api_key? && ping?
       end
 
-      def emit(title:, description:, artifacts:, tags: [])
+      def emit(title:, description:, artifacts:, tags: [], **_options)
         return if artifacts.empty?
 
-        the_hive.alert.create(
+        api.alert.create(
           title: title,
           description: description,
-          artifacts: artifacts.map(&:to_h),
-          tags: tags
+          artifacts: artifacts.map { |artifact| { data: artifact.data, data_type: artifact.data_type, message: description } },
+          tags: tags,
+          type: "external",
+          source: "mihari"
         )
-
-        save_as_cache artifacts.map(&:data)
       end
 
       private
 
       def config_keys
-        %w(THEHIVE_API_ENDPOINT THEHIVE_API_KEY)
+        %w(thehive_api_endpoint thehive_api_key)
       end
 
-      def the_hive
-        @the_hive ||= Mihari::TheHive.new
+      def api
+        @api ||= Hachi::API.new(api_endpoint: Mihari.config.thehive_api_endpoint, api_key: Mihari.config.thehive_api_key)
       end
 
-      def cache
-        @cache ||= Cache.new
+      # @return [true, false]
+      def api_endpont?
+        !Mihari.config.thehive_api_endpoint.nil?
       end
 
-      def save_as_cache(data)
-        cache.save data
+      # @return [true, false]
+      def api_key?
+        !Mihari.config.thehive_api_key.nil?
+      end
+
+      def ping?
+        base_url = Mihari.config.thehive_api_endpoint
+        base_url = base_url.end_with?("/") ? base_url[0..-2] : base_url
+        url = "#{base_url}/index.html"
+
+        http = Net::Ping::HTTP.new(url)
+        http.ping?
       end
     end
   end

data/lib/mihari/models/alert.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Mihari
+  class Alert < ActiveRecord::Base
+    has_many :taggings, dependent: :destroy
+    has_many :artifacts, dependent: :destroy
+    has_many :tags, through: :taggings
+  end
+end

data/lib/mihari/models/artifact.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+class ArtifactValidator < ActiveModel::Validator
+  def validate(record)
+    return if record.data_type
+
+    record.errors[:data] << "#{record.data} is not supported"
+  end
+end
+
+module Mihari
+  class Artifact < ActiveRecord::Base
+    include ActiveModel::Validations
+    validates_with ArtifactValidator
+
+    def initialize(attributes)
+      super
+      self.data_type = TypeChecker.type(data)
+    end
+
+    def unique?
+      self.class.find_by(data: data).nil?
+    end
+  end
+end

data/lib/mihari/models/tag.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Mihari
+  class Tag < ActiveRecord::Base
+    has_many :taggings, dependent: :destroy
+    has_many :tags, through: :taggings
+  end
+end

data/lib/mihari/models/tagging.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Mihari
+  class Tagging < ActiveRecord::Base
+    belongs_to :alert
+    belongs_to :tag
+  end
+end

data/lib/mihari/notifiers/slack.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "slack-notifier"
+require "mihari/slack_monkeypatch"
+
 module Mihari
   module Notifiers
     class Slack < Base
@@ -8,15 +11,15 @@ module Mihari
       DEFAULT_USERNAME = "mihari"
 
       def slack_channel
-        ENV.fetch SLACK_CHANNEL_KEY, "#general"
+        Mihari.config.slack_channel || "#general"
       end
 
       def slack_webhook_url
-        ENV.fetch SLACK_WEBHOOK_URL_KEY
+        Mihari.config.slack_webhook_url
       end
 
       def slack_webhook_url?
-        ENV.key? SLACK_WEBHOOK_URL_KEY
+        !Mihari.config.slack_webhook_url.nil?
       end
 
       def valid?
@@ -25,7 +28,7 @@ module Mihari
 
       def notify(text:, attachments: [], mrkdwn: true)
         notifier = ::Slack::Notifier.new(slack_webhook_url, channel: slack_channel, username: DEFAULT_USERNAME)
-        notifier.post(text: text, attachments: attachments, mrkdwn: true)
+        notifier.post(text: text, attachments: attachments, mrkdwn: mrkdwn)
       end
     end
   end

data/lib/mihari/serializers/alert.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "active_model_serializers"
+
+module Mihari
+  class AlertSerializer < ActiveModel::Serializer
+    attributes :title, :description, :source, :created_at
+
+    has_many :artifacts
+    has_many :tags, through: :taggings
+  end
+end

data/lib/mihari/serializers/artifact.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "active_model_serializers"
+
+module Mihari
+  class ArtifactSerializer < ActiveModel::Serializer
+    attributes :data, :data_type
+  end
+end

data/lib/mihari/serializers/tag.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "active_model_serializers"
+
+module Mihari
+  class TagSerializer < ActiveModel::Serializer
+    attributes :name
+  end
+end

data/lib/mihari/slack_monkeypatch.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Slack
+  class Notifier
+    module Util
+      class LinkFormatter
+        class << self
+          def format(string, opts = {})
+            # Resolve warning in Ruby 2.7
+            LinkFormatter.new(string, **opts).formatted
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/status.rb

@@ -4,7 +4,7 @@ module Mihari
   class Status
     def check
       statuses.map do |key, value|
-        [key, convert(value)]
+        [key, convert(**value)]
       end.to_h
     end
 

data/lib/mihari/type_checker.rb

@@ -44,7 +44,7 @@ module Mihari
 
     # @return [true, false]
     def mail?
-      EmailAddress.valid? data
+      EmailAddress.valid? data, host_validation: :syntax
     end
 
     # @return [String, nil]

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "0.17.4"
+  VERSION = "1.1.1"
 end

data/mihari.gemspec

@@ -26,35 +26,42 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "coveralls", "~> 0.8"
-  spec.add_development_dependency "fakefs", "~> 1.0"
+  spec.add_development_dependency "execjs", "~> 2.7"
+  spec.add_development_dependency "fakefs", "~> 1.2"
+  spec.add_development_dependency "pre-commit", "~> 0.39"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
+  spec.add_development_dependency "rubocop", "~> 0.88"
+  spec.add_development_dependency "rubocop-performance", "~> 1.7"
   spec.add_development_dependency "timecop", "~> 0.9"
-  spec.add_development_dependency "vcr", "~> 5.0"
-  spec.add_development_dependency "webmock", "~> 3.7"
+  spec.add_development_dependency "vcr", "~> 6.0"
+  spec.add_development_dependency "webmock", "~> 3.8"
 
+  spec.add_dependency "active_model_serializers", "~> 0.10"
+  spec.add_dependency "activerecord", "~> 6.0"
   spec.add_dependency "addressable", "~> 2.7"
   spec.add_dependency "binaryedge", "~> 0.1"
   spec.add_dependency "censu", "~> 0.2"
-  spec.add_dependency "crtsh-rb", "~> 0.2"
+  spec.add_dependency "crtsh-rb", "~> 0.3"
   spec.add_dependency "dnpedia", "~> 0.1"
   spec.add_dependency "dnstwister", "~> 0.1"
   spec.add_dependency "email_address", "~> 0.1"
   spec.add_dependency "hachi", "~> 0.3"
-  spec.add_dependency "lightly", "~> 0.3"
   spec.add_dependency "mem", "~> 0.1"
   spec.add_dependency "misp", "~> 0.1"
   spec.add_dependency "murmurhash3", "~> 0.1"
   spec.add_dependency "net-ping", "~> 2.0"
-  spec.add_dependency "onyphe", "~> 1.1"
+  spec.add_dependency "onyphe", "~> 2.0"
   spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
+  spec.add_dependency "pg", "~> 1.2"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "pulsedive", "~> 0.1"
   spec.add_dependency "securitytrails", "~> 1.0"
   spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.3"
+  spec.add_dependency "sqlite3", "~> 1.4"
   spec.add_dependency "thor", "~> 1.0"
   spec.add_dependency "urlscan", "~> 0.5"
   spec.add_dependency "virustotalx", "~> 1.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 0.17.4
+  version: 1.1.1
 platform: ruby
 authors:
 - Manabu Niseki
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-01-13 00:00:00.000000000 Z
+date: 2020-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,20 +38,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: execjs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: pre-commit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.39'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.39'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +108,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.88'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.88'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -100,28 +156,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '3.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: active_model_serializers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -170,14 +254,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: dnpedia
   requirement: !ruby/object:Gem::Requirement
@@ -234,20 +318,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
-- !ruby/object:Gem::Dependency
-  name: lightly
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: mem
   requirement: !ruby/object:Gem::Requirement
@@ -310,14 +380,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -360,6 +430,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: public_suffix
   requirement: !ruby/object:Gem::Requirement
@@ -430,6 +514,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -496,6 +594,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".travis.yml"
 - Gemfile
 - LICENSE
@@ -503,6 +602,7 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- config/pre_commit.yml
 - docker/Dockerfile
 - examples/ipinfo_hosted_domains.rb
 - exe/mihari
@@ -531,27 +631,31 @@ files:
 - lib/mihari/analyzers/urlscan.rb
 - lib/mihari/analyzers/virustotal.rb
 - lib/mihari/analyzers/zoomeye.rb
-- lib/mihari/artifact.rb
-- lib/mihari/cache.rb
 - lib/mihari/cli.rb
 - lib/mihari/config.rb
 - lib/mihari/configurable.rb
+- lib/mihari/database.rb
 - lib/mihari/emitters/base.rb
+- lib/mihari/emitters/database.rb
 - lib/mihari/emitters/misp.rb
 - lib/mihari/emitters/slack.rb
 - lib/mihari/emitters/stdout.rb
 - lib/mihari/emitters/the_hive.rb
 - lib/mihari/errors.rb
 - lib/mihari/html.rb
+- lib/mihari/models/alert.rb
+- lib/mihari/models/artifact.rb
+- lib/mihari/models/tag.rb
+- lib/mihari/models/tagging.rb
 - lib/mihari/notifiers/base.rb
 - lib/mihari/notifiers/exception_notifier.rb
 - lib/mihari/notifiers/slack.rb
 - lib/mihari/retriable.rb
+- lib/mihari/serializers/alert.rb
+- lib/mihari/serializers/artifact.rb
+- lib/mihari/serializers/tag.rb
+- lib/mihari/slack_monkeypatch.rb
 - lib/mihari/status.rb
-- lib/mihari/the_hive.rb
-- lib/mihari/the_hive/alert.rb
-- lib/mihari/the_hive/artifact.rb
-- lib/mihari/the_hive/base.rb
 - lib/mihari/type_checker.rb
 - lib/mihari/version.rb
 - mihari.gemspec
@@ -563,7 +667,7 @@ homepage: https://github.com/ninoseki/mihari
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -578,8 +682,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: A framework for continuous malicious hosts monitoring.
 test_files: []