mihari 1.0.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44cc5eea0746aa47918f5102cf4834fcd436b67f371b37f8a1e4fe6999902c82
-  data.tar.gz: e0dd701f1d5b705239d73125ca8865fa15c949d4b62cb59a2c9beef8e01e363d
+  metadata.gz: 6d6b5d42dc4fbe937cc101f7dbdfa1491104f2b6f70fdf690b2ad51db02304c5
+  data.tar.gz: 32b5df5f6970d6aaf6f9d5c57ca9bc86d8e43f0298f22c6c9e811cf60a6d1089
 SHA512:
-  metadata.gz: 36722428cc688eaefd3f293f1de99db20b8671945082b1d5f1d08d61c9b977491e46f45122311010650d80b1f0077dd7ec9323e304c02fdcf9c4fc1b22c66672
-  data.tar.gz: ad9e0603040bc4069194df7f21c0963e096beaa3e6e2ac13ea0d66803217619b885f3afef4624c0c45ae4282c64f8bda905bf61d8c564f9aef8c7a5b33bb2dca
+  metadata.gz: 16cdfd458c0d464eb618fc5aae8386c1cb37fd554fd6aba275323cb539a08916f3cd855e4e5683085e03f206debe879af6f2e3c7a42b1cd0b2a6395d86e3f55a
+  data.tar.gz: 93f9e9e2165e79ef6602d1628b5689e0993ae4d88605db4f6aa4ff79dc6a21d25437997a1402f76ce3a5e2e9155ad68a441a8cc8351b54c5f4d1c3a72eb82df4

data/.travis.yml

@@ -1,8 +1,13 @@
 ---
-sudo: false
 language: ruby
 cache: bundler
+services:
+  - postgresql
 rvm:
   - 2.6
   - 2.7
+env:
+  - DATABASE=":memory:"
+  - DATABASE="postgresql://postgres@0.0.0.0:5432/travis_ci_test"
 before_install: gem install bundler -v 2.1
+before_script: psql -c 'create database travis_ci_test;' -U postgres

data/README.md

@@ -11,7 +11,7 @@ Mihari is a helper to run queries & manage results continuously. Mihari can be u
 ## How it works
 
 - Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes) from the results.
-- Mihari checks whether a DB (SQLite3) contains the artifacts or not.
+- Mihari checks whether a DB (SQLite3 or PostgreSQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
     - Mihari creates an alert on TheHive. (Optional)
     - Mihari sends a notification to Slack. (Optional)
@@ -33,6 +33,17 @@ Mihari is a helper to run queries & manage results continuously. Mihari can be u
 
 ![img](https://github.com/ninoseki/mihari/raw/master/screenshots/misp.png)
 
+## Requirements
+
+- Ruby 2.6+
+- SQLite3
+- libpq
+
+```bash
+# For Debian / Ubuntu
+apt-get install sqlite3 libsqlite3-dev libpq-dev
+```
+
 ## Installation
 
 ```bash
@@ -56,9 +67,11 @@ Mihari supports the following services by default.
 - [DN Pedia](https://dnpedia.com/)
 - [dnstwister](https://dnstwister.report/)
 - [Onyphe](https://onyphe.io)
+- [OTX](https://otx.alienvault.com/)
 - [PassiveTotal](https://community.riskiq.com/)
 - [SecurityTrails](https://securitytrails.com/)
 - [Shodan](https://shodan.io)
+- [Spyse](https://spyse.com)
 - [urlscan.io](https://urlscan.io)
 - [VirusTotal](http://virustotal.com)
 - [ZoomEye](https://zoomeye.org)
@@ -78,6 +91,7 @@ Commands:
   mihari http_hash                            # Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
   mihari import_from_json                     # Give a JSON input via STDIN
   mihari onyphe [QUERY]                       # Onyphe datascan search by a query
+  mihari otx [IP|DOMAIN]                      # OTX lookup by an IP or domain
   mihari passive_dns [IP|DOMAIN]              # Cross search with passive DNS services by an ip or domain
   mihari passive_ssl [SHA1]                   # Cross search with passive SSL services by an SHA1 certificate fingerprint
   mihari passivetotal [IP|DOMAIN|EMAIL|SHA1]  # PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint
@@ -86,6 +100,7 @@ Commands:
   mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by an ip, domain or email
   mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed search by a regexp
   mihari shodan [QUERY]                       # Shodan host search by a query
+  mihari spyse [QUERY]                        # Spyse search by a query
   mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)
   mihari status                               # Show the current configuration status
   mihari urlscan [QUERY]                      # urlscan search by a given query
@@ -105,7 +120,7 @@ You can get aggregated results by using the following commands.
 
 | Command         | Desc.                                                                                                   |
 |-----------------|---------------------------------------------------------------------------------------------------------|
-| passive_dns     | Passive DNS lookup with CIRCL passive DNS, PassiveTotal, Pulsedive, SecurityTrails and VirusTotal       |
+| passive_dns     | Passive DNS lookup with CIRCL passive DNS, OTX, PassiveTotal, Pulsedive, SecurityTrails and VirusTotal  |
 | passive_ssl     | Passive SSL lookup with CIRCL passive SSL and PassiveTotal                                              |
 | reverse_whois   | Revese Whois lookup with PassiveTotal and SecurityTrails                                                |
 | http_hash       | HTTP response hash lookup with BinaryEdge(SHA256), Censys(SHA256), Onyphpe(MD5) and Shodan(MurmurHash3) |
@@ -189,29 +204,31 @@ The input is a JSON data should have `title`, `description` and `artifacts` key.
 
 Configuration can be done via environment variables or a YAML file.
 
-| Key                    | Description                    | Default     |
-|------------------------|--------------------------------|-------------|
-| DATABASE               | A path to the SQLite database  | `mihari.db` |
-| BINARYEDGE_API_KEY     | BinaryEdge API key             |             |
-| CENSYS_ID              | Censys API ID                  |             |
-| CENSYS_SECRET          | Censys secret                  |             |
-| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password |             |
-| CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username |             |
-| MISP_API_ENDPOINT      | MISP URL                       |             |
-| MISP_API_KEY           | MISP API key                   |             |
-| ONYPHE_API_KEY         | Onyphe API key                 |             |
-| PASSIVETOTAL_API_KEY   | PassiveTotal API key           |             |
-| PASSIVETOTAL_USERNAME  | PassiveTotal username          |             |
-| PULSEDIVE_API_KEY      | Pulsedive API key              |             |
-| SECURITYTRAILS_API_KEY | SecurityTrails API key         |             |
-| SHODAN_API_KEY         | Shodan API key                 |             |
-| SLACK_CHANNEL          | Slack channel name             | `#general`  |
-| SLACK_WEBHOOK_URL      | Slack Webhook URL              |             |
-| THEHIVE_API_ENDPOINT   | TheHive URL                    |             |
-| THEHIVE_API_KEY        | TheHive API key                |             |
-| VIRUSTOTAL_API_KEY     | VirusTotal API key             |             |
-| ZOOMEYE_PASSWORD       | ZoomEye password               |             |
-| ZOOMEYE_USERNAMME      | ZoomEye username               |             |
+| Key                    | Description                                                                                     | Default     |
+|------------------------|-------------------------------------------------------------------------------------------------|-------------|
+| DATABASE               | A path to the SQLite database or a DB URL (e.g. `postgres://postgres:pass@db.host:5432/somedb`) | `mihari.db` |
+| BINARYEDGE_API_KEY     | BinaryEdge API key                                                                              |             |
+| CENSYS_ID              | Censys API ID                                                                                   |             |
+| CENSYS_SECRET          | Censys secret                                                                                   |             |
+| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password                                                                  |             |
+| CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username                                                                  |             |
+| MISP_API_ENDPOINT      | MISP URL                                                                                        |             |
+| MISP_API_KEY           | MISP API key                                                                                    |             |
+| ONYPHE_API_KEY         | Onyphe API key                                                                                  |             |
+| OTX_API_KEY            | OTX API key                                                                                     |             |
+| PASSIVETOTAL_API_KEY   | PassiveTotal API key                                                                            |             |
+| PASSIVETOTAL_USERNAME  | PassiveTotal username                                                                           |             |
+| PULSEDIVE_API_KEY      | Pulsedive API key                                                                               |             |
+| SECURITYTRAILS_API_KEY | SecurityTrails API key                                                                          |             |
+| SHODAN_API_KEY         | Shodan API key                                                                                  |             |
+| SLACK_CHANNEL          | Slack channel name                                                                              | `#general`  |
+| SLACK_WEBHOOK_URL      | Slack Webhook URL                                                                               |             |
+| SPYSE_API_KEY          | Spyse API key                                                                                   |             |
+| THEHIVE_API_ENDPOINT   | TheHive URL                                                                                     |             |
+| THEHIVE_API_KEY        | TheHive API key                                                                                 |             |
+| VIRUSTOTAL_API_KEY     | VirusTotal API key                                                                              |             |
+| ZOOMEYE_PASSWORD       | ZoomEye password                                                                                |             |
+| ZOOMEYE_USERNAMME      | ZoomEye username                                                                                |             |
 
 Instead of using environment variables, you can use a YAML file for configuration.
 

data/docker/Dockerfile

@@ -1,5 +1,5 @@
-FROM ruby:2.6-alpine3.10
-RUN apk --no-cache add git build-base ruby-dev \
+FROM ruby:2.7-alpine3.10
+RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/mihari.git \
   && cd mihari \

data/lib/mihari.rb

@@ -50,11 +50,13 @@ require "mihari/analyzers/crtsh"
 require "mihari/analyzers/dnpedia"
 require "mihari/analyzers/dnstwister"
 require "mihari/analyzers/onyphe"
+require "mihari/analyzers/otx"
 require "mihari/analyzers/passivetotal"
 require "mihari/analyzers/pulsedive"
 require "mihari/analyzers/securitytrails_domain_feed"
 require "mihari/analyzers/securitytrails"
 require "mihari/analyzers/shodan"
+require "mihari/analyzers/spyse"
 require "mihari/analyzers/urlscan"
 require "mihari/analyzers/virustotal"
 require "mihari/analyzers/zoomeye"
@@ -71,9 +73,9 @@ require "mihari/notifiers/slack"
 require "mihari/notifiers/exception_notifier"
 
 require "mihari/emitters/base"
+require "mihari/emitters/database"
 require "mihari/emitters/misp"
 require "mihari/emitters/slack"
-require "mihari/emitters/sqlite"
 require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
 

data/lib/mihari/alert_viewer.rb

@@ -2,17 +2,21 @@
 
 module Mihari
   class AlertViewer
-    attr_reader :limit
+    def list(title: nil, source: nil, tag: nil, limit: 5)
+      limit = limit.to_i
+      raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
 
-    def initialize(limit: 5)
-      @limit = limit.to_i
-      raise ArgumentError, "limit should be bigger than zero" unless @limit.positive?
-    end
+      relation = Alert.includes(:tags, :artifacts)
+      relation = relation.where(title: title) if title
+      relation = relation.where(source: source) if source
+      relation = relation.where(tags: { name: tag } ) if tag
 
-    def list
-      alerts = Alert.order(id: :desc).limit(limit).includes(:tags, :artifacts)
+      alerts = relation.limit(limit).order(id: :desc)
       alerts.map do |alert|
-        AlertSerializer.new(alert).as_json
+        json = AlertSerializer.new(alert).as_json
+        json[:artifacts] = (json.dig(:artifacts) || []).map { |artifact_| artifact_.dig(:data) }
+        json[:tags] = (json.dig(:tags) || []).map { |tag_| tag_.dig(:name) }
+        json
       end
     end
   end

data/lib/mihari/analyzers/binaryedge.rb

@@ -52,7 +52,7 @@ module Mihari
       end
 
       def config_keys
-        [Mihari.config.binaryedge_api_key]
+        %w(binaryedge_api_key)
       end
 
       def api

data/lib/mihari/analyzers/censys.rb

@@ -86,7 +86,7 @@ module Mihari
       end
 
       def config_keys
-        [Mihari.config.censys_id, Mihari.config.censys_secret]
+        %w(censys_id censys_secret)
       end
 
       def api

data/lib/mihari/analyzers/circl.rb

@@ -27,7 +27,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.circl_passive_password, Mihari.config.circl_passive_username]
+        %w(circl_passive_password circl_passive_username)
       end
 
       def api

data/lib/mihari/analyzers/onyphe.rb

@@ -35,7 +35,7 @@ module Mihari
       PAGE_SIZE = 10
 
       def config_keys
-        [Mihari.config.onyphe_api_key]
+        %w(onyphe_api_key)
       end
 
       def api

data/lib/mihari/analyzers/otx.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "otx_ruby"
+
+module Mihari
+  module Analyzers
+    class OTX < Base
+      attr_reader :query
+      attr_reader :type
+
+      attr_reader :title
+      attr_reader :description
+      attr_reader :tags
+
+      def initialize(query, title: nil, description: nil, tags: [])
+        super()
+
+        @query = query
+        @type = TypeChecker.type(query)
+
+        @title = title || "OTX lookup"
+        @description = description || "query = #{query}"
+        @tags = tags
+      end
+
+      def artifacts
+        lookup || []
+      end
+
+      private
+
+      def config_keys
+        %w(otx_api_key)
+      end
+
+      def domain_client
+        @domain_client ||= ::OTX::Domain.new(Mihari.config.otx_api_key)
+      end
+
+      def ip_client
+        @ip_client ||= ::OTX::IP.new(Mihari.config.otx_api_key)
+      end
+
+      def valid_type?
+        %w(ip domain).include? type
+      end
+
+      def lookup
+        case type
+        when "domain"
+          domain_lookup
+        when "ip"
+          ip_lookup
+        else
+          raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        end
+      end
+
+      def domain_lookup
+        records = domain_client.get_passive_dns(query)
+        records.map do |record|
+          record.address if record.record_type == "A"
+        end.compact.uniq
+      end
+
+      def ip_lookup
+        records = ip_client.get_passive_dns(query)
+        records.map do |record|
+          record.hostname if record.record_type == "A"
+        end.compact.uniq
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/passive_dns.rb

@@ -14,6 +14,7 @@ module Mihari
 
       ANALYZERS = [
         Mihari::Analyzers::CIRCL,
+        Mihari::Analyzers::OTX,
         Mihari::Analyzers::PassiveTotal,
         Mihari::Analyzers::Pulsedive,
         Mihari::Analyzers::SecurityTrails,
@@ -55,7 +56,7 @@ module Mihari
         analyzer.artifacts
       rescue ArgumentError, InvalidInputError => _e
         nil
-      rescue ::PassiveCIRCL::Error, ::PassiveTotal::Error, ::Pulsedive::ResponseError, ::SecurityTrails::Error, ::VirusTotal::Error => _e
+      rescue Faraday::Error, ::PassiveCIRCL::Error, ::PassiveTotal::Error, ::Pulsedive::ResponseError, ::SecurityTrails::Error, ::VirusTotal::Error => _e
         nil
       end
     end

data/lib/mihari/analyzers/passivetotal.rb

@@ -30,7 +30,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.passivetotal_username, Mihari.config.passivetotal_api_key]
+        %w(passivetotal_username passivetotal_api_key)
       end
 
       def api

data/lib/mihari/analyzers/pulsedive.rb

@@ -30,7 +30,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.pulsedive_api_key]
+        %w(pulsedive_api_key)
       end
 
       def api

data/lib/mihari/analyzers/securitytrails.rb

@@ -30,7 +30,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.securitytrails_api_key]
+        %w(securitytrails_api_key)
       end
 
       def api

data/lib/mihari/analyzers/securitytrails_domain_feed.rb

@@ -32,7 +32,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.securitytrails_api_key]
+        %w(securitytrails_api_key)
       end
 
       def api

data/lib/mihari/analyzers/shodan.rb

@@ -36,7 +36,7 @@ module Mihari
       PAGE_SIZE = 100
 
       def config_keys
-        [Mihari.config.shodan_api_key]
+        %w(shodan_api_key)
       end
 
       def api

data/lib/mihari/analyzers/spyse.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "spyse"
+require "json"
+
+module Mihari
+  module Analyzers
+    class Spyse < Base
+      attr_reader :query
+      attr_reader :type
+
+      attr_reader :title
+      attr_reader :description
+      attr_reader :tags
+
+      def initialize(query, title: nil, description: nil, tags: [], type: "domain")
+        super()
+
+        @query = query
+
+        @title = title || "Spyse lookup"
+        @description = description || "query = #{query}"
+        @tags = tags
+        @type = type
+      end
+
+      def artifacts
+        lookup || []
+      end
+
+      private
+
+      def search_params
+        @search_params ||= JSON.parse(query)
+      end
+
+      def config_keys
+        %w(spyse_api_key)
+      end
+
+      def api
+        @api ||= ::Spyse::API.new(Mihari.config.spyse_api_key)
+      end
+
+      def valid_type?
+        %w(ip domain cert).include? type
+      end
+
+      def domain_lookup
+        res = api.domain.search(search_params, limit: 100)
+        items = res.dig("data", "items") || []
+        items.map do |item|
+          item.dig("name")
+        end.uniq.compact
+      end
+
+      def ip_lookup
+        res = api.ip.search(search_params, limit: 100)
+        items = res.dig("data", "items") || []
+        items.map do |item|
+          item.dig("ip")
+        end.uniq.compact
+      end
+
+      def lookup
+        case type
+        when "domain"
+          domain_lookup
+        when "ip"
+          ip_lookup
+        else
+          raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        end
+      end
+    end
+  end
+end

data/lib/mihari/analyzers/virustotal.rb

@@ -30,7 +30,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.virustotal_api_key]
+        %w(virustotal_api_key)
       end
 
       def api

data/lib/mihari/analyzers/zoomeye.rb

@@ -41,7 +41,7 @@ module Mihari
       end
 
       def config_keys
-        [Mihari.config.zoomeye_password, Mihari.config.zoomeye_username]
+        %w(zoomeye_password zoomeye_username)
       end
 
       def api

data/lib/mihari/cli.rb

@@ -164,6 +164,27 @@ module Mihari
       end
     end
 
+    desc "otx [IP|DOMAIN]", "OTX lookup by an IP or domain"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    def otx(domain)
+      with_error_handling do
+        run_analyzer Analyzers::OTX, query: refang(domain), options: options
+      end
+    end
+
+    desc "spyse [QUERY]", "Spyse search by a query"
+    method_option :title, type: :string, desc: "title"
+    method_option :description, type: :string, desc: "description"
+    method_option :tags, type: :array, desc: "tags"
+    method_option :type, type: :string, desc: "type to search (ip or domain)", default: "doamin"
+    def spyse(query)
+      with_error_handling do
+        run_analyzer Analyzers::Spyse, query: query, options: options
+      end
+    end
+
     desc "passive_dns [IP|DOMAIN]", "Cross search with passive DNS services by an ip or domain"
     method_option :title, type: :string, desc: "title"
     method_option :description, type: :string, desc: "description"
@@ -249,10 +270,15 @@ module Mihari
 
     desc "alerts", "Show the alerts on TheHive"
     method_option :limit, type: :string, default: "5", desc: "Number of alerts to show (or 'all' to show all the alerts)"
+    method_option :title, type: :string, desc: "Title to filter"
+    method_option :source, type: :string, desc: "Source to filter"
+    method_option :tag, type: :string, desc: "Tag to filter"
     def alerts
       with_error_handling do
-        viewer = AlertViewer.new(limit: options["limit"])
-        alerts = viewer.list
+        load_configuration
+
+        viewer = AlertViewer.new
+        alerts = viewer.list(limit: options["limit"], title: options["title"], source: options["source"], tag: options[:tag])
         puts JSON.pretty_generate(alerts)
       end
     end
@@ -261,6 +287,7 @@ module Mihari
     def status
       with_error_handling do
         load_configuration
+
         puts JSON.pretty_generate(Status.check)
       end
     end
@@ -289,8 +316,7 @@ module Mihari
         return unless config
 
         Config.load_from_yaml(config)
-        # reload database settings
-        load File.expand_path("./database.rb", __dir__)
+        Database.connect
       end
 
       def run_analyzer(analyzer_class, query:, options:)

data/lib/mihari/config.rb

@@ -12,6 +12,7 @@ module Mihari
     attr_accessor :misp_api_endpoint
     attr_accessor :misp_api_key
     attr_accessor :onyphe_api_key
+    attr_accessor :otx_api_key
     attr_accessor :passivetotal_api_key
     attr_accessor :passivetotal_username
     attr_accessor :pulsedive_api_key
@@ -19,6 +20,7 @@ module Mihari
     attr_accessor :shodan_api_key
     attr_accessor :slack_channel
     attr_accessor :slack_webhook_url
+    attr_accessor :spyse_api_key
     attr_accessor :thehive_api_endpoint
     attr_accessor :thehive_api_key
     attr_accessor :virustotal_api_key
@@ -40,6 +42,7 @@ module Mihari
       @misp_api_endpoint = ENV["MISP_API_ENDPOINT"]
       @misp_api_key = ENV["MISP_API_KEY"]
       @onyphe_api_key = ENV["ONYPHE_API_KEY"]
+      @otx_api_key = ENV["OTX_API_KEY"]
       @passivetotal_api_key = ENV["PASSIVETOTAL_API_KEY"]
       @passivetotal_username = ENV["PASSIVETOTAL_USERNAME"]
       @pulsedive_api_key = ENV["PULSEDIVE_API_KEY"]
@@ -47,6 +50,7 @@ module Mihari
       @shodan_api_key = ENV["SHODAN_API_KEY"]
       @slack_channel = ENV["SLACK_CHANNEL"]
       @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
+      @spyse_api_key = ENV["SPYSE_API_KEY"]
       @thehive_api_endpoint = ENV["THEHIVE_API_ENDPOINT"]
       @thehive_api_key = ENV["THEHIVE_API_KEY"]
       @virustotal_api_key = ENV["VIRUSTOTAL_API_KEY"]

data/lib/mihari/configurable.rb

@@ -3,7 +3,7 @@
 module Mihari
   module Configurable
     def configured?
-      config_keys.all? { |key| !key.nil? }
+      config_keys.all? { |key| Mihari.config.send(key) }
     end
 
     def configuration_status

data/lib/mihari/database.rb

@@ -2,11 +2,6 @@
 
 require "active_record"
 
-ActiveRecord::Base.establish_connection(
-  adapter: "sqlite3",
-  database: Mihari.config.database
-)
-
 class InitialSchema < ActiveRecord::Migration[6.0]
   def change
     create_table :tags, if_not_exists: true do |t|
@@ -37,9 +32,37 @@ class InitialSchema < ActiveRecord::Migration[6.0]
   end
 end
 
-begin
-  ActiveRecord::Migration.verbose = false
-  InitialSchema.migrate(:up)
-rescue StandardError
-  # Do nothing
+def adapter
+  return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
+
+  "sqlite3"
 end
+
+module Mihari
+  class Database
+    class << self
+      def connect
+        case adapter
+        when "postgresql"
+          ActiveRecord::Base.establish_connection(Mihari.config.database)
+        else
+          ActiveRecord::Base.establish_connection(
+            adapter: adapter,
+            database: Mihari.config.database
+          )
+        end
+
+        ActiveRecord::Migration.verbose = false
+        InitialSchema.migrate(:up)
+      rescue StandardError
+        # Do nothing
+      end
+
+      def destroy!
+        InitialSchema.migrate(:down) if ActiveRecord::Base.connected?
+      end
+    end
+  end
+end
+
+Mihari::Database.connect

data/lib/mihari/emitters/{sqlite.rb → database.rb}

@@ -2,7 +2,7 @@
 
 module Mihari
   module Emitters
-    class SQLite < Base
+    class Database < Base
       def valid?
         true
       end

data/lib/mihari/emitters/misp.rb

@@ -35,7 +35,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.misp_api_endpoint, Mihari.config.misp_api_key]
+        %w(misp_api_endpoint misp_api_key)
       end
 
       def build_attribute(artifact)

data/lib/mihari/emitters/slack.rb

@@ -4,6 +4,8 @@ require "slack-notifier"
 require "digest/sha2"
 require "mem"
 
+require "mihari/slack_monkeypatch"
+
 module Mihari
   module Emitters
     class Attachment
@@ -135,7 +137,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.slack_webhook_url]
+        %w(slack_webhook_url)
       end
     end
   end

data/lib/mihari/emitters/the_hive.rb

@@ -27,7 +27,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.thehive_api_endpoint, Mihari.config.thehive_api_key]
+        %w(thehive_api_endpoint thehive_api_key)
       end
 
       def api

data/lib/mihari/notifiers/slack.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "slack-notifier"
+require "mihari/slack_monkeypatch"
+
 module Mihari
   module Notifiers
     class Slack < Base

data/lib/mihari/retriable.rb

@@ -7,10 +7,10 @@ module Mihari
       begin
         try += 1
         yield
-      rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error => _e
+      rescue Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, Timeout::Error, ::Shodan::Error => e
         sleep interval
         retry if try < times
-        raise
+        raise e
       end
     end
   end

data/lib/mihari/slack_monkeypatch.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Slack
+  class Notifier
+    module Util
+      class LinkFormatter
+        class << self
+          def format(string, opts = {})
+            # Resolve warning in Ruby 2.7
+            LinkFormatter.new(string, **opts).formatted
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "1.0.1"
+  VERSION = "1.3.0"
 end

data/mihari.gemspec

@@ -26,15 +26,15 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "coveralls", "~> 0.8"
-  spec.add_development_dependency "execjs", "~> 2.0"
+  spec.add_development_dependency "execjs", "~> 2.7"
   spec.add_development_dependency "fakefs", "~> 1.2"
   spec.add_development_dependency "pre-commit", "~> 0.39"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "rubocop", "~> 0.82"
-  spec.add_development_dependency "rubocop-performance", "~> 1.5"
+  spec.add_development_dependency "rubocop", "~> 0.88"
+  spec.add_development_dependency "rubocop-performance", "~> 1.7"
   spec.add_development_dependency "timecop", "~> 0.9"
-  spec.add_development_dependency "vcr", "~> 5.1"
+  spec.add_development_dependency "vcr", "~> 6.0"
   spec.add_development_dependency "webmock", "~> 3.8"
 
   spec.add_dependency "active_model_serializers", "~> 0.10"
@@ -52,14 +52,17 @@ Gem::Specification.new do |spec|
   spec.add_dependency "murmurhash3", "~> 0.1"
   spec.add_dependency "net-ping", "~> 2.0"
   spec.add_dependency "onyphe", "~> 2.0"
+  spec.add_dependency "otx_ruby", "~> 0.9"
   spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
+  spec.add_dependency "pg", "~> 1.2"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "pulsedive", "~> 0.1"
   spec.add_dependency "securitytrails", "~> 1.0"
   spec.add_dependency "shodanx", "~> 0.2"
   spec.add_dependency "slack-notifier", "~> 2.3"
+  spec.add_dependency "spysex", "~> 0.1"
   spec.add_dependency "sqlite3", "~> 1.4"
   spec.add_dependency "thor", "~> 1.0"
   spec.add_dependency "urlscan", "~> 0.5"

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Manabu Niseki
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-05-03 00:00:00.000000000 Z
+date: 2020-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
@@ -114,28 +114,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.82'
+        version: '0.88'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.82'
+        version: '0.88'
 - !ruby/object:Gem::Dependency
   name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -156,14 +156,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -388,6 +388,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: otx_ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -430,6 +444,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: public_suffix
   requirement: !ruby/object:Gem::Requirement
@@ -500,6 +528,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: spysex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -605,6 +647,7 @@ files:
 - lib/mihari/analyzers/free_text.rb
 - lib/mihari/analyzers/http_hash.rb
 - lib/mihari/analyzers/onyphe.rb
+- lib/mihari/analyzers/otx.rb
 - lib/mihari/analyzers/passive_dns.rb
 - lib/mihari/analyzers/passive_ssl.rb
 - lib/mihari/analyzers/passivetotal.rb
@@ -613,6 +656,7 @@ files:
 - lib/mihari/analyzers/securitytrails.rb
 - lib/mihari/analyzers/securitytrails_domain_feed.rb
 - lib/mihari/analyzers/shodan.rb
+- lib/mihari/analyzers/spyse.rb
 - lib/mihari/analyzers/ssh_fingerprint.rb
 - lib/mihari/analyzers/urlscan.rb
 - lib/mihari/analyzers/virustotal.rb
@@ -622,9 +666,9 @@ files:
 - lib/mihari/configurable.rb
 - lib/mihari/database.rb
 - lib/mihari/emitters/base.rb
+- lib/mihari/emitters/database.rb
 - lib/mihari/emitters/misp.rb
 - lib/mihari/emitters/slack.rb
-- lib/mihari/emitters/sqlite.rb
 - lib/mihari/emitters/stdout.rb
 - lib/mihari/emitters/the_hive.rb
 - lib/mihari/errors.rb
@@ -640,10 +684,12 @@ files:
 - lib/mihari/serializers/alert.rb
 - lib/mihari/serializers/artifact.rb
 - lib/mihari/serializers/tag.rb
+- lib/mihari/slack_monkeypatch.rb
 - lib/mihari/status.rb
 - lib/mihari/type_checker.rb
 - lib/mihari/version.rb
 - mihari.gemspec
+- renovate.json
 - screenshots/alert.png
 - screenshots/eyecatch.png
 - screenshots/misp.png
@@ -652,7 +698,7 @@ homepage: https://github.com/ninoseki/mihari
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -668,7 +714,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A framework for continuous malicious hosts monitoring.
 test_files: []