mihari 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44cc5eea0746aa47918f5102cf4834fcd436b67f371b37f8a1e4fe6999902c82
-  data.tar.gz: e0dd701f1d5b705239d73125ca8865fa15c949d4b62cb59a2c9beef8e01e363d
+  metadata.gz: 46761bb6c390e76763b5b76c711cd222a08735bb3e539862bb2cac211d44bfc3
+  data.tar.gz: 19cdc765f98da35dd299038ce8f310ce30cd4a7903a2c56344e97631fdf32f87
 SHA512:
-  metadata.gz: 36722428cc688eaefd3f293f1de99db20b8671945082b1d5f1d08d61c9b977491e46f45122311010650d80b1f0077dd7ec9323e304c02fdcf9c4fc1b22c66672
-  data.tar.gz: ad9e0603040bc4069194df7f21c0963e096beaa3e6e2ac13ea0d66803217619b885f3afef4624c0c45ae4282c64f8bda905bf61d8c564f9aef8c7a5b33bb2dca
+  metadata.gz: eda0f8f7537c6a9a456c225545bc615d3573f4789ce9b411faf3fc83cf155417adac87bd043e605303874199d24395f2a71e06275cae5909b9e2fb95221e4651
+  data.tar.gz: 38a9250b72efb654b1e8c10904077f205838a7d61742f3e545cef26d3703a161c60b77c1459ee10fd0d2fa814e1b234663fd13132cc2d406833aaf6d51519de8

data/.travis.yml

@@ -1,8 +1,13 @@
 ---
-sudo: false
 language: ruby
 cache: bundler
+services:
+  - postgresql
 rvm:
   - 2.6
   - 2.7
+env:
+  - DATABASE=":memory:"
+  - DATABASE="postgresql://postgres@0.0.0.0:5432/travis_ci_test"
 before_install: gem install bundler -v 2.1
+before_script: psql -c 'create database travis_ci_test;' -U postgres

data/README.md

@@ -11,7 +11,7 @@ Mihari is a helper to run queries & manage results continuously. Mihari can be u
 ## How it works
 
 - Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes) from the results.
-- Mihari checks whether a DB (SQLite3) contains the artifacts or not.
+- Mihari checks whether a DB (SQLite3 or PostgreSQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
     - Mihari creates an alert on TheHive. (Optional)
     - Mihari sends a notification to Slack. (Optional)
@@ -189,29 +189,29 @@ The input is a JSON data should have `title`, `description` and `artifacts` key.
 
 Configuration can be done via environment variables or a YAML file.
 
-| Key                    | Description                    | Default     |
-|------------------------|--------------------------------|-------------|
-| DATABASE               | A path to the SQLite database  | `mihari.db` |
-| BINARYEDGE_API_KEY     | BinaryEdge API key             |             |
-| CENSYS_ID              | Censys API ID                  |             |
-| CENSYS_SECRET          | Censys secret                  |             |
-| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password |             |
-| CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username |             |
-| MISP_API_ENDPOINT      | MISP URL                       |             |
-| MISP_API_KEY           | MISP API key                   |             |
-| ONYPHE_API_KEY         | Onyphe API key                 |             |
-| PASSIVETOTAL_API_KEY   | PassiveTotal API key           |             |
-| PASSIVETOTAL_USERNAME  | PassiveTotal username          |             |
-| PULSEDIVE_API_KEY      | Pulsedive API key              |             |
-| SECURITYTRAILS_API_KEY | SecurityTrails API key         |             |
-| SHODAN_API_KEY         | Shodan API key                 |             |
-| SLACK_CHANNEL          | Slack channel name             | `#general`  |
-| SLACK_WEBHOOK_URL      | Slack Webhook URL              |             |
-| THEHIVE_API_ENDPOINT   | TheHive URL                    |             |
-| THEHIVE_API_KEY        | TheHive API key                |             |
-| VIRUSTOTAL_API_KEY     | VirusTotal API key             |             |
-| ZOOMEYE_PASSWORD       | ZoomEye password               |             |
-| ZOOMEYE_USERNAMME      | ZoomEye username               |             |
+| Key                    | Description                                                                                     | Default     |
+|------------------------|-------------------------------------------------------------------------------------------------|-------------|
+| DATABASE               | A path to the SQLite database or a DB URL (e.g. `postgres://postgres:pass@db.host:5432/somedb`) | `mihari.db` |
+| BINARYEDGE_API_KEY     | BinaryEdge API key                                                                              |             |
+| CENSYS_ID              | Censys API ID                                                                                   |             |
+| CENSYS_SECRET          | Censys secret                                                                                   |             |
+| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password                                                                  |             |
+| CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username                                                                  |             |
+| MISP_API_ENDPOINT      | MISP URL                                                                                        |             |
+| MISP_API_KEY           | MISP API key                                                                                    |             |
+| ONYPHE_API_KEY         | Onyphe API key                                                                                  |             |
+| PASSIVETOTAL_API_KEY   | PassiveTotal API key                                                                            |             |
+| PASSIVETOTAL_USERNAME  | PassiveTotal username                                                                           |             |
+| PULSEDIVE_API_KEY      | Pulsedive API key                                                                               |             |
+| SECURITYTRAILS_API_KEY | SecurityTrails API key                                                                          |             |
+| SHODAN_API_KEY         | Shodan API key                                                                                  |             |
+| SLACK_CHANNEL          | Slack channel name                                                                              | `#general`  |
+| SLACK_WEBHOOK_URL      | Slack Webhook URL                                                                               |             |
+| THEHIVE_API_ENDPOINT   | TheHive URL                                                                                     |             |
+| THEHIVE_API_KEY        | TheHive API key                                                                                 |             |
+| VIRUSTOTAL_API_KEY     | VirusTotal API key                                                                              |             |
+| ZOOMEYE_PASSWORD       | ZoomEye password                                                                                |             |
+| ZOOMEYE_USERNAMME      | ZoomEye username                                                                                |             |
 
 Instead of using environment variables, you can use a YAML file for configuration.
 

data/docker/Dockerfile

@@ -1,12 +1,12 @@
 FROM ruby:2.6-alpine3.10
-RUN apk --no-cache add git build-base ruby-dev \
+RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/mihari.git \
   && cd mihari \
   && gem build mihari.gemspec -o mihari.gem \
   && gem install mihari.gem \
   && rm -rf /tmp/mihari \
-  && apk del --purge git build-base ruby-dev
+  && apk del --purge git build-base ruby-dev sqlite-dev postgresql-dev
 
 ENTRYPOINT ["mihari"]
 

data/lib/mihari.rb

@@ -71,9 +71,9 @@ require "mihari/notifiers/slack"
 require "mihari/notifiers/exception_notifier"
 
 require "mihari/emitters/base"
+require "mihari/emitters/database"
 require "mihari/emitters/misp"
 require "mihari/emitters/slack"
-require "mihari/emitters/sqlite"
 require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
 

data/lib/mihari/alert_viewer.rb

@@ -2,17 +2,21 @@
 
 module Mihari
   class AlertViewer
-    attr_reader :limit
+    def list(title: nil, source: nil, tag: nil, limit: 5)
+      limit = limit.to_i
+      raise ArgumentError, "limit should be bigger than zero" unless limit.positive?
 
-    def initialize(limit: 5)
-      @limit = limit.to_i
-      raise ArgumentError, "limit should be bigger than zero" unless @limit.positive?
-    end
+      relation = Alert.includes(:tags, :artifacts)
+      relation = relation.where(title: title) if title
+      relation = relation.where(source: source) if source
+      relation = relation.where(tags: { name: tag } ) if tag
 
-    def list
-      alerts = Alert.order(id: :desc).limit(limit).includes(:tags, :artifacts)
+      alerts = relation.limit(limit).order(id: :desc)
       alerts.map do |alert|
-        AlertSerializer.new(alert).as_json
+        json = AlertSerializer.new(alert).as_json
+        json[:artifacts] = (json.dig(:artifacts) || []).map { |artifact_| artifact_.dig(:data) }
+        json[:tags] = (json.dig(:tags) || []).map { |tag_| tag_.dig(:name) }
+        json
       end
     end
   end

data/lib/mihari/analyzers/binaryedge.rb

@@ -52,7 +52,7 @@ module Mihari
       end
 
       def config_keys
-        [Mihari.config.binaryedge_api_key]
+        %w(binaryedge_api_key)
       end
 
       def api

data/lib/mihari/analyzers/censys.rb

@@ -86,7 +86,7 @@ module Mihari
       end
 
       def config_keys
-        [Mihari.config.censys_id, Mihari.config.censys_secret]
+        %w(censys_id censys_secret)
       end
 
       def api

data/lib/mihari/analyzers/circl.rb

@@ -27,7 +27,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.circl_passive_password, Mihari.config.circl_passive_username]
+        %w(circl_passive_password circl_passive_username)
       end
 
       def api

data/lib/mihari/analyzers/onyphe.rb

@@ -35,7 +35,7 @@ module Mihari
       PAGE_SIZE = 10
 
       def config_keys
-        [Mihari.config.onyphe_api_key]
+        %w(onyphe_api_key)
       end
 
       def api

data/lib/mihari/analyzers/passivetotal.rb

@@ -30,7 +30,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.passivetotal_username, Mihari.config.passivetotal_api_key]
+        %w(passivetotal_username passivetotal_api_key)
       end
 
       def api

data/lib/mihari/analyzers/pulsedive.rb

@@ -30,7 +30,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.pulsedive_api_key]
+        %w(pulsedive_api_key)
       end
 
       def api

data/lib/mihari/analyzers/securitytrails.rb

@@ -30,7 +30,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.securitytrails_api_key]
+        %w(securitytrails_api_key)
       end
 
       def api

data/lib/mihari/analyzers/securitytrails_domain_feed.rb

@@ -32,7 +32,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.securitytrails_api_key]
+        %w(securitytrails_api_key)
       end
 
       def api

data/lib/mihari/analyzers/shodan.rb

@@ -36,7 +36,7 @@ module Mihari
       PAGE_SIZE = 100
 
       def config_keys
-        [Mihari.config.shodan_api_key]
+        %w(shodan_api_key)
       end
 
       def api

data/lib/mihari/analyzers/virustotal.rb

@@ -30,7 +30,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.virustotal_api_key]
+        %w(virustotal_api_key)
       end
 
       def api

data/lib/mihari/analyzers/zoomeye.rb

@@ -41,7 +41,7 @@ module Mihari
       end
 
       def config_keys
-        [Mihari.config.zoomeye_password, Mihari.config.zoomeye_username]
+        %w(zoomeye_password zoomeye_username)
       end
 
       def api

data/lib/mihari/cli.rb

@@ -249,10 +249,15 @@ module Mihari
 
     desc "alerts", "Show the alerts on TheHive"
     method_option :limit, type: :string, default: "5", desc: "Number of alerts to show (or 'all' to show all the alerts)"
+    method_option :title, type: :string, desc: "Title to filter"
+    method_option :source, type: :string, desc: "Source to filter"
+    method_option :tag, type: :string, desc: "Tag to filter"
     def alerts
       with_error_handling do
-        viewer = AlertViewer.new(limit: options["limit"])
-        alerts = viewer.list
+        load_configuration
+
+        viewer = AlertViewer.new
+        alerts = viewer.list(limit: options["limit"], title: options["title"], source: options["source"], tag: options[:tag])
         puts JSON.pretty_generate(alerts)
       end
     end
@@ -261,6 +266,7 @@ module Mihari
     def status
       with_error_handling do
         load_configuration
+
         puts JSON.pretty_generate(Status.check)
       end
     end
@@ -289,8 +295,7 @@ module Mihari
         return unless config
 
         Config.load_from_yaml(config)
-        # reload database settings
-        load File.expand_path("./database.rb", __dir__)
+        Database.connect
       end
 
       def run_analyzer(analyzer_class, query:, options:)

data/lib/mihari/configurable.rb

@@ -3,7 +3,7 @@
 module Mihari
   module Configurable
     def configured?
-      config_keys.all? { |key| !key.nil? }
+      config_keys.all? { |key| Mihari.config.send(key) }
     end
 
     def configuration_status

data/lib/mihari/database.rb

@@ -2,11 +2,6 @@
 
 require "active_record"
 
-ActiveRecord::Base.establish_connection(
-  adapter: "sqlite3",
-  database: Mihari.config.database
-)
-
 class InitialSchema < ActiveRecord::Migration[6.0]
   def change
     create_table :tags, if_not_exists: true do |t|
@@ -37,9 +32,37 @@ class InitialSchema < ActiveRecord::Migration[6.0]
   end
 end
 
-begin
-  ActiveRecord::Migration.verbose = false
-  InitialSchema.migrate(:up)
-rescue StandardError
-  # Do nothing
+def adapter
+  return "postgresql" if Mihari.config.database.start_with?("postgresql://", "postgres://")
+
+  "sqlite3"
 end
+
+module Mihari
+  class Database
+    class << self
+      def connect
+        case adapter
+        when "postgresql"
+          ActiveRecord::Base.establish_connection(Mihari.config.database)
+        else
+          ActiveRecord::Base.establish_connection(
+            adapter: adapter,
+            database: Mihari.config.database
+          )
+        end
+
+        ActiveRecord::Migration.verbose = false
+        InitialSchema.migrate(:up)
+      rescue StandardError
+        # Do nothing
+      end
+
+      def destroy!
+        InitialSchema.migrate(:down) if ActiveRecord::Base.connected?
+      end
+    end
+  end
+end
+
+Mihari::Database.connect

data/lib/mihari/emitters/{sqlite.rb → database.rb}

@@ -2,7 +2,7 @@
 
 module Mihari
   module Emitters
-    class SQLite < Base
+    class Database < Base
       def valid?
         true
       end

data/lib/mihari/emitters/misp.rb

@@ -35,7 +35,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.misp_api_endpoint, Mihari.config.misp_api_key]
+        %w(misp_api_endpoint misp_api_key)
       end
 
       def build_attribute(artifact)

data/lib/mihari/emitters/slack.rb

@@ -135,7 +135,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.slack_webhook_url]
+        %w(slack_webhook_url)
       end
     end
   end

data/lib/mihari/emitters/the_hive.rb

@@ -27,7 +27,7 @@ module Mihari
       private
 
       def config_keys
-        [Mihari.config.thehive_api_endpoint, Mihari.config.thehive_api_key]
+        %w(thehive_api_endpoint thehive_api_key)
       end
 
       def api

data/lib/mihari/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mihari
-  VERSION = "1.0.1"
+  VERSION = "1.1.0"
 end

data/mihari.gemspec

@@ -31,7 +31,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pre-commit", "~> 0.39"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "rubocop", "~> 0.82"
   spec.add_development_dependency "rubocop-performance", "~> 1.5"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 5.1"
@@ -55,6 +54,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "passive_circl", "~> 0.1"
   spec.add_dependency "passivetotalx", "~> 0.1"
+  spec.add_dependency "pg", "~> 1.2"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "pulsedive", "~> 0.1"
   spec.add_dependency "securitytrails", "~> 1.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mihari
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-03 00:00:00.000000000 Z
+date: 2020-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,20 +108,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.9'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.82'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.82'
 - !ruby/object:Gem::Dependency
   name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
@@ -430,6 +416,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: public_suffix
   requirement: !ruby/object:Gem::Requirement
@@ -622,9 +622,9 @@ files:
 - lib/mihari/configurable.rb
 - lib/mihari/database.rb
 - lib/mihari/emitters/base.rb
+- lib/mihari/emitters/database.rb
 - lib/mihari/emitters/misp.rb
 - lib/mihari/emitters/slack.rb
-- lib/mihari/emitters/sqlite.rb
 - lib/mihari/emitters/stdout.rb
 - lib/mihari/emitters/the_hive.rb
 - lib/mihari/errors.rb