mihari 1.3.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d6b5d42dc4fbe937cc101f7dbdfa1491104f2b6f70fdf690b2ad51db02304c5
-  data.tar.gz: 32b5df5f6970d6aaf6f9d5c57ca9bc86d8e43f0298f22c6c9e811cf60a6d1089
+  metadata.gz: 70afeb6b1ddaa263689beb836de85264ad1e871a1887a0574adfc22f00e006d8
+  data.tar.gz: ad614363a9a3320c2dfa34ec19bd4712d08a9a6e662cdf11b4b70c775f55e592
 SHA512:
-  metadata.gz: 16cdfd458c0d464eb618fc5aae8386c1cb37fd554fd6aba275323cb539a08916f3cd855e4e5683085e03f206debe879af6f2e3c7a42b1cd0b2a6395d86e3f55a
-  data.tar.gz: 93f9e9e2165e79ef6602d1628b5689e0993ae4d88605db4f6aa4ff79dc6a21d25437997a1402f76ce3a5e2e9155ad68a441a8cc8351b54c5f4d1c3a72eb82df4
+  metadata.gz: 8b7b9b86a6ec5341ce03b5652e29d45ccfafcd6ee09ca469ccf88d9872a965aded13fae1925c647cbbdedb389532469060698e35a06905197ffb143b223d0a93
+  data.tar.gz: 5d839d16358cf855658bc85d8e30868ce9fcae63cf9064461e67343cd979cee90daad2822260d96da9f00acadb9f981e29de89eb07b35047f9cd3ae039a37c3e

data/.github/workflows/test.yml

@@ -0,0 +1,44 @@
+name: Ruby CI
+
+on: [pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    services:
+      db:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, '3.0']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby 2.7
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Build and test with Rake
+      env:
+        DATABASE: postgresql://postgres:postgres@localhost:5432/test
+      run: |
+        sudo apt-get -yqq install libpq-dev
+        gem install bundler
+        bundle install
+        bundle exec rake

data/README.md

@@ -10,12 +10,12 @@ Mihari is a helper to run queries & manage results continuously. Mihari can be u
 
 ## How it works
 
-- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes) from the results.
+- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes).
 - Mihari checks whether a DB (SQLite3 or PostgreSQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
-    - Mihari creates an alert on TheHive. (Optional)
-    - Mihari sends a notification to Slack. (Optional)
-    - Mihari creates an event on MISP. (Optional)
+    - Mihari creates an alert on TheHive.
+    - Mihari sends a notification to Slack.
+    - Mihari creates an event on MISP.
 
 ![img](https://github.com/ninoseki/mihari/raw/master/screenshots/eyecatch.png)
 
@@ -35,9 +35,8 @@ Mihari is a helper to run queries & manage results continuously. Mihari can be u
 
 ## Requirements
 
-- Ruby 2.6+
-- SQLite3
-- libpq
+- Ruby (2.7 or 3.0)
+- SQLite3 or PostgreSQL
 
 ```bash
 # For Debian / Ubuntu
@@ -226,6 +225,7 @@ Configuration can be done via environment variables or a YAML file.
 | SPYSE_API_KEY          | Spyse API key                                                                                   |             |
 | THEHIVE_API_ENDPOINT   | TheHive URL                                                                                     |             |
 | THEHIVE_API_KEY        | TheHive API key                                                                                 |             |
+| URLSCAN_API_KEY        | urlscan.io API key                                                                              |             |
 | VIRUSTOTAL_API_KEY     | VirusTotal API key                                                                              |             |
 | ZOOMEYE_PASSWORD       | ZoomEye password                                                                                |             |
 | ZOOMEYE_USERNAMME      | ZoomEye username                                                                                |             |

data/Rakefile

@@ -2,6 +2,7 @@
 
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "standard/rake"
 
 RSpec::Core::RakeTask.new(:spec)
 

data/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2.7-alpine3.10
+FROM ruby:3.0.0-alpine3.13
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/mihari.git \

data/lib/mihari/alert_viewer.rb

@@ -9,13 +9,13 @@ module Mihari
       relation = Alert.includes(:tags, :artifacts)
       relation = relation.where(title: title) if title
       relation = relation.where(source: source) if source
-      relation = relation.where(tags: { name: tag } ) if tag
+      relation = relation.where(tags: {name: tag}) if tag
 
       alerts = relation.limit(limit).order(id: :desc)
       alerts.map do |alert|
         json = AlertSerializer.new(alert).as_json
-        json[:artifacts] = (json.dig(:artifacts) || []).map { |artifact_| artifact_.dig(:data) }
-        json[:tags] = (json.dig(:tags) || []).map { |tag_| tag_.dig(:name) }
+        json[:artifacts] = (json[:artifacts] || []).map { |artifact_| artifact_[:data] }
+        json[:tags] = (json[:tags] || []).map { |tag_| tag_[:name] }
         json
       end
     end

data/lib/mihari/analyzers/base.rb

@@ -42,7 +42,7 @@ module Mihari
 
       def run_emitter(emitter)
         emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
-      rescue StandardError => e
+      rescue => e
         puts "Emission by #{emitter.class} is failed: #{e}"
       end
 

data/lib/mihari/analyzers/basic.rb

@@ -4,12 +4,11 @@ module Mihari
   module Analyzers
     class Basic < Base
       attr_accessor :title
-      attr_reader :description
-      attr_reader :artifacts
-      attr_reader :source
-      attr_reader :tags
+      attr_reader :description, :artifacts, :source, :tags
 
       def initialize(title:, description:, artifacts:, source:, tags: [])
+        super()
+
         @title = title
         @description = description
         @artifacts = artifacts

data/lib/mihari/analyzers/binaryedge.rb

@@ -5,10 +5,7 @@ require "binaryedge"
 module Mihari
   module Analyzers
     class BinaryEdge < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
+      attr_reader :title, :description, :query, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -24,7 +21,7 @@ module Mihari
         return [] unless results || results.empty?
 
         results.map do |result|
-          events = result.dig("events") || []
+          events = result["events"] || []
           events.map do |event|
             event.dig "target", "ip"
           end.compact
@@ -37,13 +34,17 @@ module Mihari
 
       def search_with_page(query, page: 1)
         api.host.search(query, page: page)
+      rescue ::BinaryEdge::Error => e
+        raise RetryableError, e if e.message.include?("Request time limit exceeded")
+
+        raise e
       end
 
       def search
         responses = []
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
-          total = res.dig("total").to_i
+          total = res["total"].to_i
 
           responses << res
           break if total <= page * PAGE_SIZE
@@ -52,7 +53,7 @@ module Mihari
       end
 
       def config_keys
-        %w(binaryedge_api_key)
+        %w[binaryedge_api_key]
       end
 
       def api

data/lib/mihari/analyzers/censys.rb

@@ -5,11 +5,7 @@ require "censu"
 module Mihari
   module Analyzers
     class Censys < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
-      attr_reader :type
+      attr_reader :title, :description, :query, :tags, :type
 
       def initialize(query, title: nil, description: nil, tags: [], type: "ipv4")
         super()
@@ -37,7 +33,7 @@ module Mihari
       private
 
       def valid_type?
-        %w(ipv4 websites certificates).include? type
+        %w[ipv4 websites certificates].include? type
       end
 
       def normalize(domain)
@@ -86,7 +82,7 @@ module Mihari
       end
 
       def config_keys
-        %w(censys_id censys_secret)
+        %w[censys_id censys_secret]
       end
 
       def api

data/lib/mihari/analyzers/circl.rb

@@ -5,9 +5,7 @@ require "passive_circl"
 module Mihari
   module Analyzers
     class CIRCL < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -27,7 +25,7 @@ module Mihari
       private
 
       def config_keys
-        %w(circl_passive_password circl_passive_username)
+        %w[circl_passive_password circl_passive_username]
       end
 
       def api
@@ -41,7 +39,7 @@ module Mihari
         when "hash"
           passive_ssl_lookup
         else
-          raise InvalidInputError, "#{@query}(type: #{@type || 'unknown'}) is not supported."
+          raise InvalidInputError, "#{@query}(type: #{@type || "unknown"}) is not supported."
         end
       end
 

data/lib/mihari/analyzers/crtsh.rb

@@ -5,11 +5,7 @@ require "crtsh"
 module Mihari
   module Analyzers
     class Crtsh < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
-      attr_reader :exclude_expired
+      attr_reader :title, :description, :query, :tags, :exclude_expired
 
       def initialize(query, title: nil, description: nil, tags: [], exclude_expired: nil)
         super()
@@ -24,7 +20,7 @@ module Mihari
 
       def artifacts
         results = search
-        name_values = results.map { |result| result.dig("name_value") }.compact
+        name_values = results.map { |result| result["name_value"] }.compact
         name_values.map(&:lines).flatten.uniq.map(&:chomp)
       end
 

data/lib/mihari/analyzers/dnpedia.rb

@@ -5,10 +5,7 @@ require "dnpedia"
 module Mihari
   module Analyzers
     class DNPedia < Base
-      attr_reader :query
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -31,9 +28,9 @@ module Mihari
 
       def lookup
         res = api.search(query)
-        rows = res.dig("rows") || []
+        rows = res["rows"] || []
         rows.map do |row|
-          [row.dig("name"), row.dig("zoneid")].join(".")
+          [row["name"], row["zoneid"]].join(".")
         end
       end
     end

data/lib/mihari/analyzers/dnstwister.rb

@@ -7,12 +7,7 @@ require "parallel"
 module Mihari
   module Analyzers
     class DNSTwister < Base
-      attr_reader :query
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :type, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -47,11 +42,11 @@ module Mihari
       end
 
       def lookup
-        raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         res = api.fuzz(query)
-        fuzzy_domains = res.dig("fuzzy_domains") || []
-        domains = fuzzy_domains.map { |domain| domain.dig("domain") }
+        fuzzy_domains = res["fuzzy_domains"] || []
+        domains = fuzzy_domains.map { |domain| domain["domain"] }
         Parallel.map(domains) do |domain|
           resolvable?(domain) ? domain : nil
         end.compact

data/lib/mihari/analyzers/free_text.rb

@@ -5,15 +5,11 @@ require "parallel"
 module Mihari
   module Analyzers
     class FreeText < Base
-      attr_reader :query
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :title, :description, :tags
 
       ANALYZERS = [
         Mihari::Analyzers::BinaryEdge,
-        Mihari::Analyzers::Censys,
+        Mihari::Analyzers::Censys
       ].freeze
 
       def initialize(query, title: nil, description: nil, tags: [])

data/lib/mihari/analyzers/http_hash.rb

@@ -5,15 +5,7 @@ require "parallel"
 module Mihari
   module Analyzers
     class HTTPHash < Base
-      attr_reader :md5
-      attr_reader :sha256
-      attr_reader :mmh3
-
-      attr_reader :html
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :md5, :sha256, :mmh3, :html, :title, :description, :tags
 
       def initialize(_query, md5: nil, sha256: nil, mmh3: nil, html: nil, title: nil, description: nil, tags: [])
         super()
@@ -57,7 +49,7 @@ module Mihari
         [
           md5 ? "md5:#{md5}" : nil,
           sha256 ? "sha256:#{sha256}" : nil,
-          mmh3 ? "mmh3:#{mmh3}" : nil,
+          mmh3 ? "mmh3:#{mmh3}" : nil
         ].compact.join(",")
       end
 
@@ -92,7 +84,7 @@ module Mihari
           binary_edge,
           censys,
           onyphe,
-          shodan,
+          shodan
         ].compact
       end
 

data/lib/mihari/analyzers/onyphe.rb

@@ -5,10 +5,7 @@ require "onyphe"
 module Mihari
   module Analyzers
     class Onyphe < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
+      attr_reader :title, :description, :query, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -35,7 +32,7 @@ module Mihari
       PAGE_SIZE = 10
 
       def config_keys
-        %w(onyphe_api_key)
+        %w[onyphe_api_key]
       end
 
       def api
@@ -51,7 +48,7 @@ module Mihari
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
           responses << res
-          total = res.dig("total").to_i
+          total = res["total"].to_i
           break if total <= page * PAGE_SIZE
         end
         responses

data/lib/mihari/analyzers/otx.rb

@@ -5,12 +5,7 @@ require "otx_ruby"
 module Mihari
   module Analyzers
     class OTX < Base
-      attr_reader :query
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :type, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -30,7 +25,7 @@ module Mihari
       private
 
       def config_keys
-        %w(otx_api_key)
+        %w[otx_api_key]
       end
 
       def domain_client
@@ -42,7 +37,7 @@ module Mihari
       end
 
       def valid_type?
-        %w(ip domain).include? type
+        %w[ip domain].include? type
       end
 
       def lookup
@@ -52,7 +47,7 @@ module Mihari
         when "ip"
           ip_lookup
         else
-          raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+          raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
         end
       end