mihari 1.3.1 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ece49c7c528579aec4fec009765915a2afe72df87eca5c39ccb59a416a200c4
-  data.tar.gz: d4c595a7c38fb8e8b8350e38f90f572d8a74eef836817fea4ba21bb6893c3af3
+  metadata.gz: 951201ccebc7b6c4c117a687c1abce9ab24fa8d450f5a0f0badeeececa6db5cb
+  data.tar.gz: aed5f37c4031ffbc1a635ddd4fc979f4e40ab68a94ef880730bc48a2df600678
 SHA512:
-  metadata.gz: 6ea5989d78febe490473a21c2c1958e96ea2f920eee5eee90c4521ca774c5fbd35b10efb41270adb1f5344189461afc010a2b38aec2aeae86d901e5dd7aea9d1
-  data.tar.gz: 3c18daec3b396ac0ec96cdb22cf444c8a12f1031189b68c90961bd24ccc8143e0682294b067ca21dd59115712f7d811aaf0352c6d27bf319dcce6e988b3362a9
+  metadata.gz: a81ff55bf880a3581ad52f6bf2d8652e1d7824119ed31daed81eb0e8f215d4d26a19a10d646b5497346e6795a320de18cd63fa817958081c2ba4393efad4c20a
+  data.tar.gz: b3fa4c9979fc22863d0e171d87918c320ec177b5e27c0820f4997919cb714d8c19516d37e15eccaed8b0a81102498820b4865864dcb56c650c186ef1bd057b56

data/.github/workflows/test.yml

@@ -0,0 +1,44 @@
+name: Ruby CI
+
+on: [pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    services:
+      db:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, '3.0']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby 2.7
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Build and test with Rake
+      env:
+        DATABASE: postgresql://postgres:postgres@localhost:5432/test
+      run: |
+        sudo apt-get -yqq install libpq-dev
+        gem install bundler
+        bundle install
+        bundle exec rake

data/README.md

@@ -1,49 +1,59 @@
 # mihari
 
 [![Gem Version](https://badge.fury.io/rb/mihari.svg)](https://badge.fury.io/rb/mihari)
-[![Build Status](https://travis-ci.com/ninoseki/mihari.svg?branch=master)](https://travis-ci.com/ninoseki/mihari)
+[![Ruby CI](https://github.com/ninoseki/mihari/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/test.yml)
 [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/mihari)](https://hub.docker.com/r/ninoseki/mihari)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 
+![img](https://github.com/ninoseki/mihari/raw/master/images/logo.png)
+
 Mihari is a helper to run queries & manage results continuously. Mihari can be used for C2, landing page and phishing hunting.
 
 ## How it works
 
-- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes) from the results.
+- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes).
 - Mihari checks whether a DB (SQLite3 or PostgreSQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
-    - Mihari creates an alert on TheHive. (Optional)
-    - Mihari sends a notification to Slack. (Optional)
-    - Mihari creates an event on MISP. (Optional)
+    - Mihari creates an alert on TheHive.
+    - Mihari sends a notification to Slack.
+    - Mihari creates an event on MISP.
 
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/eyecatch.png)
+![img](https://github.com/ninoseki/mihari/raw/master/images/eyecatch.png)
 
 ### Screenshots
 
 - TheHive alert example
 
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/alert.png)
+![img](https://github.com/ninoseki/mihari/raw/master/images/alert.png)
 
 - Slack notification example
 
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/slack.png)
+![img](https://github.com/ninoseki/mihari/raw/master/images/slack.png)
 
 - MISP event example
 
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/misp.png)
+![img](https://github.com/ninoseki/mihari/raw/master/images/misp.png)
 
 ## Requirements
 
-- Ruby 2.6+
-- SQLite3
-- libpq
+- Ruby (2.7 or 3.0)
+- SQLite3 or PostgreSQL
 
 ```bash
 # For Debian / Ubuntu
 apt-get install sqlite3 libsqlite3-dev libpq-dev
 ```
 
+## Supported platforms & databases
+
+| Name       | Supported versions |
+|------------|--------------------|
+| PostgreSQL | v12                |
+| SQLite     | v3                 |
+| MISP       | v2.4               |
+| TheHive    | v3.x & v4.x        |
+
 ## Installation
 
 ```bash
@@ -69,6 +79,7 @@ Mihari supports the following services by default.
 - [Onyphe](https://onyphe.io)
 - [OTX](https://otx.alienvault.com/)
 - [PassiveTotal](https://community.riskiq.com/)
+- [Pulsedive](https://pulsedive.com/)
 - [SecurityTrails](https://securitytrails.com/)
 - [Shodan](https://shodan.io)
 - [Spyse](https://spyse.com)

data/Rakefile

@@ -2,6 +2,7 @@
 
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "standard/rake"
 
 RSpec::Core::RakeTask.new(:spec)
 

data/docker/Dockerfile

@@ -1,4 +1,5 @@
-FROM ruby:2.7-alpine3.10
+FROM ruby:3.0.0-alpine3.13
+
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/mihari.git \
@@ -10,4 +11,4 @@ RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
 
 ENTRYPOINT ["mihari"]
 
-CMD ["--help"]
+CMD ["--help"]

data/lib/mihari/alert_viewer.rb

@@ -9,13 +9,13 @@ module Mihari
       relation = Alert.includes(:tags, :artifacts)
       relation = relation.where(title: title) if title
       relation = relation.where(source: source) if source
-      relation = relation.where(tags: { name: tag } ) if tag
+      relation = relation.where(tags: {name: tag}) if tag
 
       alerts = relation.limit(limit).order(id: :desc)
       alerts.map do |alert|
         json = AlertSerializer.new(alert).as_json
-        json[:artifacts] = (json.dig(:artifacts) || []).map { |artifact_| artifact_.dig(:data) }
-        json[:tags] = (json.dig(:tags) || []).map { |tag_| tag_.dig(:name) }
+        json[:artifacts] = (json[:artifacts] || []).map { |artifact_| artifact_[:data] }
+        json[:tags] = (json[:tags] || []).map { |tag_| tag_[:name] }
         json
       end
     end

data/lib/mihari/analyzers/base.rb

@@ -42,7 +42,7 @@ module Mihari
 
       def run_emitter(emitter)
         emitter.run(title: title, description: description, artifacts: unique_artifacts, source: source, tags: tags)
-      rescue StandardError => e
+      rescue => e
         puts "Emission by #{emitter.class} is failed: #{e}"
       end
 

data/lib/mihari/analyzers/basic.rb

@@ -4,12 +4,11 @@ module Mihari
   module Analyzers
     class Basic < Base
       attr_accessor :title
-      attr_reader :description
-      attr_reader :artifacts
-      attr_reader :source
-      attr_reader :tags
+      attr_reader :description, :artifacts, :source, :tags
 
       def initialize(title:, description:, artifacts:, source:, tags: [])
+        super()
+
         @title = title
         @description = description
         @artifacts = artifacts

data/lib/mihari/analyzers/binaryedge.rb

@@ -5,10 +5,7 @@ require "binaryedge"
 module Mihari
   module Analyzers
     class BinaryEdge < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
+      attr_reader :title, :description, :query, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -24,7 +21,7 @@ module Mihari
         return [] unless results || results.empty?
 
         results.map do |result|
-          events = result.dig("events") || []
+          events = result["events"] || []
           events.map do |event|
             event.dig "target", "ip"
           end.compact
@@ -47,7 +44,7 @@ module Mihari
         responses = []
         (1..Float::INFINITY).each do |page|
           res = search_with_page(query, page: page)
-          total = res.dig("total").to_i
+          total = res["total"].to_i
 
           responses << res
           break if total <= page * PAGE_SIZE
@@ -56,7 +53,7 @@ module Mihari
       end
 
       def config_keys
-        %w(binaryedge_api_key)
+        %w[binaryedge_api_key]
       end
 
       def api

data/lib/mihari/analyzers/censys.rb

@@ -5,11 +5,7 @@ require "censu"
 module Mihari
   module Analyzers
     class Censys < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
-      attr_reader :type
+      attr_reader :title, :description, :query, :tags, :type
 
       def initialize(query, title: nil, description: nil, tags: [], type: "ipv4")
         super()
@@ -37,7 +33,7 @@ module Mihari
       private
 
       def valid_type?
-        %w(ipv4 websites certificates).include? type
+        %w[ipv4 websites certificates].include? type
       end
 
       def normalize(domain)
@@ -86,7 +82,7 @@ module Mihari
       end
 
       def config_keys
-        %w(censys_id censys_secret)
+        %w[censys_id censys_secret]
       end
 
       def api

data/lib/mihari/analyzers/circl.rb

@@ -5,9 +5,7 @@ require "passive_circl"
 module Mihari
   module Analyzers
     class CIRCL < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -27,7 +25,7 @@ module Mihari
       private
 
       def config_keys
-        %w(circl_passive_password circl_passive_username)
+        %w[circl_passive_password circl_passive_username]
       end
 
       def api
@@ -41,7 +39,7 @@ module Mihari
         when "hash"
           passive_ssl_lookup
         else
-          raise InvalidInputError, "#{@query}(type: #{@type || 'unknown'}) is not supported."
+          raise InvalidInputError, "#{@query}(type: #{@type || "unknown"}) is not supported."
         end
       end
 

data/lib/mihari/analyzers/crtsh.rb

@@ -5,11 +5,7 @@ require "crtsh"
 module Mihari
   module Analyzers
     class Crtsh < Base
-      attr_reader :title
-      attr_reader :description
-      attr_reader :query
-      attr_reader :tags
-      attr_reader :exclude_expired
+      attr_reader :title, :description, :query, :tags, :exclude_expired
 
       def initialize(query, title: nil, description: nil, tags: [], exclude_expired: nil)
         super()
@@ -24,7 +20,7 @@ module Mihari
 
       def artifacts
         results = search
-        name_values = results.map { |result| result.dig("name_value") }.compact
+        name_values = results.map { |result| result["name_value"] }.compact
         name_values.map(&:lines).flatten.uniq.map(&:chomp)
       end
 

data/lib/mihari/analyzers/dnpedia.rb

@@ -5,10 +5,7 @@ require "dnpedia"
 module Mihari
   module Analyzers
     class DNPedia < Base
-      attr_reader :query
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -31,9 +28,9 @@ module Mihari
 
       def lookup
         res = api.search(query)
-        rows = res.dig("rows") || []
+        rows = res["rows"] || []
         rows.map do |row|
-          [row.dig("name"), row.dig("zoneid")].join(".")
+          [row["name"], row["zoneid"]].join(".")
         end
       end
     end

data/lib/mihari/analyzers/dnstwister.rb

@@ -7,12 +7,7 @@ require "parallel"
 module Mihari
   module Analyzers
     class DNSTwister < Base
-      attr_reader :query
-      attr_reader :type
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :type, :title, :description, :tags
 
       def initialize(query, title: nil, description: nil, tags: [])
         super()
@@ -47,11 +42,11 @@ module Mihari
       end
 
       def lookup
-        raise InvalidInputError, "#{query}(type: #{type || 'unknown'}) is not supported." unless valid_type?
+        raise InvalidInputError, "#{query}(type: #{type || "unknown"}) is not supported." unless valid_type?
 
         res = api.fuzz(query)
-        fuzzy_domains = res.dig("fuzzy_domains") || []
-        domains = fuzzy_domains.map { |domain| domain.dig("domain") }
+        fuzzy_domains = res["fuzzy_domains"] || []
+        domains = fuzzy_domains.map { |domain| domain["domain"] }
         Parallel.map(domains) do |domain|
           resolvable?(domain) ? domain : nil
         end.compact

data/lib/mihari/analyzers/free_text.rb

@@ -5,15 +5,11 @@ require "parallel"
 module Mihari
   module Analyzers
     class FreeText < Base
-      attr_reader :query
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :query, :title, :description, :tags
 
       ANALYZERS = [
         Mihari::Analyzers::BinaryEdge,
-        Mihari::Analyzers::Censys,
+        Mihari::Analyzers::Censys
       ].freeze
 
       def initialize(query, title: nil, description: nil, tags: [])

data/lib/mihari/analyzers/http_hash.rb

@@ -5,15 +5,7 @@ require "parallel"
 module Mihari
   module Analyzers
     class HTTPHash < Base
-      attr_reader :md5
-      attr_reader :sha256
-      attr_reader :mmh3
-
-      attr_reader :html
-
-      attr_reader :title
-      attr_reader :description
-      attr_reader :tags
+      attr_reader :md5, :sha256, :mmh3, :html, :title, :description, :tags
 
       def initialize(_query, md5: nil, sha256: nil, mmh3: nil, html: nil, title: nil, description: nil, tags: [])
         super()
@@ -57,7 +49,7 @@ module Mihari
         [
           md5 ? "md5:#{md5}" : nil,
           sha256 ? "sha256:#{sha256}" : nil,
-          mmh3 ? "mmh3:#{mmh3}" : nil,
+          mmh3 ? "mmh3:#{mmh3}" : nil
         ].compact.join(",")
       end
 
@@ -92,7 +84,7 @@ module Mihari
           binary_edge,
           censys,
           onyphe,
-          shodan,
+          shodan
         ].compact
       end