mihari 1.5.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 951201ccebc7b6c4c117a687c1abce9ab24fa8d450f5a0f0badeeececa6db5cb
-  data.tar.gz: aed5f37c4031ffbc1a635ddd4fc979f4e40ab68a94ef880730bc48a2df600678
+  metadata.gz: 3cf190c27b1be3c0eeb690354361c3bb3295beb8609661a861fcc31e227cb7f0
+  data.tar.gz: e989209e3668e9342803ed1b91c9e668bcf0519e659f9fb88687cc38460818cb
 SHA512:
-  metadata.gz: a81ff55bf880a3581ad52f6bf2d8652e1d7824119ed31daed81eb0e8f215d4d26a19a10d646b5497346e6795a320de18cd63fa817958081c2ba4393efad4c20a
-  data.tar.gz: b3fa4c9979fc22863d0e171d87918c320ec177b5e27c0820f4997919cb714d8c19516d37e15eccaed8b0a81102498820b4865864dcb56c650c186ef1bd057b56
+  metadata.gz: 0a40de96264eaf211240b0020aa3494e6a64bddf8e2aa74461e91122fff29d72a5ab66032e226fa80c19c984fcb7a3e37d1e78aa0da317f6fd1445945b93d200
+  data.tar.gz: b0b55e75e84037cf7761fd28128504d45fc049de01d0f4db65184b2e8ff0e9bc3948e06b8165e90c19244fb1c8b8af33efa77d260bb323d4084d85e21f72245f

data/.github/workflows/test.yml

@@ -4,11 +4,10 @@ on: [pull_request]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     services:
-      db:
+      postgres:
         image: postgres:12
         env:
           POSTGRES_USER: postgres
@@ -22,23 +21,48 @@ jobs:
         ports:
           - 5432:5432
 
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_USER: mysql
+          MYSQL_PASSWORD: mysql
+          MYSQL_DATABASE: test
+          MYSQL_ROOT_PASSWORD: rootpassword
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.7, '3.0']
+        ruby: [2.7, "3.0"]
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby 2.7
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby }}
-        bundler-cache: true
-    - name: Build and test with Rake
-      env:
-        DATABASE: postgresql://postgres:postgres@localhost:5432/test
-      run: |
-        sudo apt-get -yqq install libpq-dev
-        gem install bundler
-        bundle install
-        bundle exec rake
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get -yqq install libpq-dev libmysqlclient-dev
+          gem install bundler
+          bundle install
+
+      - name: Test with PostgreSQL
+        env:
+          DATABASE: postgresql://postgres:postgres@localhost:5432/test
+        run: |
+          bundle exec rake
+
+      - name: Test with MySQL
+        env:
+          DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
+        run: |
+          bundle exec rake

data/README.md

@@ -8,65 +8,22 @@
 
 ![img](https://github.com/ninoseki/mihari/raw/master/images/logo.png)
 
-Mihari is a helper to run queries & manage results continuously. Mihari can be used for C2, landing page and phishing hunting.
+Mihari is a framework for continuous OSINT based threat hunting.
 
 ## How it works
 
-- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes).
-- Mihari checks whether a DB (SQLite3 or PostgreSQL) contains the artifacts or not.
+- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs or hashes).
+- Mihari checks whether a DB (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
     - Mihari creates an alert on TheHive.
     - Mihari sends a notification to Slack.
     - Mihari creates an event on MISP.
 
-![img](https://github.com/ninoseki/mihari/raw/master/images/eyecatch.png)
+Also, you can check the alerts on a built-in web app.
 
-### Screenshots
+![img](https://github.com/ninoseki/mihari/raw/master/images/web_alerts.png)
 
-- TheHive alert example
-
-![img](https://github.com/ninoseki/mihari/raw/master/images/alert.png)
-
-- Slack notification example
-
-![img](https://github.com/ninoseki/mihari/raw/master/images/slack.png)
-
-- MISP event example
-
-![img](https://github.com/ninoseki/mihari/raw/master/images/misp.png)
-
-## Requirements
-
-- Ruby (2.7 or 3.0)
-- SQLite3 or PostgreSQL
-
-```bash
-# For Debian / Ubuntu
-apt-get install sqlite3 libsqlite3-dev libpq-dev
-```
-
-## Supported platforms & databases
-
-| Name       | Supported versions |
-|------------|--------------------|
-| PostgreSQL | v12                |
-| SQLite     | v3                 |
-| MISP       | v2.4               |
-| TheHive    | v3.x & v4.x        |
-
-## Installation
-
-```bash
-gem install mihari
-```
-
-Or you can use this tool with Docker.
-
-```bash
-docker pull ninoseki/mihari
-```
-
-## Basic usage
+## Supported services
 
 Mihari supports the following services by default.
 
@@ -87,233 +44,15 @@ Mihari supports the following services by default.
 - [VirusTotal](http://virustotal.com)
 - [ZoomEye](https://zoomeye.org)
 
-```bash
-$ mihari
-Commands:
-  mihari alerts                               # Show the alerts on TheHive
-  mihari binaryedge [QUERY]                   # BinaryEdge host search by a query
-  mihari censys [QUERY]                       # Censys IPv4 search by a query
-  mihari circl [DOMAIN|SHA1]                  # CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint
-  mihari crtsh [QUERY]                        # crt.sh search by a query
-  mihari dnpedia [QUERY]                      # DNPedia domain search by a query
-  mihari dnstwister [DOMAIN]                  # dnstwister lookup by a domain
-  mihari free_text [TEXT]                     # Cross search with search engines by a free text
-  mihari help [COMMAND]                       # Describe available commands or one specific command
-  mihari http_hash                            # Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
-  mihari import_from_json                     # Give a JSON input via STDIN
-  mihari onyphe [QUERY]                       # Onyphe datascan search by a query
-  mihari otx [IP|DOMAIN]                      # OTX lookup by an IP or domain
-  mihari passive_dns [IP|DOMAIN]              # Cross search with passive DNS services by an ip or domain
-  mihari passive_ssl [SHA1]                   # Cross search with passive SSL services by an SHA1 certificate fingerprint
-  mihari passivetotal [IP|DOMAIN|EMAIL|SHA1]  # PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint
-  mihari pulsedive [IP|DOMAIN]                # Pulsedive lookup by an ip or domain
-  mihari reverse_whois [EMAIL]                # Cross search with reverse whois services by an email
-  mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by an ip, domain or email
-  mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed search by a regexp
-  mihari shodan [QUERY]                       # Shodan host search by a query
-  mihari spyse [QUERY]                        # Spyse search by a query
-  mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)
-  mihari status                               # Show the current configuration status
-  mihari urlscan [QUERY]                      # urlscan search by a given query
-  mihari virustotal [IP|DOMAIN]               # VirusTotal resolutions lookup by an ip or domain
-  mihari zoomeye [QUERY]                      # ZoomEye search by a query
-
-Options:
-  [--config=CONFIG]  # path to config file
-
-```
-
-### Cross searches
-
-Mihari has cross search features. A cross search is a search across a number of services.
-
-You can get aggregated results by using the following commands.
-
-| Command         | Desc.                                                                                                   |
-|-----------------|---------------------------------------------------------------------------------------------------------|
-| passive_dns     | Passive DNS lookup with CIRCL passive DNS, OTX, PassiveTotal, Pulsedive, SecurityTrails and VirusTotal  |
-| passive_ssl     | Passive SSL lookup with CIRCL passive SSL and PassiveTotal                                              |
-| reverse_whois   | Revese Whois lookup with PassiveTotal and SecurityTrails                                                |
-| http_hash       | HTTP response hash lookup with BinaryEdge(SHA256), Censys(SHA256), Onyphpe(MD5) and Shodan(MurmurHash3) |
-| free_text       | Free text lookup with BinaryEdge and Censys                                                             |
-| ssh_fingerprint | SSH fingerprint lookup with BinaryEdge and Shodan                                                       |
-
-#### http_hash command
-
-The usage of `http_hash` command is a little bit tricky.
-
-```bash
-$ mihari help http_hash
-Usage:
-  mihari http_hash
-
-Options:
-  [--title=TITLE]              # title
-  [--description=DESCRIPTION]  # description
-  [--tags=one two three]       # tags
-  [--md5=MD5]                  # MD5 hash
-  [--sha256=SHA256]            # SHA256 hash
-  [--mmh3=N]                   # MurmurHash3 hash
-
-Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
-
-```
-
-There are 2 ways to use this command.
-
-First one is passing `--md5`, `--sha256` and `--mmh3` parameters.
-
-```bash
-mihari http_hash --md5=881191f7736b5b8cfad5959ca99d2a51 --sha256=b064187ebdc51721708ad98cd89dacc346017cb0fb0457d530032d387f1ff20e --mmh3=-1467534799
-```
-
-Another one is passing `--html` parameter. In this case, hashes of an HTML file are automatically calculated.
-
-```bash
-wget http://example.com -O /tmp/index.html
-mihari http_hash --html /tmp/index.html
-```
-
-### Example usages
-
-```bash
-# Censys lookup for PANDA C2
-mihari censys '("PANDA" AND "SMAdmin" AND "layui")' --title "PANDA C2"
-
-# VirusTotal passive DNS lookup of a FAKESPY host
-mihari virustotal "jppost-hi.top" --title "FAKESPY passive DNS"
-
-# You can pass a "defanged" indicator as an input
-mihari virustotal "jppost-hi[.]top" --title "FAKESPY passive DNS"
-```
-
-### Import from JSON
-
-```bash
-echo '{ "title": "test", "description": "test", "artifacts": ["1.1.1.1", "github.com", "2.2.2.2"] }' | mihari import_from_json
-```
-
-The input is a JSON data should have `title`, `description` and `artifacts` key. `tags` key is an optional parameter.
-
-```json
-{
-  "title": "test",
-  "description": "test",
-  "artifacts": ["1.1.1.1", "github.com"],
-  "tags": ["test"]
-}
-```
-
-| Key         | Desc.                                                                      | Required or optional |
-|-------------|----------------------------------------------------------------------------|----------------------|
-| title       | A title of an alert                                                        | Required             |
-| description | A description of an alert                                                  | Required             |
-| artifacts   | An array of artifacts (supported data types: ip, domain, url, email, hash) | Required             |
-| tags        | An array of tags                                                           | Optional             |
-
-## Configuration
-
-Configuration can be done via environment variables or a YAML file.
-
-| Key                    | Description                                                                                     | Default     |
-|------------------------|-------------------------------------------------------------------------------------------------|-------------|
-| DATABASE               | A path to the SQLite database or a DB URL (e.g. `postgres://postgres:pass@db.host:5432/somedb`) | `mihari.db` |
-| BINARYEDGE_API_KEY     | BinaryEdge API key                                                                              |             |
-| CENSYS_ID              | Censys API ID                                                                                   |             |
-| CENSYS_SECRET          | Censys secret                                                                                   |             |
-| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password                                                                  |             |
-| CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username                                                                  |             |
-| MISP_API_ENDPOINT      | MISP URL                                                                                        |             |
-| MISP_API_KEY           | MISP API key                                                                                    |             |
-| ONYPHE_API_KEY         | Onyphe API key                                                                                  |             |
-| OTX_API_KEY            | OTX API key                                                                                     |             |
-| PASSIVETOTAL_API_KEY   | PassiveTotal API key                                                                            |             |
-| PASSIVETOTAL_USERNAME  | PassiveTotal username                                                                           |             |
-| PULSEDIVE_API_KEY      | Pulsedive API key                                                                               |             |
-| SECURITYTRAILS_API_KEY | SecurityTrails API key                                                                          |             |
-| SHODAN_API_KEY         | Shodan API key                                                                                  |             |
-| SLACK_CHANNEL          | Slack channel name                                                                              | `#general`  |
-| SLACK_WEBHOOK_URL      | Slack Webhook URL                                                                               |             |
-| SPYSE_API_KEY          | Spyse API key                                                                                   |             |
-| THEHIVE_API_ENDPOINT   | TheHive URL                                                                                     |             |
-| THEHIVE_API_KEY        | TheHive API key                                                                                 |             |
-| URLSCAN_API_KEY        | urlscan.io API key                                                                              |             |
-| VIRUSTOTAL_API_KEY     | VirusTotal API key                                                                              |             |
-| ZOOMEYE_PASSWORD       | ZoomEye password                                                                                |             |
-| ZOOMEYE_USERNAMME      | ZoomEye username                                                                                |             |
-
-Instead of using environment variables, you can use a YAML file for configuration.
-
-```bash
-mihari virustotal 1.1.1.1 --config /path/to/yaml.yml
-```
-
-The YAML file should be a YAML hash like below:
-
-```yaml
-database: /tmp/mihari.db
-thehive_api_endpoint: https://localhost
-thehive_api_key: foo
-virustotal_api_key: foo
-```
-
-You can check the configuration status via `status` command.
-
-```bash
-mihari status
-```
-
-## How to create a custom script
-
-Create a class which extends `Mihari::Analyzers::Base` and implements the following methods.
-
-| Name           | Desc.                                                                      | @return       | Required or optional |
-|----------------|----------------------------------------------------------------------------|---------------|----------------------|
-| `#title`       | A title of an alert                                                        | String        | Required             |
-| `#description` | A description of an alert                                                  | String        | Required             |
-| `#artifacts`   | An array of artifacts (supported data types: ip, domain, url, email, hash) | Array<String> | Required             |
-| `#tags`        | An array of tags                                                           | Array<String> | Optional             |
-
-```ruby
-require "mihari"
-
-module Mihari
-  module Analyzers
-    class Example < Base
-      def title
-        "example"
-      end
-
-      def description
-        "example"
-      end
-
-      def artifacts
-        ["9.9.9.9", "example.com"]
-      end
-
-      def tags
-        ["example"]
-      end
-    end
-  end
-end
-
-example = Mihari::Analyzers::Example.new
-example.run
-```
-
-See `/examples` for more.
+See [Usage](https://github.com/ninoseki/mihari/wiki/Usage) for more information.
 
-## Using it with Docker
+## Docs
 
-```bash
-$ docker run --rm ninoseki/mihari
-# Note that you should pass configurations via environment variables
-$ docker run --rm ninoseki/mihari -e THEHIVE_API_ENDPOINT="http://THEHIVE_URL" -e THEHIVE_API_KEY="API KEY" mihari
-# or
-$ docker run --rm ninoseki/mihari --env-file ~/.mihari.env mihari
-```
+- [Requirements & Installation](https://github.com/ninoseki/mihari/wiki/Requirements-&-Installation)
+- [Usage](https://github.com/ninoseki/mihari/wiki/Usage)
+- [Configuration](https://github.com/ninoseki/mihari/wiki/Configuration)
+- [Custom script](https://github.com/ninoseki/mihari/wiki/Custom-script)
+- [Docker](https://github.com/ninoseki/mihari/wiki/Docker)
 
 ## License
 

data/build_frontend.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+CURRENT_DIR=${PWD}
+
+mkdir -p tmp
+cd tmp
+git clone https://github.com/ninoseki/mihari-frontend.git
+cd mihari-frontend
+npm install
+npm run build
+
+cp -r dist/* ${CURRENT_DIR}/lib/mihari/web/public
+
+rm -rf ${CURRENT_DIR}/tmp/mihari-frontend

data/lib/mihari.rb

@@ -79,8 +79,8 @@ require "mihari/emitters/slack"
 require "mihari/emitters/stdout"
 require "mihari/emitters/the_hive"
 
-require "mihari/alert_viewer"
-
 require "mihari/status"
 
+require "mihari/web/app"
+
 require "mihari/cli"

data/lib/mihari/cli.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-require "thor"
 require "json"
+require "rack/builder"
+require "rack/handler/webrick"
+require "thor"
 
 module Mihari
   class CLI < Thor
@@ -275,34 +277,21 @@ module Mihari
       end
     end
 
-    desc "alerts", "Show the alerts on TheHive"
-    method_option :limit, type: :string, default: "5", desc: "Number of alerts to show (or 'all' to show all the alerts)"
-    method_option :title, type: :string, desc: "Title to filter"
-    method_option :source, type: :string, desc: "Source to filter"
-    method_option :tag, type: :string, desc: "Tag to filter"
-    def alerts
-      with_error_handling do
-        load_configuration
+    desc "web", "Launch the web app"
+    method_option :port, type: :numeric, default: 9292
+    method_option :host, type: :string, default: "localhost"
+    def web
+      port = options["port"].to_i || 9292
+      host = options["host"] || "localhost"
 
-        viewer = AlertViewer.new
-        alerts = viewer.list(limit: options["limit"], title: options["title"], source: options["source"], tag: options[:tag])
-        puts JSON.pretty_generate(alerts)
-      end
-    end
-
-    desc "status", "Show the current configuration status"
-    def status
-      with_error_handling do
-        load_configuration
-
-        puts JSON.pretty_generate(Status.check)
-      end
+      load_configuration
+      Mihari::App.run!(port: port, host: host)
     end
 
     no_commands do
       def with_error_handling
         yield
-      rescue => e
+      rescue StandardError => e
         notifier = Notifiers::ExceptionNotifier.new
         notifier.notify e
       end