mihari 1.5.0 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70afeb6b1ddaa263689beb836de85264ad1e871a1887a0574adfc22f00e006d8
-  data.tar.gz: ad614363a9a3320c2dfa34ec19bd4712d08a9a6e662cdf11b4b70c775f55e592
+  metadata.gz: 14bf63fb577af68b341abac71adfcc2dc1bab435d2be6ed78d33cb59bcf54c9b
+  data.tar.gz: 772be3a526f71631ed0ee861fc9c307bff4a686dae6ab370510870f361657bc3
 SHA512:
-  metadata.gz: 8b7b9b86a6ec5341ce03b5652e29d45ccfafcd6ee09ca469ccf88d9872a965aded13fae1925c647cbbdedb389532469060698e35a06905197ffb143b223d0a93
-  data.tar.gz: 5d839d16358cf855658bc85d8e30868ce9fcae63cf9064461e67343cd979cee90daad2822260d96da9f00acadb9f981e29de89eb07b35047f9cd3ae039a37c3e
+  metadata.gz: 0b57106d1727f2d4d780bb78bd4e40f23ad89d67893741d68bd0ed6d5ef9d03e9f1aad514338b2b400d7c5c3ef2cd75fdc6252b5049263e0c8bb41ff7a929c1f
+  data.tar.gz: b44040e98a4544bfab59ed1b61f648014c4a470a6233248fef72d61c9839d92b004b8a7152a8c68c1c790d4b40acb5fe1085b41ef1c1650750387fb7281f1850

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,43 @@
+---
+name: Bug report
+about: Create a bug report to help us improve
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+<!--
+Thank you for taking the time to report a bug.
+Please make sure there is no existing issue about this kind of bug.
+-->
+
+### **Describe the bug**
+
+A clear and concise description of what the bug is.
+
+### **Steps to reproduce**
+
+- ...
+
+### **Expected behavior**
+
+A clear and concise description of what you expected to happen.
+
+### **Actual behavior**
+
+A clear and concise description of what actually happened.
+
+### **Screenshots**
+
+Add screenshots to help explain your problem.
+
+### **System Information:**
+
+- OS: [e.g. Windows10]
+- Ruby version: [e.g. 3.0]
+- Mihari version: [e.g. 2.0.0]
+
+### **Additional context**
+
+Add any other context about the problem here.

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,15 @@
+---
+name: Feature request
+about: Suggest a new Feature for Mihari
+title: "[Feature Request]"
+labels: enhancement
+assignees: ''
+
+---
+<!--
+
+1. Make sure your requested feature makes sense for Mihari.
+
+2. If you want to suggest a new integration of a service, please provide detailed information of it. (e.g. API docs)
+
+-->

data/.github/workflows/test.yml

@@ -4,11 +4,10 @@ on: [pull_request]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     services:
-      db:
+      postgres:
         image: postgres:12
         env:
           POSTGRES_USER: postgres
@@ -22,23 +21,48 @@ jobs:
         ports:
           - 5432:5432
 
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_USER: mysql
+          MYSQL_PASSWORD: mysql
+          MYSQL_DATABASE: test
+          MYSQL_ROOT_PASSWORD: rootpassword
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.7, '3.0']
+        ruby: [2.7, "3.0"]
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby 2.7
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby }}
-        bundler-cache: true
-    - name: Build and test with Rake
-      env:
-        DATABASE: postgresql://postgres:postgres@localhost:5432/test
-      run: |
-        sudo apt-get -yqq install libpq-dev
-        gem install bundler
-        bundle install
-        bundle exec rake
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get -yqq install libpq-dev libmysqlclient-dev
+          gem install bundler
+          bundle install
+
+      - name: Test with PostgreSQL
+        env:
+          DATABASE: postgresql://postgres:postgres@localhost:5432/test
+        run: |
+          bundle exec rake
+
+      - name: Test with MySQL
+        env:
+          DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
+        run: |
+          bundle exec rake

data/.rubocop.yml

@@ -4,6 +4,9 @@
 require:
   - rubocop-performance
 
+AllCops:
+  NewCops: enable
+
 Style/Alias:
   Enabled: false
   StyleGuide: https://relaxed.ruby.style/#stylealias
@@ -151,5 +154,8 @@ Lint/AssignmentInCondition:
 Layout/LineLength:
   Enabled: false
 
+Style/StringLiteralsInInterpolation:
+  Enabled: false
+
 Metrics:
   Enabled: false

data/.standard.yml

@@ -0,0 +1,4 @@
+ignore:
+  - "**/*":
+      - Layout/SpaceInsideHashLiteralBraces
+      - Style/RescueStandardError

data/README.md

@@ -1,61 +1,31 @@
 # mihari
 
 [![Gem Version](https://badge.fury.io/rb/mihari.svg)](https://badge.fury.io/rb/mihari)
-[![Build Status](https://travis-ci.com/ninoseki/mihari.svg?branch=master)](https://travis-ci.com/ninoseki/mihari)
+[![Ruby CI](https://github.com/ninoseki/mihari/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/test.yml)
 [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/mihari)](https://hub.docker.com/r/ninoseki/mihari)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 
-Mihari is a helper to run queries & manage results continuously. Mihari can be used for C2, landing page and phishing hunting.
+![img](https://github.com/ninoseki/mihari/raw/master/images/logo.png)
+
+Mihari is a framework for continuous OSINT based threat hunting.
 
 ## How it works
 
-- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes).
-- Mihari checks whether a DB (SQLite3 or PostgreSQL) contains the artifacts or not.
+![img](https://github.com/ninoseki/mihari/raw/master/images/overview.png)
+
+- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs or hashes).
+- Mihari checks whether a DB (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
     - Mihari creates an alert on TheHive.
     - Mihari sends a notification to Slack.
     - Mihari creates an event on MISP.
 
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/eyecatch.png)
-
-### Screenshots
-
-- TheHive alert example
-
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/alert.png)
-
-- Slack notification example
-
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/slack.png)
-
-- MISP event example
-
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/misp.png)
-
-## Requirements
-
-- Ruby (2.7 or 3.0)
-- SQLite3 or PostgreSQL
-
-```bash
-# For Debian / Ubuntu
-apt-get install sqlite3 libsqlite3-dev libpq-dev
-```
-
-## Installation
-
-```bash
-gem install mihari
-```
-
-Or you can use this tool with Docker.
+Also, you can check the alerts on a built-in web app.
 
-```bash
-docker pull ninoseki/mihari
-```
+![img](https://github.com/ninoseki/mihari/raw/master/images/web_alerts.png)
 
-## Basic usage
+## Supported services
 
 Mihari supports the following services by default.
 
@@ -68,6 +38,7 @@ Mihari supports the following services by default.
 - [Onyphe](https://onyphe.io)
 - [OTX](https://otx.alienvault.com/)
 - [PassiveTotal](https://community.riskiq.com/)
+- [Pulsedive](https://pulsedive.com/)
 - [SecurityTrails](https://securitytrails.com/)
 - [Shodan](https://shodan.io)
 - [Spyse](https://spyse.com)
@@ -75,233 +46,17 @@ Mihari supports the following services by default.
 - [VirusTotal](http://virustotal.com)
 - [ZoomEye](https://zoomeye.org)
 
-```bash
-$ mihari
-Commands:
-  mihari alerts                               # Show the alerts on TheHive
-  mihari binaryedge [QUERY]                   # BinaryEdge host search by a query
-  mihari censys [QUERY]                       # Censys IPv4 search by a query
-  mihari circl [DOMAIN|SHA1]                  # CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint
-  mihari crtsh [QUERY]                        # crt.sh search by a query
-  mihari dnpedia [QUERY]                      # DNPedia domain search by a query
-  mihari dnstwister [DOMAIN]                  # dnstwister lookup by a domain
-  mihari free_text [TEXT]                     # Cross search with search engines by a free text
-  mihari help [COMMAND]                       # Describe available commands or one specific command
-  mihari http_hash                            # Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
-  mihari import_from_json                     # Give a JSON input via STDIN
-  mihari onyphe [QUERY]                       # Onyphe datascan search by a query
-  mihari otx [IP|DOMAIN]                      # OTX lookup by an IP or domain
-  mihari passive_dns [IP|DOMAIN]              # Cross search with passive DNS services by an ip or domain
-  mihari passive_ssl [SHA1]                   # Cross search with passive SSL services by an SHA1 certificate fingerprint
-  mihari passivetotal [IP|DOMAIN|EMAIL|SHA1]  # PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint
-  mihari pulsedive [IP|DOMAIN]                # Pulsedive lookup by an ip or domain
-  mihari reverse_whois [EMAIL]                # Cross search with reverse whois services by an email
-  mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by an ip, domain or email
-  mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed search by a regexp
-  mihari shodan [QUERY]                       # Shodan host search by a query
-  mihari spyse [QUERY]                        # Spyse search by a query
-  mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)
-  mihari status                               # Show the current configuration status
-  mihari urlscan [QUERY]                      # urlscan search by a given query
-  mihari virustotal [IP|DOMAIN]               # VirusTotal resolutions lookup by an ip or domain
-  mihari zoomeye [QUERY]                      # ZoomEye search by a query
-
-Options:
-  [--config=CONFIG]  # path to config file
-
-```
-
-### Cross searches
-
-Mihari has cross search features. A cross search is a search across a number of services.
-
-You can get aggregated results by using the following commands.
-
-| Command         | Desc.                                                                                                   |
-|-----------------|---------------------------------------------------------------------------------------------------------|
-| passive_dns     | Passive DNS lookup with CIRCL passive DNS, OTX, PassiveTotal, Pulsedive, SecurityTrails and VirusTotal  |
-| passive_ssl     | Passive SSL lookup with CIRCL passive SSL and PassiveTotal                                              |
-| reverse_whois   | Revese Whois lookup with PassiveTotal and SecurityTrails                                                |
-| http_hash       | HTTP response hash lookup with BinaryEdge(SHA256), Censys(SHA256), Onyphpe(MD5) and Shodan(MurmurHash3) |
-| free_text       | Free text lookup with BinaryEdge and Censys                                                             |
-| ssh_fingerprint | SSH fingerprint lookup with BinaryEdge and Shodan                                                       |
-
-#### http_hash command
-
-The usage of `http_hash` command is a little bit tricky.
-
-```bash
-$ mihari help http_hash
-Usage:
-  mihari http_hash
-
-Options:
-  [--title=TITLE]              # title
-  [--description=DESCRIPTION]  # description
-  [--tags=one two three]       # tags
-  [--md5=MD5]                  # MD5 hash
-  [--sha256=SHA256]            # SHA256 hash
-  [--mmh3=N]                   # MurmurHash3 hash
-
-Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
-
-```
-
-There are 2 ways to use this command.
-
-First one is passing `--md5`, `--sha256` and `--mmh3` parameters.
-
-```bash
-mihari http_hash --md5=881191f7736b5b8cfad5959ca99d2a51 --sha256=b064187ebdc51721708ad98cd89dacc346017cb0fb0457d530032d387f1ff20e --mmh3=-1467534799
-```
-
-Another one is passing `--html` parameter. In this case, hashes of an HTML file are automatically calculated.
-
-```bash
-wget http://example.com -O /tmp/index.html
-mihari http_hash --html /tmp/index.html
-```
-
-### Example usages
-
-```bash
-# Censys lookup for PANDA C2
-mihari censys '("PANDA" AND "SMAdmin" AND "layui")' --title "PANDA C2"
-
-# VirusTotal passive DNS lookup of a FAKESPY host
-mihari virustotal "jppost-hi.top" --title "FAKESPY passive DNS"
-
-# You can pass a "defanged" indicator as an input
-mihari virustotal "jppost-hi[.]top" --title "FAKESPY passive DNS"
-```
-
-### Import from JSON
-
-```bash
-echo '{ "title": "test", "description": "test", "artifacts": ["1.1.1.1", "github.com", "2.2.2.2"] }' | mihari import_from_json
-```
-
-The input is a JSON data should have `title`, `description` and `artifacts` key. `tags` key is an optional parameter.
-
-```json
-{
-  "title": "test",
-  "description": "test",
-  "artifacts": ["1.1.1.1", "github.com"],
-  "tags": ["test"]
-}
-```
-
-| Key         | Desc.                                                                      | Required or optional |
-|-------------|----------------------------------------------------------------------------|----------------------|
-| title       | A title of an alert                                                        | Required             |
-| description | A description of an alert                                                  | Required             |
-| artifacts   | An array of artifacts (supported data types: ip, domain, url, email, hash) | Required             |
-| tags        | An array of tags                                                           | Optional             |
-
-## Configuration
-
-Configuration can be done via environment variables or a YAML file.
-
-| Key                    | Description                                                                                     | Default     |
-|------------------------|-------------------------------------------------------------------------------------------------|-------------|
-| DATABASE               | A path to the SQLite database or a DB URL (e.g. `postgres://postgres:pass@db.host:5432/somedb`) | `mihari.db` |
-| BINARYEDGE_API_KEY     | BinaryEdge API key                                                                              |             |
-| CENSYS_ID              | Censys API ID                                                                                   |             |
-| CENSYS_SECRET          | Censys secret                                                                                   |             |
-| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password                                                                  |             |
-| CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username                                                                  |             |
-| MISP_API_ENDPOINT      | MISP URL                                                                                        |             |
-| MISP_API_KEY           | MISP API key                                                                                    |             |
-| ONYPHE_API_KEY         | Onyphe API key                                                                                  |             |
-| OTX_API_KEY            | OTX API key                                                                                     |             |
-| PASSIVETOTAL_API_KEY   | PassiveTotal API key                                                                            |             |
-| PASSIVETOTAL_USERNAME  | PassiveTotal username                                                                           |             |
-| PULSEDIVE_API_KEY      | Pulsedive API key                                                                               |             |
-| SECURITYTRAILS_API_KEY | SecurityTrails API key                                                                          |             |
-| SHODAN_API_KEY         | Shodan API key                                                                                  |             |
-| SLACK_CHANNEL          | Slack channel name                                                                              | `#general`  |
-| SLACK_WEBHOOK_URL      | Slack Webhook URL                                                                               |             |
-| SPYSE_API_KEY          | Spyse API key                                                                                   |             |
-| THEHIVE_API_ENDPOINT   | TheHive URL                                                                                     |             |
-| THEHIVE_API_KEY        | TheHive API key                                                                                 |             |
-| URLSCAN_API_KEY        | urlscan.io API key                                                                              |             |
-| VIRUSTOTAL_API_KEY     | VirusTotal API key                                                                              |             |
-| ZOOMEYE_PASSWORD       | ZoomEye password                                                                                |             |
-| ZOOMEYE_USERNAMME      | ZoomEye username                                                                                |             |
-
-Instead of using environment variables, you can use a YAML file for configuration.
-
-```bash
-mihari virustotal 1.1.1.1 --config /path/to/yaml.yml
-```
-
-The YAML file should be a YAML hash like below:
-
-```yaml
-database: /tmp/mihari.db
-thehive_api_endpoint: https://localhost
-thehive_api_key: foo
-virustotal_api_key: foo
-```
-
-You can check the configuration status via `status` command.
-
-```bash
-mihari status
-```
-
-## How to create a custom script
-
-Create a class which extends `Mihari::Analyzers::Base` and implements the following methods.
-
-| Name           | Desc.                                                                      | @return       | Required or optional |
-|----------------|----------------------------------------------------------------------------|---------------|----------------------|
-| `#title`       | A title of an alert                                                        | String        | Required             |
-| `#description` | A description of an alert                                                  | String        | Required             |
-| `#artifacts`   | An array of artifacts (supported data types: ip, domain, url, email, hash) | Array<String> | Required             |
-| `#tags`        | An array of tags                                                           | Array<String> | Optional             |
-
-```ruby
-require "mihari"
-
-module Mihari
-  module Analyzers
-    class Example < Base
-      def title
-        "example"
-      end
-
-      def description
-        "example"
-      end
-
-      def artifacts
-        ["9.9.9.9", "example.com"]
-      end
-
-      def tags
-        ["example"]
-      end
-    end
-  end
-end
-
-example = Mihari::Analyzers::Example.new
-example.run
-```
-
-See `/examples` for more.
+See [Usage](https://github.com/ninoseki/mihari/wiki/Usage) for more information.
 
-## Using it with Docker
+## Docs
 
-```bash
-$ docker run --rm ninoseki/mihari
-# Note that you should pass configurations via environment variables
-$ docker run --rm ninoseki/mihari -e THEHIVE_API_ENDPOINT="http://THEHIVE_URL" -e THEHIVE_API_KEY="API KEY" mihari
-# or
-$ docker run --rm ninoseki/mihari --env-file ~/.mihari.env mihari
-```
+- [Requirements & Installation](https://github.com/ninoseki/mihari/wiki/Requirements-&-Installation)
+- [Usage](https://github.com/ninoseki/mihari/wiki/Usage)
+- [Built-in Web App](https://github.com/ninoseki/mihari/wiki/Built-in-Web-App)
+- [Configuration](https://github.com/ninoseki/mihari/wiki/Configuration)
+- [Custom Script](https://github.com/ninoseki/mihari/wiki/Custom-Script)
+- [Docker](https://github.com/ninoseki/mihari/wiki/Docker)
+- [GitHub Actions](https://github.com/ninoseki/mihari/wiki/GitHub-Actions)
 
 ## License