mihari 1.5.1 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 951201ccebc7b6c4c117a687c1abce9ab24fa8d450f5a0f0badeeececa6db5cb
-  data.tar.gz: aed5f37c4031ffbc1a635ddd4fc979f4e40ab68a94ef880730bc48a2df600678
+  metadata.gz: 81f38ae809db93f21b93a26581fe591cb534c04e6cb1882925e310c4698878b6
+  data.tar.gz: 443bfdd2bbcdd9aee9360bd8e411fab32f2d0bc0f75e32c7ac1ce6a63a021fad
 SHA512:
-  metadata.gz: a81ff55bf880a3581ad52f6bf2d8652e1d7824119ed31daed81eb0e8f215d4d26a19a10d646b5497346e6795a320de18cd63fa817958081c2ba4393efad4c20a
-  data.tar.gz: b3fa4c9979fc22863d0e171d87918c320ec177b5e27c0820f4997919cb714d8c19516d37e15eccaed8b0a81102498820b4865864dcb56c650c186ef1bd057b56
+  metadata.gz: 9575e768712943a640c83b36da8d76c7d94a15995d67eb79b0a727381251c3eccb764b4e1d9e68d4d8480863ec5a209f5aaab144f0170cc7d5c6ff5a90032f21
+  data.tar.gz: 75d8052c9abaf7d1e421dd9ddc783b9a1cd41f3f2d4ac555230abef234724acbb10063af033cad18643c55170104f035c0ba3d950a350872ee63f7c007a46d7e

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,43 @@
+---
+name: Bug report
+about: Create a bug report to help us improve
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+<!--
+Thank you for taking the time to report a bug.
+Please make sure there is no existing issue about this kind of bug.
+-->
+
+### **Describe the bug**
+
+A clear and concise description of what the bug is.
+
+### **Steps to reproduce**
+
+- ...
+
+### **Expected behavior**
+
+A clear and concise description of what you expected to happen.
+
+### **Actual behavior**
+
+A clear and concise description of what actually happened.
+
+### **Screenshots**
+
+Add screenshots to help explain your problem.
+
+### **System Information:**
+
+- OS: [e.g. Windows10]
+- Ruby version: [e.g. 3.0]
+- Mihari version: [e.g. 2.0.0]
+
+### **Additional context**
+
+Add any other context about the problem here.

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,15 @@
+---
+name: Feature request
+about: Suggest a new Feature for Mihari
+title: "[Feature Request]"
+labels: enhancement
+assignees: ''
+
+---
+<!--
+
+1. Make sure your requested feature makes sense for Mihari.
+
+2. If you want to suggest a new integration of a service, please provide detailed information of it. (e.g. API docs)
+
+-->

data/.github/workflows/test.yml

@@ -4,11 +4,10 @@ on: [pull_request]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     services:
-      db:
+      postgres:
         image: postgres:12
         env:
           POSTGRES_USER: postgres
@@ -22,23 +21,48 @@ jobs:
         ports:
           - 5432:5432
 
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_USER: mysql
+          MYSQL_PASSWORD: mysql
+          MYSQL_DATABASE: test
+          MYSQL_ROOT_PASSWORD: rootpassword
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.7, '3.0']
+        ruby: [2.7, "3.0"]
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby 2.7
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby }}
-        bundler-cache: true
-    - name: Build and test with Rake
-      env:
-        DATABASE: postgresql://postgres:postgres@localhost:5432/test
-      run: |
-        sudo apt-get -yqq install libpq-dev
-        gem install bundler
-        bundle install
-        bundle exec rake
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get -yqq install libpq-dev libmysqlclient-dev
+          gem install bundler
+          bundle install
+
+      - name: Test with PostgreSQL
+        env:
+          DATABASE: postgresql://postgres:postgres@localhost:5432/test
+        run: |
+          bundle exec rake
+
+      - name: Test with MySQL
+        env:
+          DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
+        run: |
+          bundle exec rake

data/.rubocop.yml

@@ -4,6 +4,9 @@
 require:
   - rubocop-performance
 
+AllCops:
+  NewCops: enable
+
 Style/Alias:
   Enabled: false
   StyleGuide: https://relaxed.ruby.style/#stylealias
@@ -151,5 +154,8 @@ Lint/AssignmentInCondition:
 Layout/LineLength:
   Enabled: false
 
+Style/StringLiteralsInInterpolation:
+  Enabled: false
+
 Metrics:
   Enabled: false

data/.standard.yml

@@ -0,0 +1,4 @@
+ignore:
+  - "**/*":
+      - Layout/SpaceInsideHashLiteralBraces
+      - Style/RescueStandardError

data/README.md

@@ -8,65 +8,26 @@
 
 ![img](https://github.com/ninoseki/mihari/raw/master/images/logo.png)
 
-Mihari is a helper to run queries & manage results continuously. Mihari can be used for C2, landing page and phishing hunting.
+[![](images/tines.png)](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki)
+
+Mihari is a framework for continuous OSINT based threat hunting.
 
 ## How it works
 
-- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes).
-- Mihari checks whether a DB (SQLite3 or PostgreSQL) contains the artifacts or not.
+![img](https://github.com/ninoseki/mihari/raw/master/images/overview.png)
+
+- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs or hashes).
+- Mihari checks whether a DB (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
     - Mihari creates an alert on TheHive.
     - Mihari sends a notification to Slack.
     - Mihari creates an event on MISP.
 
-![img](https://github.com/ninoseki/mihari/raw/master/images/eyecatch.png)
-
-### Screenshots
-
-- TheHive alert example
-
-![img](https://github.com/ninoseki/mihari/raw/master/images/alert.png)
-
-- Slack notification example
-
-![img](https://github.com/ninoseki/mihari/raw/master/images/slack.png)
-
-- MISP event example
-
-![img](https://github.com/ninoseki/mihari/raw/master/images/misp.png)
-
-## Requirements
-
-- Ruby (2.7 or 3.0)
-- SQLite3 or PostgreSQL
-
-```bash
-# For Debian / Ubuntu
-apt-get install sqlite3 libsqlite3-dev libpq-dev
-```
-
-## Supported platforms & databases
-
-| Name       | Supported versions |
-|------------|--------------------|
-| PostgreSQL | v12                |
-| SQLite     | v3                 |
-| MISP       | v2.4               |
-| TheHive    | v3.x & v4.x        |
-
-## Installation
-
-```bash
-gem install mihari
-```
+Also, you can check the alerts on a built-in web app.
 
-Or you can use this tool with Docker.
+![img](https://github.com/ninoseki/mihari/raw/master/images/web_alerts.png)
 
-```bash
-docker pull ninoseki/mihari
-```
-
-## Basic usage
+## Supported services
 
 Mihari supports the following services by default.
 
@@ -87,234 +48,22 @@ Mihari supports the following services by default.
 - [VirusTotal](http://virustotal.com)
 - [ZoomEye](https://zoomeye.org)
 
-```bash
-$ mihari
-Commands:
-  mihari alerts                               # Show the alerts on TheHive
-  mihari binaryedge [QUERY]                   # BinaryEdge host search by a query
-  mihari censys [QUERY]                       # Censys IPv4 search by a query
-  mihari circl [DOMAIN|SHA1]                  # CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint
-  mihari crtsh [QUERY]                        # crt.sh search by a query
-  mihari dnpedia [QUERY]                      # DNPedia domain search by a query
-  mihari dnstwister [DOMAIN]                  # dnstwister lookup by a domain
-  mihari free_text [TEXT]                     # Cross search with search engines by a free text
-  mihari help [COMMAND]                       # Describe available commands or one specific command
-  mihari http_hash                            # Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
-  mihari import_from_json                     # Give a JSON input via STDIN
-  mihari onyphe [QUERY]                       # Onyphe datascan search by a query
-  mihari otx [IP|DOMAIN]                      # OTX lookup by an IP or domain
-  mihari passive_dns [IP|DOMAIN]              # Cross search with passive DNS services by an ip or domain
-  mihari passive_ssl [SHA1]                   # Cross search with passive SSL services by an SHA1 certificate fingerprint
-  mihari passivetotal [IP|DOMAIN|EMAIL|SHA1]  # PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint
-  mihari pulsedive [IP|DOMAIN]                # Pulsedive lookup by an ip or domain
-  mihari reverse_whois [EMAIL]                # Cross search with reverse whois services by an email
-  mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by an ip, domain or email
-  mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed search by a regexp
-  mihari shodan [QUERY]                       # Shodan host search by a query
-  mihari spyse [QUERY]                        # Spyse search by a query
-  mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)
-  mihari status                               # Show the current configuration status
-  mihari urlscan [QUERY]                      # urlscan search by a given query
-  mihari virustotal [IP|DOMAIN]               # VirusTotal resolutions lookup by an ip or domain
-  mihari zoomeye [QUERY]                      # ZoomEye search by a query
-
-Options:
-  [--config=CONFIG]  # path to config file
-
-```
-
-### Cross searches
-
-Mihari has cross search features. A cross search is a search across a number of services.
-
-You can get aggregated results by using the following commands.
-
-| Command         | Desc.                                                                                                   |
-|-----------------|---------------------------------------------------------------------------------------------------------|
-| passive_dns     | Passive DNS lookup with CIRCL passive DNS, OTX, PassiveTotal, Pulsedive, SecurityTrails and VirusTotal  |
-| passive_ssl     | Passive SSL lookup with CIRCL passive SSL and PassiveTotal                                              |
-| reverse_whois   | Revese Whois lookup with PassiveTotal and SecurityTrails                                                |
-| http_hash       | HTTP response hash lookup with BinaryEdge(SHA256), Censys(SHA256), Onyphpe(MD5) and Shodan(MurmurHash3) |
-| free_text       | Free text lookup with BinaryEdge and Censys                                                             |
-| ssh_fingerprint | SSH fingerprint lookup with BinaryEdge and Shodan                                                       |
-
-#### http_hash command
-
-The usage of `http_hash` command is a little bit tricky.
-
-```bash
-$ mihari help http_hash
-Usage:
-  mihari http_hash
-
-Options:
-  [--title=TITLE]              # title
-  [--description=DESCRIPTION]  # description
-  [--tags=one two three]       # tags
-  [--md5=MD5]                  # MD5 hash
-  [--sha256=SHA256]            # SHA256 hash
-  [--mmh3=N]                   # MurmurHash3 hash
-
-Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
-
-```
-
-There are 2 ways to use this command.
-
-First one is passing `--md5`, `--sha256` and `--mmh3` parameters.
-
-```bash
-mihari http_hash --md5=881191f7736b5b8cfad5959ca99d2a51 --sha256=b064187ebdc51721708ad98cd89dacc346017cb0fb0457d530032d387f1ff20e --mmh3=-1467534799
-```
+See [Usage](https://github.com/ninoseki/mihari/wiki/Usage) for more information.
 
-Another one is passing `--html` parameter. In this case, hashes of an HTML file are automatically calculated.
+## Docs
 
-```bash
-wget http://example.com -O /tmp/index.html
-mihari http_hash --html /tmp/index.html
-```
-
-### Example usages
-
-```bash
-# Censys lookup for PANDA C2
-mihari censys '("PANDA" AND "SMAdmin" AND "layui")' --title "PANDA C2"
-
-# VirusTotal passive DNS lookup of a FAKESPY host
-mihari virustotal "jppost-hi.top" --title "FAKESPY passive DNS"
-
-# You can pass a "defanged" indicator as an input
-mihari virustotal "jppost-hi[.]top" --title "FAKESPY passive DNS"
-```
-
-### Import from JSON
-
-```bash
-echo '{ "title": "test", "description": "test", "artifacts": ["1.1.1.1", "github.com", "2.2.2.2"] }' | mihari import_from_json
-```
-
-The input is a JSON data should have `title`, `description` and `artifacts` key. `tags` key is an optional parameter.
-
-```json
-{
-  "title": "test",
-  "description": "test",
-  "artifacts": ["1.1.1.1", "github.com"],
-  "tags": ["test"]
-}
-```
-
-| Key         | Desc.                                                                      | Required or optional |
-|-------------|----------------------------------------------------------------------------|----------------------|
-| title       | A title of an alert                                                        | Required             |
-| description | A description of an alert                                                  | Required             |
-| artifacts   | An array of artifacts (supported data types: ip, domain, url, email, hash) | Required             |
-| tags        | An array of tags                                                           | Optional             |
-
-## Configuration
-
-Configuration can be done via environment variables or a YAML file.
-
-| Key                    | Description                                                                                     | Default     |
-|------------------------|-------------------------------------------------------------------------------------------------|-------------|
-| DATABASE               | A path to the SQLite database or a DB URL (e.g. `postgres://postgres:pass@db.host:5432/somedb`) | `mihari.db` |
-| BINARYEDGE_API_KEY     | BinaryEdge API key                                                                              |             |
-| CENSYS_ID              | Censys API ID                                                                                   |             |
-| CENSYS_SECRET          | Censys secret                                                                                   |             |
-| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password                                                                  |             |
-| CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username                                                                  |             |
-| MISP_API_ENDPOINT      | MISP URL                                                                                        |             |
-| MISP_API_KEY           | MISP API key                                                                                    |             |
-| ONYPHE_API_KEY         | Onyphe API key                                                                                  |             |
-| OTX_API_KEY            | OTX API key                                                                                     |             |
-| PASSIVETOTAL_API_KEY   | PassiveTotal API key                                                                            |             |
-| PASSIVETOTAL_USERNAME  | PassiveTotal username                                                                           |             |
-| PULSEDIVE_API_KEY      | Pulsedive API key                                                                               |             |
-| SECURITYTRAILS_API_KEY | SecurityTrails API key                                                                          |             |
-| SHODAN_API_KEY         | Shodan API key                                                                                  |             |
-| SLACK_CHANNEL          | Slack channel name                                                                              | `#general`  |
-| SLACK_WEBHOOK_URL      | Slack Webhook URL                                                                               |             |
-| SPYSE_API_KEY          | Spyse API key                                                                                   |             |
-| THEHIVE_API_ENDPOINT   | TheHive URL                                                                                     |             |
-| THEHIVE_API_KEY        | TheHive API key                                                                                 |             |
-| URLSCAN_API_KEY        | urlscan.io API key                                                                              |             |
-| VIRUSTOTAL_API_KEY     | VirusTotal API key                                                                              |             |
-| ZOOMEYE_PASSWORD       | ZoomEye password                                                                                |             |
-| ZOOMEYE_USERNAMME      | ZoomEye username                                                                                |             |
-
-Instead of using environment variables, you can use a YAML file for configuration.
-
-```bash
-mihari virustotal 1.1.1.1 --config /path/to/yaml.yml
-```
-
-The YAML file should be a YAML hash like below:
-
-```yaml
-database: /tmp/mihari.db
-thehive_api_endpoint: https://localhost
-thehive_api_key: foo
-virustotal_api_key: foo
-```
-
-You can check the configuration status via `status` command.
-
-```bash
-mihari status
-```
-
-## How to create a custom script
-
-Create a class which extends `Mihari::Analyzers::Base` and implements the following methods.
-
-| Name           | Desc.                                                                      | @return       | Required or optional |
-|----------------|----------------------------------------------------------------------------|---------------|----------------------|
-| `#title`       | A title of an alert                                                        | String        | Required             |
-| `#description` | A description of an alert                                                  | String        | Required             |
-| `#artifacts`   | An array of artifacts (supported data types: ip, domain, url, email, hash) | Array<String> | Required             |
-| `#tags`        | An array of tags                                                           | Array<String> | Optional             |
-
-```ruby
-require "mihari"
-
-module Mihari
-  module Analyzers
-    class Example < Base
-      def title
-        "example"
-      end
-
-      def description
-        "example"
-      end
-
-      def artifacts
-        ["9.9.9.9", "example.com"]
-      end
-
-      def tags
-        ["example"]
-      end
-    end
-  end
-end
-
-example = Mihari::Analyzers::Example.new
-example.run
-```
-
-See `/examples` for more.
-
-## Using it with Docker
-
-```bash
-$ docker run --rm ninoseki/mihari
-# Note that you should pass configurations via environment variables
-$ docker run --rm ninoseki/mihari -e THEHIVE_API_ENDPOINT="http://THEHIVE_URL" -e THEHIVE_API_KEY="API KEY" mihari
-# or
-$ docker run --rm ninoseki/mihari --env-file ~/.mihari.env mihari
-```
+- [Requirements & Installation](https://github.com/ninoseki/mihari/wiki/Requirements-&-Installation)
+- [Usage](https://github.com/ninoseki/mihari/wiki/Usage)
+- [Built-in Web App](https://github.com/ninoseki/mihari/wiki/Built-in-Web-App)
+- [Configuration](https://github.com/ninoseki/mihari/wiki/Configuration)
+- [Custom Script](https://github.com/ninoseki/mihari/wiki/Custom-Script)
+- [Docker](https://github.com/ninoseki/mihari/wiki/Docker)
+- [GitHub Actions](https://github.com/ninoseki/mihari/wiki/GitHub-Actions)
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Acknowledgement
+
+Mihari is proudly supported by [Tines.io](https://tines.io?utm_source=github&utm_medium=sponsorship&utm_campaign=ninoseki), The SOAR Platform for Enterprise Security Teams.