mihari 1.3.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 512d3ee8179279b931bd9510c652693ead1108ead99c823e26880e2a75234b24
-  data.tar.gz: 81946c213ef30712644637e8ea5e01bea36956aad077ed37bdac60d0adf71f19
+  metadata.gz: 3cf190c27b1be3c0eeb690354361c3bb3295beb8609661a861fcc31e227cb7f0
+  data.tar.gz: e989209e3668e9342803ed1b91c9e668bcf0519e659f9fb88687cc38460818cb
 SHA512:
-  metadata.gz: d6e8b1e9a8791aebfe042de31b1f895fce4bc20d26a417672eaf4dafa83319f2cfb407ee635fe004f70a9d78441c43398c0af79cde01b3bb39bdcfb1dfd9e0c3
-  data.tar.gz: '09ad98242f96474358908d68ef24f2f8711206b698499d18419346004a853ed5cace40df63e1055e82444e43ac9160fa02fc40d0a662d7a17d290a0dee13fb50'
+  metadata.gz: 0a40de96264eaf211240b0020aa3494e6a64bddf8e2aa74461e91122fff29d72a5ab66032e226fa80c19c984fcb7a3e37d1e78aa0da317f6fd1445945b93d200
+  data.tar.gz: b0b55e75e84037cf7761fd28128504d45fc049de01d0f4db65184b2e8ff0e9bc3948e06b8165e90c19244fb1c8b8af33efa77d260bb323d4084d85e21f72245f

data/.github/workflows/test.yml

@@ -0,0 +1,68 @@
+name: Ruby CI
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_USER: mysql
+          MYSQL_PASSWORD: mysql
+          MYSQL_DATABASE: test
+          MYSQL_ROOT_PASSWORD: rootpassword
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, "3.0"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get -yqq install libpq-dev libmysqlclient-dev
+          gem install bundler
+          bundle install
+
+      - name: Test with PostgreSQL
+        env:
+          DATABASE: postgresql://postgres:postgres@localhost:5432/test
+        run: |
+          bundle exec rake
+
+      - name: Test with MySQL
+        env:
+          DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
+        run: |
+          bundle exec rake

data/README.md

@@ -1,62 +1,29 @@
 # mihari
 
 [![Gem Version](https://badge.fury.io/rb/mihari.svg)](https://badge.fury.io/rb/mihari)
-[![Build Status](https://travis-ci.com/ninoseki/mihari.svg?branch=master)](https://travis-ci.com/ninoseki/mihari)
+[![Ruby CI](https://github.com/ninoseki/mihari/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/mihari/actions/workflows/test.yml)
 [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/mihari)](https://hub.docker.com/r/ninoseki/mihari)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/mihari/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/mihari?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/mihari/badge)](https://www.codefactor.io/repository/github/ninoseki/mihari)
 
-Mihari is a helper to run queries & manage results continuously. Mihari can be used for C2, landing page and phishing hunting.
+![img](https://github.com/ninoseki/mihari/raw/master/images/logo.png)
+
+Mihari is a framework for continuous OSINT based threat hunting.
 
 ## How it works
 
-- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs and hashes) from the results.
-- Mihari checks whether a DB (SQLite3 or PostgreSQL) contains the artifacts or not.
+- Mihari makes a query against Shodan, Censys, VirusTotal, SecurityTrails, etc. and extracts artifacts (IP addresses, domains, URLs or hashes).
+- Mihari checks whether a DB (SQLite3, PostgreSQL or MySQL) contains the artifacts or not.
   - If it doesn't contain the artifacts:
-    - Mihari creates an alert on TheHive. (Optional)
-    - Mihari sends a notification to Slack. (Optional)
-    - Mihari creates an event on MISP. (Optional)
-
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/eyecatch.png)
-
-### Screenshots
-
-- TheHive alert example
-
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/alert.png)
-
-- Slack notification example
-
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/slack.png)
-
-- MISP event example
+    - Mihari creates an alert on TheHive.
+    - Mihari sends a notification to Slack.
+    - Mihari creates an event on MISP.
 
-![img](https://github.com/ninoseki/mihari/raw/master/screenshots/misp.png)
+Also, you can check the alerts on a built-in web app.
 
-## Requirements
+![img](https://github.com/ninoseki/mihari/raw/master/images/web_alerts.png)
 
-- Ruby 2.6+
-- SQLite3
-- libpq
-
-```bash
-# For Debian / Ubuntu
-apt-get install sqlite3 libsqlite3-dev libpq-dev
-```
-
-## Installation
-
-```bash
-gem install mihari
-```
-
-Or you can use this tool with Docker.
-
-```bash
-docker pull ninoseki/mihari
-```
-
-## Basic usage
+## Supported services
 
 Mihari supports the following services by default.
 
@@ -69,6 +36,7 @@ Mihari supports the following services by default.
 - [Onyphe](https://onyphe.io)
 - [OTX](https://otx.alienvault.com/)
 - [PassiveTotal](https://community.riskiq.com/)
+- [Pulsedive](https://pulsedive.com/)
 - [SecurityTrails](https://securitytrails.com/)
 - [Shodan](https://shodan.io)
 - [Spyse](https://spyse.com)
@@ -76,233 +44,15 @@ Mihari supports the following services by default.
 - [VirusTotal](http://virustotal.com)
 - [ZoomEye](https://zoomeye.org)
 
-```bash
-$ mihari
-Commands:
-  mihari alerts                               # Show the alerts on TheHive
-  mihari binaryedge [QUERY]                   # BinaryEdge host search by a query
-  mihari censys [QUERY]                       # Censys IPv4 search by a query
-  mihari circl [DOMAIN|SHA1]                  # CIRCL passive DNS/SSL lookup by a domain or SHA1 certificate fingerprint
-  mihari crtsh [QUERY]                        # crt.sh search by a query
-  mihari dnpedia [QUERY]                      # DNPedia domain search by a query
-  mihari dnstwister [DOMAIN]                  # dnstwister lookup by a domain
-  mihari free_text [TEXT]                     # Cross search with search engines by a free text
-  mihari help [COMMAND]                       # Describe available commands or one specific command
-  mihari http_hash                            # Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
-  mihari import_from_json                     # Give a JSON input via STDIN
-  mihari onyphe [QUERY]                       # Onyphe datascan search by a query
-  mihari otx [IP|DOMAIN]                      # OTX lookup by an IP or domain
-  mihari passive_dns [IP|DOMAIN]              # Cross search with passive DNS services by an ip or domain
-  mihari passive_ssl [SHA1]                   # Cross search with passive SSL services by an SHA1 certificate fingerprint
-  mihari passivetotal [IP|DOMAIN|EMAIL|SHA1]  # PassiveTotal lookup by an ip, domain, email or SHA1 certificate fingerprint
-  mihari pulsedive [IP|DOMAIN]                # Pulsedive lookup by an ip or domain
-  mihari reverse_whois [EMAIL]                # Cross search with reverse whois services by an email
-  mihari securitytrails [IP|DOMAIN|EMAIL]     # SecurityTrails lookup by an ip, domain or email
-  mihari securitytrails_domain_feed [REGEXP]  # SecurityTrails new domain feed search by a regexp
-  mihari shodan [QUERY]                       # Shodan host search by a query
-  mihari spyse [QUERY]                        # Spyse search by a query
-  mihari ssh_fingerprint [FINGERPRINT]        # Cross search with search engines by an SSH fingerprint (e.g. dc:14:de:8e:d7:c1:15:43:23:82:25:81:d2:59:e8:c0)
-  mihari status                               # Show the current configuration status
-  mihari urlscan [QUERY]                      # urlscan search by a given query
-  mihari virustotal [IP|DOMAIN]               # VirusTotal resolutions lookup by an ip or domain
-  mihari zoomeye [QUERY]                      # ZoomEye search by a query
-
-Options:
-  [--config=CONFIG]  # path to config file
-
-```
-
-### Cross searches
-
-Mihari has cross search features. A cross search is a search across a number of services.
-
-You can get aggregated results by using the following commands.
-
-| Command         | Desc.                                                                                                   |
-|-----------------|---------------------------------------------------------------------------------------------------------|
-| passive_dns     | Passive DNS lookup with CIRCL passive DNS, OTX, PassiveTotal, Pulsedive, SecurityTrails and VirusTotal  |
-| passive_ssl     | Passive SSL lookup with CIRCL passive SSL and PassiveTotal                                              |
-| reverse_whois   | Revese Whois lookup with PassiveTotal and SecurityTrails                                                |
-| http_hash       | HTTP response hash lookup with BinaryEdge(SHA256), Censys(SHA256), Onyphpe(MD5) and Shodan(MurmurHash3) |
-| free_text       | Free text lookup with BinaryEdge and Censys                                                             |
-| ssh_fingerprint | SSH fingerprint lookup with BinaryEdge and Shodan                                                       |
-
-#### http_hash command
-
-The usage of `http_hash` command is a little bit tricky.
-
-```bash
-$ mihari help http_hash
-Usage:
-  mihari http_hash
-
-Options:
-  [--title=TITLE]              # title
-  [--description=DESCRIPTION]  # description
-  [--tags=one two three]       # tags
-  [--md5=MD5]                  # MD5 hash
-  [--sha256=SHA256]            # SHA256 hash
-  [--mmh3=N]                   # MurmurHash3 hash
-
-Cross search with search engines by a hash of an HTTP response (SHA256, MD5 and MurmurHash3)
-
-```
-
-There are 2 ways to use this command.
-
-First one is passing `--md5`, `--sha256` and `--mmh3` parameters.
-
-```bash
-mihari http_hash --md5=881191f7736b5b8cfad5959ca99d2a51 --sha256=b064187ebdc51721708ad98cd89dacc346017cb0fb0457d530032d387f1ff20e --mmh3=-1467534799
-```
-
-Another one is passing `--html` parameter. In this case, hashes of an HTML file are automatically calculated.
-
-```bash
-wget http://example.com -O /tmp/index.html
-mihari http_hash --html /tmp/index.html
-```
-
-### Example usages
-
-```bash
-# Censys lookup for PANDA C2
-mihari censys '("PANDA" AND "SMAdmin" AND "layui")' --title "PANDA C2"
-
-# VirusTotal passive DNS lookup of a FAKESPY host
-mihari virustotal "jppost-hi.top" --title "FAKESPY passive DNS"
-
-# You can pass a "defanged" indicator as an input
-mihari virustotal "jppost-hi[.]top" --title "FAKESPY passive DNS"
-```
-
-### Import from JSON
-
-```bash
-echo '{ "title": "test", "description": "test", "artifacts": ["1.1.1.1", "github.com", "2.2.2.2"] }' | mihari import_from_json
-```
-
-The input is a JSON data should have `title`, `description` and `artifacts` key. `tags` key is an optional parameter.
-
-```json
-{
-  "title": "test",
-  "description": "test",
-  "artifacts": ["1.1.1.1", "github.com"],
-  "tags": ["test"]
-}
-```
-
-| Key         | Desc.                                                                      | Required or optional |
-|-------------|----------------------------------------------------------------------------|----------------------|
-| title       | A title of an alert                                                        | Required             |
-| description | A description of an alert                                                  | Required             |
-| artifacts   | An array of artifacts (supported data types: ip, domain, url, email, hash) | Required             |
-| tags        | An array of tags                                                           | Optional             |
-
-## Configuration
-
-Configuration can be done via environment variables or a YAML file.
-
-| Key                    | Description                                                                                     | Default     |
-|------------------------|-------------------------------------------------------------------------------------------------|-------------|
-| DATABASE               | A path to the SQLite database or a DB URL (e.g. `postgres://postgres:pass@db.host:5432/somedb`) | `mihari.db` |
-| BINARYEDGE_API_KEY     | BinaryEdge API key                                                                              |             |
-| CENSYS_ID              | Censys API ID                                                                                   |             |
-| CENSYS_SECRET          | Censys secret                                                                                   |             |
-| CIRCL_PASSIVE_PASSWORD | CIRCL passive DNS/SSL password                                                                  |             |
-| CIRCL_PASSIVE_USERNAME | CIRCL passive DNS/SSL username                                                                  |             |
-| MISP_API_ENDPOINT      | MISP URL                                                                                        |             |
-| MISP_API_KEY           | MISP API key                                                                                    |             |
-| ONYPHE_API_KEY         | Onyphe API key                                                                                  |             |
-| OTX_API_KEY            | OTX API key                                                                                     |             |
-| PASSIVETOTAL_API_KEY   | PassiveTotal API key                                                                            |             |
-| PASSIVETOTAL_USERNAME  | PassiveTotal username                                                                           |             |
-| PULSEDIVE_API_KEY      | Pulsedive API key                                                                               |             |
-| SECURITYTRAILS_API_KEY | SecurityTrails API key                                                                          |             |
-| SHODAN_API_KEY         | Shodan API key                                                                                  |             |
-| SLACK_CHANNEL          | Slack channel name                                                                              | `#general`  |
-| SLACK_WEBHOOK_URL      | Slack Webhook URL                                                                               |             |
-| SPYSE_API_KEY          | Spyse API key                                                                                   |             |
-| THEHIVE_API_ENDPOINT   | TheHive URL                                                                                     |             |
-| THEHIVE_API_KEY        | TheHive API key                                                                                 |             |
-| URLSCAN_API_KEY        | urlscan.io API key                                                                              |             |
-| VIRUSTOTAL_API_KEY     | VirusTotal API key                                                                              |             |
-| ZOOMEYE_PASSWORD       | ZoomEye password                                                                                |             |
-| ZOOMEYE_USERNAMME      | ZoomEye username                                                                                |             |
-
-Instead of using environment variables, you can use a YAML file for configuration.
-
-```bash
-mihari virustotal 1.1.1.1 --config /path/to/yaml.yml
-```
-
-The YAML file should be a YAML hash like below:
-
-```yaml
-database: /tmp/mihari.db
-thehive_api_endpoint: https://localhost
-thehive_api_key: foo
-virustotal_api_key: foo
-```
-
-You can check the configuration status via `status` command.
-
-```bash
-mihari status
-```
-
-## How to create a custom script
-
-Create a class which extends `Mihari::Analyzers::Base` and implements the following methods.
-
-| Name           | Desc.                                                                      | @return       | Required or optional |
-|----------------|----------------------------------------------------------------------------|---------------|----------------------|
-| `#title`       | A title of an alert                                                        | String        | Required             |
-| `#description` | A description of an alert                                                  | String        | Required             |
-| `#artifacts`   | An array of artifacts (supported data types: ip, domain, url, email, hash) | Array<String> | Required             |
-| `#tags`        | An array of tags                                                           | Array<String> | Optional             |
-
-```ruby
-require "mihari"
-
-module Mihari
-  module Analyzers
-    class Example < Base
-      def title
-        "example"
-      end
-
-      def description
-        "example"
-      end
-
-      def artifacts
-        ["9.9.9.9", "example.com"]
-      end
-
-      def tags
-        ["example"]
-      end
-    end
-  end
-end
-
-example = Mihari::Analyzers::Example.new
-example.run
-```
-
-See `/examples` for more.
+See [Usage](https://github.com/ninoseki/mihari/wiki/Usage) for more information.
 
-## Using it with Docker
+## Docs
 
-```bash
-$ docker run --rm ninoseki/mihari
-# Note that you should pass configurations via environment variables
-$ docker run --rm ninoseki/mihari -e THEHIVE_API_ENDPOINT="http://THEHIVE_URL" -e THEHIVE_API_KEY="API KEY" mihari
-# or
-$ docker run --rm ninoseki/mihari --env-file ~/.mihari.env mihari
-```
+- [Requirements & Installation](https://github.com/ninoseki/mihari/wiki/Requirements-&-Installation)
+- [Usage](https://github.com/ninoseki/mihari/wiki/Usage)
+- [Configuration](https://github.com/ninoseki/mihari/wiki/Configuration)
+- [Custom script](https://github.com/ninoseki/mihari/wiki/Custom-script)
+- [Docker](https://github.com/ninoseki/mihari/wiki/Docker)
 
 ## License
 

data/Rakefile

@@ -2,6 +2,7 @@
 
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "standard/rake"
 
 RSpec::Core::RakeTask.new(:spec)
 

data/build_frontend.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+CURRENT_DIR=${PWD}
+
+mkdir -p tmp
+cd tmp
+git clone https://github.com/ninoseki/mihari-frontend.git
+cd mihari-frontend
+npm install
+npm run build
+
+cp -r dist/* ${CURRENT_DIR}/lib/mihari/web/public
+
+rm -rf ${CURRENT_DIR}/tmp/mihari-frontend

data/docker/Dockerfile

@@ -1,4 +1,5 @@
-FROM ruby:2.7-alpine3.10
+FROM ruby:3.0.0-alpine3.13
+
 RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/mihari.git \
@@ -10,4 +11,4 @@ RUN apk --no-cache add git build-base ruby-dev sqlite-dev postgresql-dev \
 
 ENTRYPOINT ["mihari"]
 
-CMD ["--help"]
+CMD ["--help"]