miteru 0.14.5 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cf6445b9388cdcfb797cdb8c91615a288757508874d0550bab8c6b052c375d6
-  data.tar.gz: dd9f428317a0f325c2c467234dda21f05bb7cd616114f49adeec9374a4680451
+  metadata.gz: 58464f5c1b0b17549201ea547edbcf22cd83bcbbd7ef1a1233ab2d04eeb4c13c
+  data.tar.gz: 6a8048507e00ded2fe11ef1a71b05536a267ee54b6e64b7662cf5f238ace84fd
 SHA512:
-  metadata.gz: 5f740f27cacdb1020a49bc787aa8486cbe6b6ac6fe5e4ad3093af0601928dcd12c78895c39c548a28de18fc35766dbb89507687c81e2bb162d6e69e111242fbd
-  data.tar.gz: a110c028b6129a2d0bc903962d1f945be96426b8c909ae6e34bb1955d3ca657b1c9ef5a48bc1e147c54f992517cc86c0ca18047a79cdcb2b2a95bda5c5148caa
+  metadata.gz: efb6fcb505fff1451ab5cbff5a0bbc1d3e480aa7cc4f51005b84065c5c174b4c12e9609f07217f7cbf0bdb844dd8efe7430e99e2f3bb7ea3fa7207d9ff531ae4
+  data.tar.gz: 5ab2d22a2c9c647665a3feefd9b579ac336e434a84320db6df4ba3b7e1ea4d44a14b1db7a91f8e63add7e7a0d544d0d791cd07f1d4258065642c2fae35c0002b

data/.github/workflows/test.yml

@@ -0,0 +1,68 @@
+name: Ruby CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_USER: mysql
+          MYSQL_PASSWORD: mysql
+          MYSQL_DATABASE: test
+          MYSQL_ROOT_PASSWORD: rootpassword
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, "3.0"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get -yqq install libpq-dev libmysqlclient-dev
+          gem install bundler
+          bundle install
+
+      - name: Test with PostgreSQL
+        env:
+          MITERU_DATABASE: postgresql://postgres:postgres@localhost:5432/test
+        run: |
+          bundle exec rake
+
+      - name: Test with MySQL
+        env:
+          MITERU_DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
+        run: |
+          bundle exec rake

data/.gitignore

@@ -51,3 +51,6 @@ Gemfile.lock
 
 ## RSpec
 .rspec_status
+
+# SQLite database
+*.db

data/.overcommit.yml

@@ -0,0 +1,12 @@
+PreCommit:
+  BundleCheck:
+    enabled: true
+
+  RuboCop:
+    enabled: true
+    required_executable: bundle
+    command: ["bundle", "exec", "standardrb"]
+    on_warn: fail
+
+  YamlSyntax:
+    enabled: true

data/.standard.yml

@@ -0,0 +1,4 @@
+ignore:
+  - "**/*":
+      - Layout/SpaceInsideHashLiteralBraces
+      - Style/RescueStandardError

data/README.md

@@ -1,109 +1,40 @@
 # Miteru
 
 [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
-[![Build Status](https://travis-ci.com/ninoseki/miteru.svg?branch=master)](https://travis-ci.com/ninoseki/miteru)
-[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)](https://hub.docker.com/repository/docker/ninoseki/miteru)
+[![Ruby CI](https://github.com/ninoseki/miteru/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/miteru/actions/workflows/test.yml)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
 
 Miteru is an experimental phishing kit detection tool.
 
+## Disclaimer
+
+This tool is for research purposes only. The use of this tool is your responsibility.
+I take no responsibility and/or liability for how you choose to use this tool.
+
 ## How it works
 
 - It collects phishy URLs from the following feeds:
-  - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#certstream-suspicious)
-  - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
-  - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
-  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
+  - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#task.source%3Acertstream-suspicious)
+  - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#task.source%3Aopenphish)
+  - [PhishTank feed via urlscan.io](https://urlscan.io/search/#task.source%3Aphishtank)
+  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#task.source%3Aurlhaus)
   - urlscan.io phish feed (available for Pro users)
   - [Ayashige feed](https://github.com/ninoseki/ayashige)
   - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
   - [PhishStats feed](https://phishstats.info/)
 - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
-  - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
+  - Note: Supported compressed files are: `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 
 ## Features
 
-- [x] Phishing kit detection & collection.
-- [x] Slack notification.
-- [x] Threading.
-
-## Installation
-
-```bash
-gem install miteru
-```
-
-## Usage
-
-```bash
-$ miteru
-Commands:
-  miteru execute         # Execute the crawler
-  miteru help [COMMAND]  # Describe available commands or one specific command
-```
-
-```bash
-$ miteru help execute
-Usage:
-  miteru execute
-
-Options:
-  [--auto-download], [--no-auto-download]              # Enable or disable auto-download of phishing kits
-  [--ayashige], [--no-ayashige]                        # Enable or disable ayashige(ninoseki/ayashige) feed
-  [--directory-traveling], [--no-directory-traveling]  # Enable or disable directory traveling
-  [--download-to=DOWNLOAD_TO]                          # Directory to download file(s)
-                                                       # Default: /tmp
-  [--post-to-slack], [--no-post-to-slack]              # Post a message to Slack if it detects a phishing kit
-  [--size=N]                                           # Number of urlscan.io's results. (Max: 10,000)
-                                                       # Default: 100
-  [--threads=N]                                        # Number of threads to use
-  [--verbose], [--no-verbose]
-                                                       # Default: true
-
-Execute the crawler
-```
-
-```bash
-$ miteru execute
-...
-https://dummy1.com: it doesn't contain a phishing kit.
-https://dummy2.com: it doesn't contain a phishing kit.
-https://dummy3.com: it doesn't contain a phishing kit.
-https://dummy4.com: it might contain a phishing kit (dummy.zip).
-```
-
-## Using Docker (alternative if you don't install Ruby)
-
-```bash
-$ docker pull ninoseki/miteru
-# ex. auto-download detected phishing kit(s) into host machines's /tmp directory
-$ docker run --rm -v /tmp:/tmp ninoseki/miteru execute --auto-download
-```
-
-## Configuration
-
-For using `--post-to-slack` feature, you should set the following environment variables:
-
-- `SLACK_WEBHOOK_URL`: Your Slack Webhook URL.
-- `SLACK_CHANNEL`: Slack channel to post a message (default: "#general").
-
-If you are a urlscan.io Pro user, set your API key as an environment variable `URLSCAN_API_KEY`.
-
-It enables you to subscribe the urlscan.io phish feed.
-
-## Examples
-
-### Aasciinema cast
-
-[![asciicast](https://asciinema.org/a/hHpkHhMLiiv17gmdRhVMtZWwM.svg)](https://asciinema.org/a/hHpkHhMLiiv17gmdRhVMtZWwM)
-
-### Slack notification
-
-![img](./screenshots/slack.png)
+- [x] Phishing kit detection & collection
+- [x] Slack notification
+- [x] Threading
 
-## Alternatives
+## Docs
 
-- [t4d/StalkPhish](https://github.com/t4d/StalkPhish): The Phishing kits stalker, harvesting phishing kits for investigations.
-- [duo-labs/phish-collect](https://github.com/duo-labs/phish-collect): Python script to hunt phishing kits.
-- [leunammejii/analyst_arsenal](https://github.com/leunammejii/analyst_arsenal): A tool belt for analysts to continue fighting the good fight.
+- [Requirements & Installation](https://github.com/ninoseki/miteru/wiki/Requirements-&-Installation)
+- [Usage](https://github.com/ninoseki/miteru/wiki/Usage)
+- [Configuration](https://github.com/ninoseki/miteru/wiki/Configuration)
+- [Alternatives](https://github.com/ninoseki/miteru/wiki/Alternatives)

data/docker/Dockerfile

@@ -1,10 +1,13 @@
-FROM ruby:2.6-alpine3.10
-RUN apk --no-cache add git build-base ruby-dev \
+FROM ruby:3-alpine3.13
+
+RUN apk --no-cache add git build-base ruby-dev mysql-client mysql-dev sqlite-dev postgresql-client postgresql-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/miteru.git \
   && cd miteru \
   && gem build miteru.gemspec -o miteru.gem \
   && gem install miteru.gem \
+  && gem install mysql2 \
+  && gem install pg \
   && rm -rf /tmp/miteru \
   && apk del --purge git build-base ruby-dev
 

data/exe/miteru

@@ -5,4 +5,5 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "miteru"
 
-Miteru::CLI.start
+ARGV.unshift(Miteru::CLI.default_task) unless Miteru::CLI.all_tasks.key?(ARGV[0])
+Miteru::CLI.start(ARGV)

data/lib/miteru/attachement.rb

@@ -5,6 +5,7 @@ require "uri"
 module Miteru
   class Attachement
     attr_reader :url
+
     def initialize(url)
       @url = url
     end
@@ -31,7 +32,7 @@ module Miteru
       {
         type: "button",
         text: "Lookup on VirusTotal",
-        url: _vt_link,
+        url: _vt_link
       }
     end
 
@@ -41,7 +42,7 @@ module Miteru
       {
         type: "button",
         text: "Lookup on urlscan.io",
-        url: _urlscan_link,
+        url: _urlscan_link
       }
     end
 

data/lib/miteru/cli.rb

@@ -5,11 +5,11 @@ require "thor"
 module Miteru
   class CLI < Thor
     method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of phishing kits"
-    method_option :ayashige, type: :boolean, default: false, desc: "Enable or disable ayashige(ninoseki/ayashige) feed"
+    method_option :ayashige, type: :boolean, default: false, desc: "Enable or disable Ayashige(ninoseki/ayashige) feed"
     method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
-    method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
-    method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
-    method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 10,000)"
+    method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
+    method_option :post_to_slack, type: :boolean, default: false, desc: "Enable or disable Slack notification"
+    method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results to fetch. (Max: 10,000)"
     method_option :threads, type: :numeric, desc: "Number of threads to use"
     method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"
@@ -29,5 +29,13 @@ module Miteru
 
       Crawler.execute
     end
+
+    default_command :execute
+
+    class << self
+      def exit_on_failure?
+        true
+      end
+    end
   end
 end

data/lib/miteru/configuration.rb

@@ -28,8 +28,19 @@ module Miteru
     # @return [Boolean]
     attr_accessor :verbose
 
+    # @return [String]
+    attr_accessor :database
+
+    # @return [String, nil]
+    attr_accessor :slack_webhook_url
+
+    # @return [String]
+    attr_accessor :slack_channel
+
+    # @return [Array<String>]
     attr_reader :valid_extensions
 
+    # @return [Array<String>]
     attr_reader :valid_mime_types
 
     def initialize
@@ -41,6 +52,10 @@ module Miteru
       @size = 100
       @threads = Parallel.processor_count
       @verbose = false
+      @database = ENV["MITERU_DATABASE"] || "miteru.db"
+
+      @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
+      @slack_channel = ENV["SLACK_CHANNEL"] || "#general"
 
       @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
       @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
@@ -65,6 +80,10 @@ module Miteru
     def verbose?
       @verbose
     end
+
+    def slack_webhook_url?
+      @slack_webhook_url
+    end
   end
 
   class << self

data/lib/miteru/crawler.rb

@@ -6,8 +6,7 @@ require "uri"
 
 module Miteru
   class Crawler
-    attr_reader :downloader
-    attr_reader :feeds
+    attr_reader :downloader, :feeds
 
     def initialize
       @downloader = Downloader.new(Miteru.configuration.download_to)

data/lib/miteru/database.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+class InitialSchema < ActiveRecord::Migration[6.1]
+  def change
+    create_table :records, if_not_exists: true do |t|
+      t.string :hash, null: false, index: { unique: true }
+      t.string :hostname, null: false
+      t.json :headers, null: false
+      t.text :filename, null: false
+      t.string :downloaded_as, null: false
+      t.integer :filesize, null: false
+      t.string :mime_type, null: false
+      t.text :url, null: false
+
+      t.timestamps
+    end
+  end
+end
+
+def adapter
+  return "postgresql" if Miteru.configuration.database.start_with?("postgresql://", "postgres://")
+  return "mysql2" if Miteru.configuration.database.start_with?("mysql2://")
+
+  "sqlite3"
+end
+
+module Miteru
+  class Database
+    class << self
+      def connect
+        case adapter
+        when "postgresql", "mysql2"
+          ActiveRecord::Base.establish_connection(Miteru.configuration.database)
+        else
+          ActiveRecord::Base.establish_connection(
+            adapter: adapter,
+            database: Miteru.configuration.database
+          )
+        end
+
+        # ActiveRecord::Base.logger = Logger.new STDOUT
+        ActiveRecord::Migration.verbose = false
+
+        InitialSchema.migrate(:up)
+      rescue StandardError => _e
+        # Do nothing
+      end
+
+      def close
+        ActiveRecord::Base.clear_active_connections!
+        ActiveRecord::Base.connection.close
+      end
+
+      def destroy!
+        return unless ActiveRecord::Base.connected?
+
+        InitialSchema.migrate(:down)
+      end
+    end
+  end
+end
+
+Miteru::Database.connect

data/lib/miteru/downloader.rb

@@ -6,13 +6,12 @@ require "uri"
 
 module Miteru
   class Downloader
-    attr_reader :base_dir
-    attr_reader :memo
+    attr_reader :base_dir, :memo
 
     def initialize(base_dir = "/tmp")
       @base_dir = base_dir
       @memo = {}
-      raise ArgumentError, "#{base_dir} is not exist." unless Dir.exist?(base_dir)
+      raise ArgumentError, "#{base_dir} doesn't exist." unless Dir.exist?(base_dir)
     end
 
     def download_kits(kits)
@@ -22,23 +21,29 @@ module Miteru
     private
 
     def download_kit(kit)
-      destination = kit.download_filepath
+      destination = kit.filepath_to_download
+
       begin
-        downloaded_filepath = HTTPClient.download(kit.url, destination)
-        hash = sha256(downloaded_filepath)
-        if duplicated?(hash)
-          puts "Do not download #{kit.url} because there is a duplicate file in the directory (SHA256: #{hash})."
-          FileUtils.rm downloaded_filepath
-        else
-          puts "Download #{kit.url} as #{downloaded_filepath}"
-        end
+        downloaded_as = HTTPClient.download(kit.url, destination)
       rescue Down::Error => e
         puts "Failed to download: #{kit.url} (#{e})"
+        return
       end
-    end
 
-    def filepath_to_download(filename)
-      "#{base_dir}/#{filename}"
+      hash = sha256(downloaded_as)
+
+      ActiveRecord::Base.connection_pool.with_connection do
+        # Remove a downloaded file if it is not unique
+        unless Record.unique_hash?(hash)
+          puts "Don't download #{kit.url}. The same hash is already recorded. (SHA256: #{hash})."
+          FileUtils.rm downloaded_as
+          return
+        end
+
+        # Record a kit in DB
+        Record.create_by_kit_and_hash(kit, hash)
+        puts "Download #{kit.url} as #{downloaded_as}"
+      end
     end
 
     def sha256(path)
@@ -49,13 +54,5 @@ module Miteru
       memo[path] = hash
       hash
     end
-
-    def sha256s
-      Dir.glob("#{base_dir}/*.{zip,rar,7z,tar,gz}").map { |path| sha256(path) }
-    end
-
-    def duplicated?(hash)
-      sha256s.count(hash) > 1
-    end
   end
 end