miteru 0.14.7 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1ae4b442c2963ff387cc1bf3bf6390a393c2f8ba416276f2dc0cc48f2ceb86b
-  data.tar.gz: ce3baa2e515837cf722a4bd90650f558a337cde2e2dd2ecef976620f7d20eddb
+  metadata.gz: 050c27599e75745a7c215f08b7ed190b43c70388d974a68945702eefdb25b7c2
+  data.tar.gz: 64c7429a4178febf6984fe3d79d3970781634971405c5d9fdb0748c61b663a32
 SHA512:
-  metadata.gz: 40afc8ff440ffad5be4e4b7efdb7670c6df1a9e05d503930424e53a6c57fcfd3270239f39a92de2ccac149e8bc6175c87833183ad822d900d526490f067c06dc
-  data.tar.gz: 2ad37a6b2ebfaf78451ba35fa70d826ad95202cf077fdba5ed02cbebd8c74104c6f7f2e49da1ea26a9d42890bbd0b4d7b76e06787cab3697ba43f441a5825230
+  metadata.gz: ff780a0db3fafdded94261c38272dc0537462ae195f646cceafc2223cd62e61fb8d809ee172f953344314a93f9e6f746b9d3b6b66efdf9b48cda2c5d8c645eb7
+  data.tar.gz: c55dfad5120175ebf43bd01a30b42517a9a5bfd6fddd12d8c6ad07acd3ae02bba67a1bc9101d26fab74406f374f16fbeb3f51e791c678dce749a403e9477f82a

data/.github/workflows/test.yml

@@ -0,0 +1,68 @@
+name: Ruby CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_USER: mysql
+          MYSQL_PASSWORD: mysql
+          MYSQL_DATABASE: test
+          MYSQL_ROOT_PASSWORD: rootpassword
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, "3.0"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get -yqq install libpq-dev libmysqlclient-dev
+          gem install bundler
+          bundle install
+
+      - name: Test with PostgreSQL
+        env:
+          MITERU_DATABASE: postgresql://postgres:postgres@localhost:5432/test
+        run: |
+          bundle exec rake
+
+      - name: Test with MySQL
+        env:
+          MITERU_DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
+        run: |
+          bundle exec rake

data/.gitignore

@@ -51,3 +51,6 @@ Gemfile.lock
 
 ## RSpec
 .rspec_status
+
+# SQLite database
+*.db

data/.overcommit.yml

@@ -0,0 +1,12 @@
+PreCommit:
+  BundleCheck:
+    enabled: true
+
+  RuboCop:
+    enabled: true
+    required_executable: bundle
+    command: ["bundle", "exec", "standardrb"]
+    on_warn: fail
+
+  YamlSyntax:
+    enabled: true

data/.standard.yml

@@ -0,0 +1,4 @@
+ignore:
+  - "**/*":
+      - Layout/SpaceInsideHashLiteralBraces
+      - Style/RescueStandardError

data/README.md

@@ -1,109 +1,40 @@
 # Miteru
 
 [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
-[![Build Status](https://travis-ci.com/ninoseki/miteru.svg?branch=master)](https://travis-ci.com/ninoseki/miteru)
-[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)](https://hub.docker.com/repository/docker/ninoseki/miteru)
+[![Ruby CI](https://github.com/ninoseki/miteru/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/miteru/actions/workflows/test.yml)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
 
 Miteru is an experimental phishing kit detection tool.
 
+## Disclaimer
+
+This tool is for research purposes only. The use of this tool is your responsibility.
+I take no responsibility and/or liability for how you choose to use this tool.
+
 ## How it works
 
 - It collects phishy URLs from the following feeds:
-  - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#certstream-suspicious)
-  - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
-  - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
-  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
+  - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#task.source%3Acertstream-suspicious)
+  - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#task.source%3Aopenphish)
+  - [PhishTank feed via urlscan.io](https://urlscan.io/search/#task.source%3Aphishtank)
+  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#task.source%3Aurlhaus)
   - urlscan.io phish feed (available for Pro users)
   - [Ayashige feed](https://github.com/ninoseki/ayashige)
   - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
   - [PhishStats feed](https://phishstats.info/)
 - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
-  - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
+  - Note: Supported compressed files are: `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 
 ## Features
 
-- [x] Phishing kit detection & collection.
-- [x] Slack notification.
-- [x] Threading.
-
-## Installation
-
-```bash
-gem install miteru
-```
-
-## Usage
-
-```bash
-$ miteru
-Commands:
-  miteru execute         # Execute the crawler
-  miteru help [COMMAND]  # Describe available commands or one specific command
-```
-
-```bash
-$ miteru help execute
-Usage:
-  miteru execute
-
-Options:
-  [--auto-download], [--no-auto-download]              # Enable or disable auto-download of phishing kits
-  [--ayashige], [--no-ayashige]                        # Enable or disable ayashige(ninoseki/ayashige) feed
-  [--directory-traveling], [--no-directory-traveling]  # Enable or disable directory traveling
-  [--download-to=DOWNLOAD_TO]                          # Directory to download file(s)
-                                                       # Default: /tmp
-  [--post-to-slack], [--no-post-to-slack]              # Post a message to Slack if it detects a phishing kit
-  [--size=N]                                           # Number of urlscan.io's results. (Max: 10,000)
-                                                       # Default: 100
-  [--threads=N]                                        # Number of threads to use
-  [--verbose], [--no-verbose]
-                                                       # Default: true
-
-Execute the crawler
-```
-
-```bash
-$ miteru execute
-...
-https://dummy1.com: it doesn't contain a phishing kit.
-https://dummy2.com: it doesn't contain a phishing kit.
-https://dummy3.com: it doesn't contain a phishing kit.
-https://dummy4.com: it might contain a phishing kit (dummy.zip).
-```
-
-## Using Docker (alternative if you don't install Ruby)
-
-```bash
-$ docker pull ninoseki/miteru
-# ex. auto-download detected phishing kit(s) into host machines's /tmp directory
-$ docker run --rm -v /tmp:/tmp ninoseki/miteru execute --auto-download
-```
-
-## Configuration
-
-For using `--post-to-slack` feature, you should set the following environment variables:
-
-- `SLACK_WEBHOOK_URL`: Your Slack Webhook URL.
-- `SLACK_CHANNEL`: Slack channel to post a message (default: "#general").
-
-If you are a urlscan.io Pro user, set your API key as an environment variable `URLSCAN_API_KEY`.
-
-It enables you to subscribe the urlscan.io phish feed.
-
-## Examples
-
-### Aasciinema cast
-
-[![asciicast](https://asciinema.org/a/hHpkHhMLiiv17gmdRhVMtZWwM.svg)](https://asciinema.org/a/hHpkHhMLiiv17gmdRhVMtZWwM)
-
-### Slack notification
-
-![img](./screenshots/slack.png)
+- [x] Phishing kit detection & collection
+- [x] Slack notification
+- [x] Threading
 
-## Alternatives
+## Docs
 
-- [t4d/StalkPhish](https://github.com/t4d/StalkPhish): The Phishing kits stalker, harvesting phishing kits for investigations.
-- [duo-labs/phish-collect](https://github.com/duo-labs/phish-collect): Python script to hunt phishing kits.
-- [leunammejii/analyst_arsenal](https://github.com/leunammejii/analyst_arsenal): A tool belt for analysts to continue fighting the good fight.
+- [Requirements & Installation](https://github.com/ninoseki/miteru/wiki/Requirements-&-Installation)
+- [Usage](https://github.com/ninoseki/miteru/wiki/Usage)
+- [Configuration](https://github.com/ninoseki/miteru/wiki/Configuration)
+- [Alternatives](https://github.com/ninoseki/miteru/wiki/Alternatives)

data/docker/Dockerfile

@@ -1,10 +1,13 @@
-FROM ruby:2.7-alpine3.10
-RUN apk --no-cache add git build-base ruby-dev \
+FROM ruby:3-alpine3.13
+
+RUN apk --no-cache add git build-base ruby-dev mysql-client mysql-dev sqlite-dev postgresql-client postgresql-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/miteru.git \
   && cd miteru \
   && gem build miteru.gemspec -o miteru.gem \
   && gem install miteru.gem \
+  && gem install mysql2 \
+  && gem install pg \
   && rm -rf /tmp/miteru \
   && apk del --purge git build-base ruby-dev
 

data/lib/miteru/attachement.rb

@@ -5,6 +5,7 @@ require "uri"
 module Miteru
   class Attachement
     attr_reader :url
+
     def initialize(url)
       @url = url
     end
@@ -31,7 +32,7 @@ module Miteru
       {
         type: "button",
         text: "Lookup on VirusTotal",
-        url: _vt_link,
+        url: _vt_link
       }
     end
 
@@ -41,7 +42,7 @@ module Miteru
       {
         type: "button",
         text: "Lookup on urlscan.io",
-        url: _urlscan_link,
+        url: _urlscan_link
       }
     end
 

data/lib/miteru/cli.rb

@@ -5,11 +5,11 @@ require "thor"
 module Miteru
   class CLI < Thor
     method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of phishing kits"
-    method_option :ayashige, type: :boolean, default: false, desc: "Enable or disable ayashige(ninoseki/ayashige) feed"
+    method_option :ayashige, type: :boolean, default: false, desc: "Enable or disable Ayashige(ninoseki/ayashige) feed"
     method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
-    method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
-    method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
-    method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 10,000)"
+    method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
+    method_option :post_to_slack, type: :boolean, default: false, desc: "Enable or disable Slack notification"
+    method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results to fetch. (Max: 10,000)"
     method_option :threads, type: :numeric, desc: "Number of threads to use"
     method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"

data/lib/miteru/configuration.rb

@@ -28,8 +28,19 @@ module Miteru
     # @return [Boolean]
     attr_accessor :verbose
 
+    # @return [String]
+    attr_accessor :database
+
+    # @return [String, nil]
+    attr_accessor :slack_webhook_url
+
+    # @return [String]
+    attr_accessor :slack_channel
+
+    # @return [Array<String>]
     attr_reader :valid_extensions
 
+    # @return [Array<String>]
     attr_reader :valid_mime_types
 
     def initialize
@@ -41,6 +52,10 @@ module Miteru
       @size = 100
       @threads = Parallel.processor_count
       @verbose = false
+      @database = ENV["MITERU_DATABASE"] || "miteru.db"
+
+      @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
+      @slack_channel = ENV["SLACK_CHANNEL"] || "#general"
 
       @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
       @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
@@ -65,6 +80,10 @@ module Miteru
     def verbose?
       @verbose
     end
+
+    def slack_webhook_url?
+      @slack_webhook_url
+    end
   end
 
   class << self

data/lib/miteru/crawler.rb

@@ -6,8 +6,7 @@ require "uri"
 
 module Miteru
   class Crawler
-    attr_reader :downloader
-    attr_reader :feeds
+    attr_reader :downloader, :feeds
 
     def initialize
       @downloader = Downloader.new(Miteru.configuration.download_to)
@@ -15,8 +14,8 @@ module Miteru
       @notifier = Notifier.new
     end
 
-    def crawl(url)
-      website = Website.new(url)
+    def crawl(entry)
+      website = Website.new(entry.url, entry.source)
       downloader.download_kits(website.kits) if website.has_kits? && auto_download?
       notify(website) if website.has_kits? || verbose?
     rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
@@ -24,11 +23,11 @@ module Miteru
     end
 
     def execute
-      suspicious_urls = feeds.suspicious_urls
-      puts "Loaded #{suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
+      suspicious_entries = feeds.suspicious_entries
+      puts "Loaded #{suspicious_entries.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
 
-      Parallel.each(suspicious_urls, in_threads: threads) do |url|
-        crawl url
+      Parallel.each(suspicious_entries, in_threads: threads) do |entry|
+        crawl entry
       end
     end
 

data/lib/miteru/database.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+class InitialSchema < ActiveRecord::Migration[6.1]
+  def change
+    create_table :records, if_not_exists: true do |t|
+      t.string :hash, null: false, index: { unique: true }
+      t.string :hostname, null: false
+      t.json :headers, null: false
+      t.text :filename, null: false
+      t.string :downloaded_as, null: false
+      t.integer :filesize, null: false
+      t.string :mime_type, null: false
+      t.text :url, null: false
+
+      t.timestamps
+    end
+  end
+end
+
+class V11Schema < ActiveRecord::Migration[6.1]
+  def change
+    add_column :records, :source, :string, if_not_exists: true
+  end
+end
+
+def adapter
+  return "postgresql" if Miteru.configuration.database.start_with?("postgresql://", "postgres://")
+  return "mysql2" if Miteru.configuration.database.start_with?("mysql2://")
+
+  "sqlite3"
+end
+
+module Miteru
+  class Database
+    class << self
+      def connect
+        case adapter
+        when "postgresql", "mysql2"
+          ActiveRecord::Base.establish_connection(Miteru.configuration.database)
+        else
+          ActiveRecord::Base.establish_connection(
+            adapter: adapter,
+            database: Miteru.configuration.database
+          )
+        end
+
+        # ActiveRecord::Base.logger = Logger.new STDOUT
+        ActiveRecord::Migration.verbose = false
+
+        InitialSchema.migrate(:up)
+        V11Schema.migrate(:up)
+      rescue StandardError => _e
+        # Do nothing
+      end
+
+      def close
+        ActiveRecord::Base.clear_active_connections!
+        ActiveRecord::Base.connection.close
+      end
+
+      def destroy!
+        return unless ActiveRecord::Base.connected?
+
+        InitialSchema.migrate(:down)
+        V11Schema.migrate(:down)
+      end
+    end
+  end
+end
+
+Miteru::Database.connect