miteru 0.14.6 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e33e427231064c47dd4e1358e71e009ebc9418af15350e1f9c7f833ea02205f
-  data.tar.gz: cf3e3f1608a6be63b7b5b93f1e12a8264935da1e059bfbda7437f91a8009bf63
+  metadata.gz: 2217ef5ac9ed2060203df9f5edebbac779fbebc53e4418f54de6d4b080ee81fc
+  data.tar.gz: 45c934a406c619b1d1cc9d03a00103ae2b732460ca70877953aaac2d36d3b0e0
 SHA512:
-  metadata.gz: e26651ad45bfc20c22b6584b3ea6f8419f99801f36e6673835034c4a77da3ea5ec2502273265fae8f89590551c4bd3eb42f33bc66a6fb3b873422b57e5136d40
-  data.tar.gz: f582e8422d5d33a97b8f9d3ae7aa58fe1346fc68284b867c0139f7bb908107b3b8eb8081f1bcdc3bbfb678427d318e6eaf269ce1d28647d317ec7dd8ed49a7d8
+  metadata.gz: 03ebfc42660abedfa61a7e931e0d863b03fc50d615cfe0c81db10c0473ae8e0908619c5bcbc656e48784d40b5ccc5b98d616efe467d2a68ecca1bb57415bacfb
+  data.tar.gz: a74accbf1d3949e8211792e02c68e6e6911acbe2cc0cd113459eb812bad29ae0dc4c434f56009a1e49fa255d58bfc5485b9cbe8e3d6e936fef68e23d671d7d86

data/.github/workflows/test.yml

@@ -0,0 +1,68 @@
+name: Ruby CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_USER: mysql
+          MYSQL_PASSWORD: mysql
+          MYSQL_DATABASE: test
+          MYSQL_ROOT_PASSWORD: rootpassword
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, "3.0"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get -yqq install libpq-dev libmysqlclient-dev
+          gem install bundler
+          bundle install
+
+      - name: Test with PostgreSQL
+        env:
+          MITERU_DATABASE: postgresql://postgres:postgres@localhost:5432/test
+        run: |
+          bundle exec rake
+
+      - name: Test with MySQL
+        env:
+          MITERU_DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
+        run: |
+          bundle exec rake

data/.gitignore

@@ -51,3 +51,6 @@ Gemfile.lock
 
 ## RSpec
 .rspec_status
+
+# SQLite database
+*.db

data/.overcommit.yml

@@ -0,0 +1,12 @@
+PreCommit:
+  BundleCheck:
+    enabled: true
+
+  RuboCop:
+    enabled: true
+    required_executable: bundle
+    command: ["bundle", "exec", "standardrb"]
+    on_warn: fail
+
+  YamlSyntax:
+    enabled: true

data/.standard.yml

@@ -0,0 +1,4 @@
+ignore:
+  - "**/*":
+      - Layout/SpaceInsideHashLiteralBraces
+      - Style/RescueStandardError

data/README.md

@@ -1,109 +1,40 @@
 # Miteru
 
 [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
-[![Build Status](https://travis-ci.com/ninoseki/miteru.svg?branch=master)](https://travis-ci.com/ninoseki/miteru)
-[![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)](https://hub.docker.com/repository/docker/ninoseki/miteru)
+[![Ruby CI](https://github.com/ninoseki/miteru/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/miteru/actions/workflows/test.yml)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
 
 Miteru is an experimental phishing kit detection tool.
 
+## Disclaimer
+
+This tool is for research purposes only. The use of this tool is your responsibility.
+I take no responsibility and/or liability for how you choose to use this tool.
+
 ## How it works
 
 - It collects phishy URLs from the following feeds:
-  - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#certstream-suspicious)
-  - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
-  - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
-  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
+  - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#task.source%3Acertstream-suspicious)
+  - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#task.source%3Aopenphish)
+  - [PhishTank feed via urlscan.io](https://urlscan.io/search/#task.source%3Aphishtank)
+  - [URLhaus feed via urlscan.io](https://urlscan.io/search/#task.source%3Aurlhaus)
   - urlscan.io phish feed (available for Pro users)
   - [Ayashige feed](https://github.com/ninoseki/ayashige)
   - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
   - [PhishStats feed](https://phishstats.info/)
 - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
-  - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
+  - Note: Supported compressed files are: `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
 
 ## Features
 
-- [x] Phishing kit detection & collection.
-- [x] Slack notification.
-- [x] Threading.
-
-## Installation
-
-```bash
-gem install miteru
-```
-
-## Usage
-
-```bash
-$ miteru
-Commands:
-  miteru execute         # Execute the crawler
-  miteru help [COMMAND]  # Describe available commands or one specific command
-```
-
-```bash
-$ miteru help execute
-Usage:
-  miteru execute
-
-Options:
-  [--auto-download], [--no-auto-download]              # Enable or disable auto-download of phishing kits
-  [--ayashige], [--no-ayashige]                        # Enable or disable ayashige(ninoseki/ayashige) feed
-  [--directory-traveling], [--no-directory-traveling]  # Enable or disable directory traveling
-  [--download-to=DOWNLOAD_TO]                          # Directory to download file(s)
-                                                       # Default: /tmp
-  [--post-to-slack], [--no-post-to-slack]              # Post a message to Slack if it detects a phishing kit
-  [--size=N]                                           # Number of urlscan.io's results. (Max: 10,000)
-                                                       # Default: 100
-  [--threads=N]                                        # Number of threads to use
-  [--verbose], [--no-verbose]
-                                                       # Default: true
-
-Execute the crawler
-```
-
-```bash
-$ miteru execute
-...
-https://dummy1.com: it doesn't contain a phishing kit.
-https://dummy2.com: it doesn't contain a phishing kit.
-https://dummy3.com: it doesn't contain a phishing kit.
-https://dummy4.com: it might contain a phishing kit (dummy.zip).
-```
-
-## Using Docker (alternative if you don't install Ruby)
-
-```bash
-$ docker pull ninoseki/miteru
-# ex. auto-download detected phishing kit(s) into host machines's /tmp directory
-$ docker run --rm -v /tmp:/tmp ninoseki/miteru execute --auto-download
-```
-
-## Configuration
-
-For using `--post-to-slack` feature, you should set the following environment variables:
-
-- `SLACK_WEBHOOK_URL`: Your Slack Webhook URL.
-- `SLACK_CHANNEL`: Slack channel to post a message (default: "#general").
-
-If you are a urlscan.io Pro user, set your API key as an environment variable `URLSCAN_API_KEY`.
-
-It enables you to subscribe the urlscan.io phish feed.
-
-## Examples
-
-### Aasciinema cast
-
-[![asciicast](https://asciinema.org/a/hHpkHhMLiiv17gmdRhVMtZWwM.svg)](https://asciinema.org/a/hHpkHhMLiiv17gmdRhVMtZWwM)
-
-### Slack notification
-
-![img](./screenshots/slack.png)
+- [x] Phishing kit detection & collection
+- [x] Slack notification
+- [x] Threading
 
-## Alternatives
+## Docs
 
-- [t4d/StalkPhish](https://github.com/t4d/StalkPhish): The Phishing kits stalker, harvesting phishing kits for investigations.
-- [duo-labs/phish-collect](https://github.com/duo-labs/phish-collect): Python script to hunt phishing kits.
-- [leunammejii/analyst_arsenal](https://github.com/leunammejii/analyst_arsenal): A tool belt for analysts to continue fighting the good fight.
+- [Requirements & Installation](https://github.com/ninoseki/miteru/wiki/Requirements-&-Installation)
+- [Usage](https://github.com/ninoseki/miteru/wiki/Usage)
+- [Configuration](https://github.com/ninoseki/miteru/wiki/Configuration)
+- [Alternatives](https://github.com/ninoseki/miteru/wiki/Alternatives)

data/docker/Dockerfile

@@ -1,10 +1,13 @@
-FROM ruby:2.7-alpine3.10
-RUN apk --no-cache add git build-base ruby-dev \
+FROM ruby:3-alpine3.13
+
+RUN apk --no-cache add git build-base ruby-dev mysql-client mysql-dev sqlite-dev postgresql-client postgresql-dev \
   && cd /tmp/ \
   && git clone https://github.com/ninoseki/miteru.git \
   && cd miteru \
   && gem build miteru.gemspec -o miteru.gem \
   && gem install miteru.gem \
+  && gem install mysql2 \
+  && gem install pg \
   && rm -rf /tmp/miteru \
   && apk del --purge git build-base ruby-dev
 

data/exe/miteru

@@ -5,4 +5,5 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "miteru"
 
-Miteru::CLI.start
+ARGV.unshift(Miteru::CLI.default_task) unless Miteru::CLI.all_tasks.key?(ARGV[0])
+Miteru::CLI.start(ARGV)

data/lib/miteru/attachement.rb

@@ -5,6 +5,7 @@ require "uri"
 module Miteru
   class Attachement
     attr_reader :url
+
     def initialize(url)
       @url = url
     end
@@ -31,7 +32,7 @@ module Miteru
       {
         type: "button",
         text: "Lookup on VirusTotal",
-        url: _vt_link,
+        url: _vt_link
       }
     end
 
@@ -41,7 +42,7 @@ module Miteru
       {
         type: "button",
         text: "Lookup on urlscan.io",
-        url: _urlscan_link,
+        url: _urlscan_link
       }
     end
 

data/lib/miteru/cli.rb

@@ -5,11 +5,11 @@ require "thor"
 module Miteru
   class CLI < Thor
     method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of phishing kits"
-    method_option :ayashige, type: :boolean, default: false, desc: "Enable or disable ayashige(ninoseki/ayashige) feed"
+    method_option :ayashige, type: :boolean, default: false, desc: "Enable or disable Ayashige(ninoseki/ayashige) feed"
     method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
-    method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
-    method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
-    method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 10,000)"
+    method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
+    method_option :post_to_slack, type: :boolean, default: false, desc: "Enable or disable Slack notification"
+    method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results to fetch. (Max: 10,000)"
     method_option :threads, type: :numeric, desc: "Number of threads to use"
     method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"
@@ -29,5 +29,13 @@ module Miteru
 
       Crawler.execute
     end
+
+    default_command :execute
+
+    class << self
+      def exit_on_failure?
+        true
+      end
+    end
   end
 end

data/lib/miteru/configuration.rb

@@ -28,8 +28,19 @@ module Miteru
     # @return [Boolean]
     attr_accessor :verbose
 
+    # @return [String]
+    attr_accessor :database
+
+    # @return [String, nil]
+    attr_accessor :slack_webhook_url
+
+    # @return [String]
+    attr_accessor :slack_channel
+
+    # @return [Array<String>]
     attr_reader :valid_extensions
 
+    # @return [Array<String>]
     attr_reader :valid_mime_types
 
     def initialize
@@ -41,6 +52,10 @@ module Miteru
       @size = 100
       @threads = Parallel.processor_count
       @verbose = false
+      @database = ENV["MITERU_DATABASE"] || "miteru.db"
+
+      @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
+      @slack_channel = ENV["SLACK_CHANNEL"] || "#general"
 
       @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
       @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
@@ -65,6 +80,10 @@ module Miteru
     def verbose?
       @verbose
     end
+
+    def slack_webhook_url?
+      @slack_webhook_url
+    end
   end
 
   class << self

data/lib/miteru/crawler.rb

@@ -6,8 +6,7 @@ require "uri"
 
 module Miteru
   class Crawler
-    attr_reader :downloader
-    attr_reader :feeds
+    attr_reader :downloader, :feeds
 
     def initialize
       @downloader = Downloader.new(Miteru.configuration.download_to)

data/lib/miteru/database.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+class InitialSchema < ActiveRecord::Migration[6.1]
+  def change
+    create_table :records, if_not_exists: true do |t|
+      t.string :hash, null: false, index: { unique: true }
+      t.string :hostname, null: false
+      t.json :headers, null: false
+      t.text :filename, null: false
+      t.string :downloaded_as, null: false
+      t.integer :filesize, null: false
+      t.string :mime_type, null: false
+      t.text :url, null: false
+
+      t.timestamps
+    end
+  end
+end
+
+def adapter
+  return "postgresql" if Miteru.configuration.database.start_with?("postgresql://", "postgres://")
+  return "mysql2" if Miteru.configuration.database.start_with?("mysql2://")
+
+  "sqlite3"
+end
+
+module Miteru
+  class Database
+    class << self
+      def connect
+        case adapter
+        when "postgresql", "mysql2"
+          ActiveRecord::Base.establish_connection(Miteru.configuration.database)
+        else
+          ActiveRecord::Base.establish_connection(
+            adapter: adapter,
+            database: Miteru.configuration.database
+          )
+        end
+
+        # ActiveRecord::Base.logger = Logger.new STDOUT
+        ActiveRecord::Migration.verbose = false
+
+        InitialSchema.migrate(:up)
+      rescue StandardError => _e
+        # Do nothing
+      end
+
+      def close
+        ActiveRecord::Base.clear_active_connections!
+        ActiveRecord::Base.connection.close
+      end
+
+      def destroy!
+        return unless ActiveRecord::Base.connected?
+
+        InitialSchema.migrate(:down)
+      end
+    end
+  end
+end
+
+Miteru::Database.connect

data/lib/miteru/downloader.rb

@@ -6,13 +6,12 @@ require "uri"
 
 module Miteru
   class Downloader
-    attr_reader :base_dir
-    attr_reader :memo
+    attr_reader :base_dir, :memo
 
     def initialize(base_dir = "/tmp")
       @base_dir = base_dir
       @memo = {}
-      raise ArgumentError, "#{base_dir} is not exist." unless Dir.exist?(base_dir)
+      raise ArgumentError, "#{base_dir} doesn't exist." unless Dir.exist?(base_dir)
     end
 
     def download_kits(kits)
@@ -22,23 +21,29 @@ module Miteru
     private
 
     def download_kit(kit)
-      destination = kit.download_filepath
+      destination = kit.filepath_to_download
+
       begin
-        downloaded_filepath = HTTPClient.download(kit.url, destination)
-        hash = sha256(downloaded_filepath)
-        if duplicated?(hash)
-          puts "Do not download #{kit.url} because there is a duplicate file in the directory (SHA256: #{hash})."
-          FileUtils.rm downloaded_filepath
-        else
-          puts "Download #{kit.url} as #{downloaded_filepath}"
-        end
+        downloaded_as = HTTPClient.download(kit.url, destination)
       rescue Down::Error => e
         puts "Failed to download: #{kit.url} (#{e})"
+        return
       end
-    end
 
-    def filepath_to_download(filename)
-      "#{base_dir}/#{filename}"
+      hash = sha256(downloaded_as)
+
+      ActiveRecord::Base.connection_pool.with_connection do
+        # Remove a downloaded file if it is not unique
+        unless Record.unique_hash?(hash)
+          puts "Don't download #{kit.url}. The same hash is already recorded. (SHA256: #{hash})."
+          FileUtils.rm downloaded_as
+          return
+        end
+
+        # Record a kit in DB
+        Record.create_by_kit_and_hash(kit, hash)
+        puts "Download #{kit.url} as #{downloaded_as}"
+      end
     end
 
     def sha256(path)
@@ -49,13 +54,5 @@ module Miteru
       memo[path] = hash
       hash
     end
-
-    def sha256s
-      Dir.glob("#{base_dir}/*.{zip,rar,7z,tar,gz}").map { |path| sha256(path) }
-    end
-
-    def duplicated?(hash)
-      sha256s.count(hash) > 1
-    end
   end
 end

data/lib/miteru/error.rb

@@ -2,5 +2,6 @@
 
 module Miteru
   class HTTPResponseError < StandardError; end
+
   class DownloadError < StandardError; end
 end

data/lib/miteru/feeds/ayashige.rb

@@ -10,10 +10,10 @@ module Miteru
       URL = "https://#{HOST}"
 
       def urls
-        url = url_for("/feed")
+        url = url_for("/api/v1/domains/")
         res = JSON.parse(get(url))
 
-        domains = res.map { |item| item["domain"] }
+        domains = res.map { |item| item["fqdn"] }
         domains.map do |domain|
           [
             "https://#{domain}",

data/lib/miteru/feeds/phishing_database.rb

@@ -6,7 +6,7 @@ require "uri"
 module Miteru
   class Feeds
     class PhishingDatabase < Feed
-      URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
+      URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE-NOW.txt"
 
       def urls
         body = get(URL)

data/lib/miteru/feeds/phishstats.rb

@@ -11,7 +11,7 @@ module Miteru
       def urls
         json = JSON.parse(get(URL))
         json.map do |entry|
-          entry.dig("url")
+          entry["url"]
         end
       rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
         puts "Failed to load PhishStats feed (#{e})"

data/lib/miteru/feeds/urlscan_pro.rb

@@ -27,7 +27,7 @@ module Miteru
 
         res = api.pro.phishfeed
         results = res["results"] || []
-        results.map { |result| result.dig("page_url") }
+        results.map { |result| result["page_url"] }
       rescue ArgumentError => _e
         []
       end

data/lib/miteru/feeds.rb

@@ -9,7 +9,7 @@ require_relative "./feeds/urlscan_pro"
 
 module Miteru
   class Feeds
-    IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
+    IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
 
     def initialize
       @feeds = [
@@ -48,7 +48,7 @@ module Miteru
       segments = uri.path.split("/")
       return [base] if segments.length.zero?
 
-      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
+      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
 
       urls.reject do |breakdowned_url|
         # Reject a url which ends with specific extension names

data/lib/miteru/http_client.rb

@@ -7,7 +7,7 @@ require "uri"
 module Miteru
   class HTTPClient
     DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
-    URLSCAN_UA = "miteru/#{Miteru::VERSION}"
+    URLSCAN_UA = "miteru/#{Miteru::VERSION}".freeze
 
     attr_reader :ssl_context
 
@@ -27,18 +27,18 @@ module Miteru
       options = options.merge default_options
 
       HTTP.follow
-          .timeout(3)
-          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
-          .head(url, options)
+        .timeout(3)
+        .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
+        .head(url, options)
     end
 
     def get(url, options = {})
       options = options.merge default_options
 
       HTTP.follow
-          .timeout(write: 2, connect: 5, read: 10)
-          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
-          .get(url, options)
+        .timeout(write: 2, connect: 5, read: 10)
+        .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
+        .get(url, options)
     end
 
     def post(url, options = {})

data/lib/miteru/kit.rb

@@ -1,18 +1,15 @@
 # frozen_string_literal: true
 
 require "cgi"
-require "securerandom"
+require "uuidtools"
+require "uri"
 
 module Miteru
   class Kit
     VALID_EXTENSIONS = Miteru.configuration.valid_extensions
     VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
 
-    attr_reader :url
-
-    attr_reader :status
-    attr_reader :content_length
-    attr_reader :mime_type
+    attr_reader :url, :status, :content_length, :mime_type, :headers
 
     def initialize(url)
       @url = url
@@ -20,9 +17,10 @@ module Miteru
       @content_length = nil
       @mime_type = nil
       @status = nil
+      @headers = nil
     end
 
-    def valid?;
+    def valid?
       # make a HEAD request for the validation
       before_validation
 
@@ -36,41 +34,50 @@ module Miteru
     end
 
     def basename
-      File.basename(url)
+      @basename ||= File.basename(url)
     end
 
     def filename
-      CGI.unescape basename
+      @filename ||= CGI.unescape(basename)
+    end
+
+    def filepath_to_download
+      "#{base_dir}/#{filename_to_download}"
     end
 
-    def download_filepath
-      "#{base_dir}/#{download_filename}"
+    def downloaded?
+      File.exist?(filepath_to_download)
     end
 
     def filesize
-      return nil unless File.exist?(download_filepath)
+      return nil unless downloaded?
 
-      File.size download_filepath
+      File.size filepath_to_download
     end
 
     def filename_with_size
       return filename unless filesize
 
-      "#{filename}(#{filesize / 1024}KB)"
+      kb = (filesize.to_f / 1024.0).ceil
+      "#{filename}(#{kb}KB)"
     end
 
-    private
-
     def id
-      @id ||= SecureRandom.hex(10)
+      @id ||= UUIDTools::UUID.random_create.to_s
     end
 
     def hostname
-      URI(url).hostname
+      @hostname ||= URI(url).hostname
     end
 
-    def download_filename
-      "#{hostname}_#{filename}_#{id}#{extname}"
+    def decoded_url
+      @decoded_url ||= URI.decode_www_form_component(url)
+    end
+
+    private
+
+    def filename_to_download
+      "#{id}#{extname}"
     end
 
     def base_dir
@@ -86,6 +93,7 @@ module Miteru
       @content_length = res.content_length
       @mime_type = res.content_type.mime_type.to_s
       @status = res.status
+      @headers = res.headers.to_h
     rescue StandardError
       # do nothing
     end

data/lib/miteru/notifier.rb

@@ -7,31 +7,19 @@ module Miteru
   class Notifier
     def notify(url:, kits:, message:)
       attachement = Attachement.new(url)
-      kits = kits.select(&:filesize)
+      kits = kits.select(&:downloaded?)
 
-      if post_to_slack? && kits.any?
-        notifier = Slack::Notifier.new(slack_webhook_url, channel: slack_channel)
-        notifier.post(text: message, attachments: attachement.to_a)
+      if notifiable? && kits.any?
+        notifier = Slack::Notifier.new(Miteru.configuration.slack_webhook_url, channel: Miteru.configuration.slack_channel)
+        notifier.post(text: message.capitalize, attachments: attachement.to_a)
       end
 
       message = message.colorize(:light_red) if kits.any?
       puts "#{url}: #{message}"
     end
 
-    def post_to_slack?
-      slack_webhook_url? && Miteru.configuration.post_to_slack?
-    end
-
-    def slack_webhook_url
-      ENV.fetch "SLACK_WEBHOOK_URL"
-    end
-
-    def slack_channel
-      ENV.fetch "SLACK_CHANNEL", "#general"
-    end
-
-    def slack_webhook_url?
-      ENV.key? "SLACK_WEBHOOK_URL"
+    def notifiable?
+      Miteru.configuration.slack_webhook_url? && Miteru.configuration.post_to_slack?
     end
   end
 end

data/lib/miteru/record.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module Miteru
+  class Record < ActiveRecord::Base
+    class << self
+      #
+      # Check uniqueness of a record by a hash
+      #
+      # @param [String] hash
+      #
+      # @return [Boolean] true if it is unique. Otherwise false.
+      #
+      def unique_hash?(hash)
+        record = find_by(hash: hash)
+        return true if record.nil?
+
+        false
+      end
+
+      #
+      # Create a new record based on a kit
+      #
+      # @param [Miteru::Kit] kit
+      # @param [String] hash
+      #
+      # @return [Miteru::Record]
+      #
+      def create_by_kit_and_hash(kit, hash)
+        record = new(
+          hash: hash,
+          hostname: kit.hostname,
+          url: kit.decoded_url,
+          headers: kit.headers,
+          filename: kit.filename,
+          filesize: kit.filesize,
+          mime_type: kit.mime_type,
+          downloaded_as: kit.filepath_to_download
+        )
+        record.save
+        record
+      rescue TypeError, ActiveRecord::RecordNotUnique => _e
+        nil
+      end
+    end
+  end
+end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.14.6"
+  VERSION = "1.0.2"
 end

data/lib/miteru/website.rb

@@ -17,10 +17,10 @@ module Miteru
     end
 
     def kits
-      @kits ||= links.map do |link|
+      @kits ||= links.filter_map do |link|
         kit = Kit.new(link)
         kit.valid? ? kit : nil
-      end.compact
+      end
     end
 
     def ok?
@@ -42,11 +42,11 @@ module Miteru
     end
 
     def message
-      return "It doesn't contain a phishing kit." unless kits?
+      return "it doesn't contain a phishing kit." unless kits?
 
       filename_with_sizes = kits.map(&:filename_with_size).join(", ")
       noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
-      "It might contain #{noun}: #{filename_with_sizes}."
+      "it might contain #{noun}: #{filename_with_sizes}."
     end
 
     def links
@@ -75,7 +75,7 @@ module Miteru
 
     def href_links
       if doc && ok? && index?
-        doc.css("a").map { |a| a.get("href") }.compact.map do |href|
+        doc.css("a").filter_map { |a| a.get("href") }.map do |href|
           href = href.start_with?("/") ? href : "/#{href}"
           url + href
         end

data/lib/miteru.rb

@@ -3,6 +3,10 @@
 require "miteru/version"
 
 require "miteru/configuration"
+require "miteru/database"
+
+require "miteru/record"
+
 require "miteru/error"
 require "miteru/http_client"
 require "miteru/kit"
@@ -14,6 +18,4 @@ require "miteru/notifier"
 require "miteru/crawler"
 require "miteru/cli"
 
-module Miteru
-  # Your code goes here...
-end
+module Miteru; end

data/miteru.gemspec

@@ -1,43 +1,51 @@
 # frozen_string_literal: true
 
-lib = File.expand_path('lib', __dir__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "miteru/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "miteru"
-  spec.version       = Miteru::VERSION
-  spec.authors       = ["Manabu Niseki"]
-  spec.email         = ["manabu.niseki@gmail.com"]
+  spec.name = "miteru"
+  spec.version = Miteru::VERSION
+  spec.authors = ["Manabu Niseki"]
+  spec.email = ["manabu.niseki@gmail.com"]
 
-  spec.summary       = "An experimental phishing kit detector"
-  spec.description   = "An experimental phishing kit detector"
-  spec.homepage      = "https://github.com/ninoseki/miteru"
-  spec.license       = "MIT"
+  spec.summary = "An experimental phishing kit detector"
+  spec.description = "An experimental phishing kit detector"
+  spec.homepage = "https://github.com/ninoseki/miteru"
+  spec.license = "MIT"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.1"
-  spec.add_development_dependency "coveralls", "~> 0.8"
+  spec.add_development_dependency "bundler", "~> 2.2"
+  spec.add_development_dependency "coveralls_reborn", "~> 0.22"
   spec.add_development_dependency "glint", "~> 0.1"
+  spec.add_development_dependency "mysql2", "~> 0.5"
+  spec.add_development_dependency "overcommit", "~> 0.58"
+  spec.add_development_dependency "pg", "~> 1.2"
   spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.9"
+  spec.add_development_dependency "rspec", "~> 3.10"
+  spec.add_development_dependency "standard", "~> 1.3"
   spec.add_development_dependency "vcr", "~> 6.0"
-  spec.add_development_dependency "webmock", "~> 3.9"
+  spec.add_development_dependency "webmock", "~> 3.14"
+  spec.add_development_dependency "webrick", "~> 1.7.0"
 
+  spec.add_dependency "activerecord", "~> 6.1"
   spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "down", "~> 5.2"
-  spec.add_dependency "http", "~> 4.4"
+  spec.add_dependency "http", "~> 5.0"
   spec.add_dependency "oga", "~> 3.3"
-  spec.add_dependency "parallel", "~> 1.19"
-  spec.add_dependency "slack-notifier", "~> 2.3"
-  spec.add_dependency "thor", "~> 1.0"
-  spec.add_dependency "urlscan", "~> 0.6"
+  spec.add_dependency "parallel", "~> 1.20"
+  spec.add_dependency "slack-notifier", "~> 2.4"
+  spec.add_dependency "sqlite3", "~> 1.4"
+  spec.add_dependency "thor", "~> 1.1"
+  spec.add_dependency "urlscan", "~> 0.7"
+  spec.add_dependency "uuidtools", "~> 2.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.14.6
+  version: 1.0.2
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-04 00:00:00.000000000 Z
+date: 2021-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: coveralls_reborn
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.22'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.22'
 - !ruby/object:Gem::Dependency
   name: glint
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: overcommit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.58'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.58'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +114,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +156,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.14'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: 1.7.0
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +226,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.4'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.4'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: oga
   requirement: !ruby/object:Gem::Requirement
@@ -170,56 +254,84 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '1.20'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '1.20'
 - !ruby/object:Gem::Dependency
   name: slack-notifier
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: urlscan
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '2.2'
 description: An experimental phishing kit detector
 email:
 - manabu.niseki@gmail.com
@@ -228,9 +340,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
+- ".overcommit.yml"
 - ".rspec"
-- ".travis.yml"
+- ".standard.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -244,6 +358,7 @@ files:
 - lib/miteru/cli.rb
 - lib/miteru/configuration.rb
 - lib/miteru/crawler.rb
+- lib/miteru/database.rb
 - lib/miteru/downloader.rb
 - lib/miteru/error.rb
 - lib/miteru/feeds.rb
@@ -256,6 +371,7 @@ files:
 - lib/miteru/http_client.rb
 - lib/miteru/kit.rb
 - lib/miteru/notifier.rb
+- lib/miteru/record.rb
 - lib/miteru/version.rb
 - lib/miteru/website.rb
 - miteru.gemspec
@@ -280,7 +396,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: An experimental phishing kit detector

data/.travis.yml

@@ -1,7 +0,0 @@
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.6
-  - 2.7
-before_install: gem install bundler -v 2.1