miteru 0.14.4 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 89102143e9e405545e94b50da7b81b90c85ae9612c8c914d4b50b6d09ca2e023
4
- data.tar.gz: 2d6e60e6301425b9ceb8ed7cf103caab144f2ddab4fdd3872ef3f332a6667e37
3
+ metadata.gz: a7b9163d8f3fc3beea8d97a85f26d08c83a9f649c3484511fc8a1a6e42e0d553
4
+ data.tar.gz: b341ec7ece5359f0358853e48a8bdd84e51d3b0fd0fc1e8e484634bd4652b276
5
5
  SHA512:
6
- metadata.gz: 3df145d910400efc069d07c718943560a064b5d9fff849fb695795bf49b1d30681e6c31946172f2cd0e7508ca46ded7ff93d2935e48fc6dc970e1817d6e6348c
7
- data.tar.gz: 70057c7c2451f4631bfda279431e40dfc86bc4931caa555141e607d0a2d73e5308bd0458ddc9046b1e0ccbe0329829ab8568202e8671c68146e5b6b4b6688705
6
+ metadata.gz: b532eaa3a58e0918695fb19a2ff56096ab80b98858466bd7efff53b28afe2aa57a9a70bf7f4043941e996be8b4afdd62119ab52ec61a363513f0612927f92bde
7
+ data.tar.gz: 13fe4fb667d5c7f270508720f61ae579cbf3d08e3fbc8d7bdabb060a7421c008e727d99101bcd09b0a7d077c2f32355ea8a5fc017a6b3b5e14f098000850512f
@@ -0,0 +1,68 @@
1
+ name: Ruby CI
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ build:
7
+ runs-on: ubuntu-latest
8
+
9
+ services:
10
+ postgres:
11
+ image: postgres:12
12
+ env:
13
+ POSTGRES_USER: postgres
14
+ POSTGRES_PASSWORD: postgres
15
+ POSTGRES_DB: test
16
+ options: >-
17
+ --health-cmd pg_isready
18
+ --health-interval 10s
19
+ --health-timeout 5s
20
+ --health-retries 5
21
+ ports:
22
+ - 5432:5432
23
+
24
+ mysql:
25
+ image: mysql:8.0
26
+ env:
27
+ MYSQL_USER: mysql
28
+ MYSQL_PASSWORD: mysql
29
+ MYSQL_DATABASE: test
30
+ MYSQL_ROOT_PASSWORD: rootpassword
31
+ ports:
32
+ - 3306:3306
33
+ options: >-
34
+ --health-cmd="mysqladmin ping"
35
+ --health-interval=10s
36
+ --health-timeout=5s
37
+ --health-retries=3
38
+
39
+ strategy:
40
+ fail-fast: false
41
+ matrix:
42
+ ruby: [2.7, "3.0"]
43
+
44
+ steps:
45
+ - uses: actions/checkout@v2
46
+ - name: Set up Ruby
47
+ uses: ruby/setup-ruby@v1
48
+ with:
49
+ ruby-version: ${{ matrix.ruby }}
50
+ bundler-cache: true
51
+
52
+ - name: Install dependencies
53
+ run: |
54
+ sudo apt-get -yqq install libpq-dev libmysqlclient-dev
55
+ gem install bundler
56
+ bundle install
57
+
58
+ - name: Test with PostgreSQL
59
+ env:
60
+ MITERU_DATABASE: postgresql://postgres:postgres@localhost:5432/test
61
+ run: |
62
+ bundle exec rake
63
+
64
+ - name: Test with MySQL
65
+ env:
66
+ MITERU_DATABASE: mysql2://mysql:mysql@127.0.0.1:3306/test
67
+ run: |
68
+ bundle exec rake
data/.gitignore CHANGED
@@ -51,3 +51,6 @@ Gemfile.lock
51
51
 
52
52
  ## RSpec
53
53
  .rspec_status
54
+
55
+ # SQLite database
56
+ *.db
data/.overcommit.yml ADDED
@@ -0,0 +1,12 @@
1
+ PreCommit:
2
+ BundleCheck:
3
+ enabled: true
4
+
5
+ RuboCop:
6
+ enabled: true
7
+ required_executable: bundle
8
+ command: ["bundle", "exec", "standardrb"]
9
+ on_warn: fail
10
+
11
+ YamlSyntax:
12
+ enabled: true
data/.standard.yml ADDED
@@ -0,0 +1,4 @@
1
+ ignore:
2
+ - "**/*":
3
+ - Layout/SpaceInsideHashLiteralBraces
4
+ - Style/RescueStandardError
data/README.md CHANGED
@@ -1,109 +1,40 @@
1
1
  # Miteru
2
2
 
3
3
  [![Gem Version](https://badge.fury.io/rb/miteru.svg)](https://badge.fury.io/rb/miteru)
4
- [![Build Status](https://travis-ci.com/ninoseki/miteru.svg?branch=master)](https://travis-ci.com/ninoseki/miteru)
5
- [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/ninoseki/miteru)](https://hub.docker.com/repository/docker/ninoseki/miteru)
4
+ [![Ruby CI](https://github.com/ninoseki/miteru/actions/workflows/test.yml/badge.svg)](https://github.com/ninoseki/miteru/actions/workflows/test.yml)
6
5
  [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/miteru/badge)](https://www.codefactor.io/repository/github/ninoseki/miteru)
7
6
  [![Coverage Status](https://coveralls.io/repos/github/ninoseki/miteru/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/miteru?branch=master)
8
7
 
9
8
  Miteru is an experimental phishing kit detection tool.
10
9
 
10
+ ## Disclaimer
11
+
12
+ This tool is for research purposes only. The use of this tool is your responsibility.
13
+ I take no responsibility and/or liability for how you choose to use this tool.
14
+
11
15
  ## How it works
12
16
 
13
17
  - It collects phishy URLs from the following feeds:
14
- - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#certstream-suspicious)
15
- - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#OpenPhish)
16
- - [PhishTank feed via urlscan.io](https://urlscan.io/search/#PhishTank)
17
- - [URLhaus feed via urlscan.io](https://urlscan.io/search/#URLHaus)
18
+ - [CertStream-Suspicious feed via urlscan.io](https://urlscan.io/search/#task.source%3Acertstream-suspicious)
19
+ - [OpenPhish feed via urlscan.io](https://urlscan.io/search/#task.source%3Aopenphish)
20
+ - [PhishTank feed via urlscan.io](https://urlscan.io/search/#task.source%3Aphishtank)
21
+ - [URLhaus feed via urlscan.io](https://urlscan.io/search/#task.source%3Aurlhaus)
18
22
  - urlscan.io phish feed (available for Pro users)
19
23
  - [Ayashige feed](https://github.com/ninoseki/ayashige)
20
24
  - [Phishing Database feed](https://github.com/mitchellkrogza/Phishing.Database)
21
25
  - [PhishStats feed](https://phishstats.info/)
22
26
  - It checks each phishy URL whether it enables directory listing and contains a phishing kit (compressed file) or not.
23
- - Note: compressed file = `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
27
+ - Note: Supported compressed files are: `*.zip`, `*.rar`, `*.7z`, `*.tar` and `*.gz`.
24
28
 
25
29
  ## Features
26
30
 
27
- - [x] Phishing kit detection & collection.
28
- - [x] Slack notification.
29
- - [x] Threading.
30
-
31
- ## Installation
32
-
33
- ```bash
34
- gem install miteru
35
- ```
36
-
37
- ## Usage
38
-
39
- ```bash
40
- $ miteru
41
- Commands:
42
- miteru execute # Execute the crawler
43
- miteru help [COMMAND] # Describe available commands or one specific command
44
- ```
45
-
46
- ```bash
47
- $ miteru help execute
48
- Usage:
49
- miteru execute
50
-
51
- Options:
52
- [--auto-download], [--no-auto-download] # Enable or disable auto-download of phishing kits
53
- [--ayashige], [--no-ayashige] # Enable or disable ayashige(ninoseki/ayashige) feed
54
- [--directory-traveling], [--no-directory-traveling] # Enable or disable directory traveling
55
- [--download-to=DOWNLOAD_TO] # Directory to download file(s)
56
- # Default: /tmp
57
- [--post-to-slack], [--no-post-to-slack] # Post a message to Slack if it detects a phishing kit
58
- [--size=N] # Number of urlscan.io's results. (Max: 10,000)
59
- # Default: 100
60
- [--threads=N] # Number of threads to use
61
- [--verbose], [--no-verbose]
62
- # Default: true
63
-
64
- Execute the crawler
65
- ```
66
-
67
- ```bash
68
- $ miteru execute
69
- ...
70
- https://dummy1.com: it doesn't contain a phishing kit.
71
- https://dummy2.com: it doesn't contain a phishing kit.
72
- https://dummy3.com: it doesn't contain a phishing kit.
73
- https://dummy4.com: it might contain a phishing kit (dummy.zip).
74
- ```
75
-
76
- ## Using Docker (alternative if you don't install Ruby)
77
-
78
- ```bash
79
- $ docker pull ninoseki/miteru
80
- # ex. auto-download detected phishing kit(s) into host machines's /tmp directory
81
- $ docker run --rm -v /tmp:/tmp ninoseki/miteru execute --auto-download
82
- ```
83
-
84
- ## Configuration
85
-
86
- For using `--post-to-slack` feature, you should set the following environment variables:
87
-
88
- - `SLACK_WEBHOOK_URL`: Your Slack Webhook URL.
89
- - `SLACK_CHANNEL`: Slack channel to post a message (default: "#general").
90
-
91
- If you are a urlscan.io Pro user, set your API key as an environment variable `URLSCAN_API_KEY`.
92
-
93
- It enables you to subscribe the urlscan.io phish feed.
94
-
95
- ## Examples
96
-
97
- ### Aasciinema cast
98
-
99
- [![asciicast](https://asciinema.org/a/hHpkHhMLiiv17gmdRhVMtZWwM.svg)](https://asciinema.org/a/hHpkHhMLiiv17gmdRhVMtZWwM)
100
-
101
- ### Slack notification
102
-
103
- ![img](./screenshots/slack.png)
31
+ - [x] Phishing kit detection & collection
32
+ - [x] Slack notification
33
+ - [x] Threading
104
34
 
105
- ## Alternatives
35
+ ## Docs
106
36
 
107
- - [t4d/StalkPhish](https://github.com/t4d/StalkPhish): The Phishing kits stalker, harvesting phishing kits for investigations.
108
- - [duo-labs/phish-collect](https://github.com/duo-labs/phish-collect): Python script to hunt phishing kits.
109
- - [leunammejii/analyst_arsenal](https://github.com/leunammejii/analyst_arsenal): A tool belt for analysts to continue fighting the good fight.
37
+ - [Requirements & Installation](https://github.com/ninoseki/miteru/wiki/Requirements-&-Installation)
38
+ - [Usage](https://github.com/ninoseki/miteru/wiki/Usage)
39
+ - [Configuration](https://github.com/ninoseki/miteru/wiki/Configuration)
40
+ - [Alternatives](https://github.com/ninoseki/miteru/wiki/Alternatives)
data/docker/Dockerfile CHANGED
@@ -1,10 +1,13 @@
1
- FROM ruby:2.6-alpine3.10
2
- RUN apk --no-cache add git build-base ruby-dev \
1
+ FROM ruby:3-alpine3.13
2
+
3
+ RUN apk --no-cache add git build-base ruby-dev mysql-client mysql-dev sqlite-dev postgresql-client postgresql-dev \
3
4
  && cd /tmp/ \
4
5
  && git clone https://github.com/ninoseki/miteru.git \
5
6
  && cd miteru \
6
7
  && gem build miteru.gemspec -o miteru.gem \
7
8
  && gem install miteru.gem \
9
+ && gem install mysql2 \
10
+ && gem install pg \
8
11
  && rm -rf /tmp/miteru \
9
12
  && apk del --purge git build-base ruby-dev
10
13
 
data/exe/miteru CHANGED
@@ -5,4 +5,5 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
5
5
 
6
6
  require "miteru"
7
7
 
8
- Miteru::CLI.start
8
+ ARGV.unshift(Miteru::CLI.default_task) unless Miteru::CLI.all_tasks.key?(ARGV[0])
9
+ Miteru::CLI.start(ARGV)
@@ -5,6 +5,7 @@ require "uri"
5
5
  module Miteru
6
6
  class Attachement
7
7
  attr_reader :url
8
+
8
9
  def initialize(url)
9
10
  @url = url
10
11
  end
@@ -31,7 +32,7 @@ module Miteru
31
32
  {
32
33
  type: "button",
33
34
  text: "Lookup on VirusTotal",
34
- url: _vt_link,
35
+ url: _vt_link
35
36
  }
36
37
  end
37
38
 
@@ -41,7 +42,7 @@ module Miteru
41
42
  {
42
43
  type: "button",
43
44
  text: "Lookup on urlscan.io",
44
- url: _urlscan_link,
45
+ url: _urlscan_link
45
46
  }
46
47
  end
47
48
 
data/lib/miteru/cli.rb CHANGED
@@ -5,11 +5,11 @@ require "thor"
5
5
  module Miteru
6
6
  class CLI < Thor
7
7
  method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of phishing kits"
8
- method_option :ayashige, type: :boolean, default: false, desc: "Enable or disable ayashige(ninoseki/ayashige) feed"
8
+ method_option :ayashige, type: :boolean, default: false, desc: "Enable or disable Ayashige(ninoseki/ayashige) feed"
9
9
  method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
10
- method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
11
- method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
12
- method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 10,000)"
10
+ method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
11
+ method_option :post_to_slack, type: :boolean, default: false, desc: "Enable or disable Slack notification"
12
+ method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results to fetch. (Max: 10,000)"
13
13
  method_option :threads, type: :numeric, desc: "Number of threads to use"
14
14
  method_option :verbose, type: :boolean, default: true
15
15
  desc "execute", "Execute the crawler"
@@ -29,5 +29,13 @@ module Miteru
29
29
 
30
30
  Crawler.execute
31
31
  end
32
+
33
+ default_command :execute
34
+
35
+ class << self
36
+ def exit_on_failure?
37
+ true
38
+ end
39
+ end
32
40
  end
33
41
  end
@@ -28,8 +28,21 @@ module Miteru
28
28
  # @return [Boolean]
29
29
  attr_accessor :verbose
30
30
 
31
+ # @return [String]
32
+ attr_accessor :database
33
+
34
+ # @return [String, nil]
35
+ attr_accessor :slack_webhook_url
36
+
37
+ # @return [String]
38
+ attr_accessor :slack_channel
39
+
40
+ # @return [Array<String>]
31
41
  attr_reader :valid_extensions
32
42
 
43
+ # @return [Array<String>]
44
+ attr_reader :valid_mime_types
45
+
33
46
  def initialize
34
47
  @auto_download = false
35
48
  @ayashige = false
@@ -39,8 +52,13 @@ module Miteru
39
52
  @size = 100
40
53
  @threads = Parallel.processor_count
41
54
  @verbose = false
55
+ @database = ENV["MITERU_DATABASE"] || "miteru.db"
56
+
57
+ @slack_webhook_url = ENV["SLACK_WEBHOOK_URL"]
58
+ @slack_channel = ENV["SLACK_CHANNEL"] || "#general"
42
59
 
43
60
  @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
61
+ @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
44
62
  end
45
63
 
46
64
  def auto_download?
@@ -62,6 +80,10 @@ module Miteru
62
80
  def verbose?
63
81
  @verbose
64
82
  end
83
+
84
+ def slack_webhook_url?
85
+ @slack_webhook_url
86
+ end
65
87
  end
66
88
 
67
89
  class << self
@@ -6,8 +6,7 @@ require "uri"
6
6
 
7
7
  module Miteru
8
8
  class Crawler
9
- attr_reader :downloader
10
- attr_reader :feeds
9
+ attr_reader :downloader, :feeds
11
10
 
12
11
  def initialize
13
12
  @downloader = Downloader.new(Miteru.configuration.download_to)
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record"
4
+
5
+ class InitialSchema < ActiveRecord::Migration[6.1]
6
+ def change
7
+ create_table :records, if_not_exists: true do |t|
8
+ t.string :hash, null: false, index: { unique: true }
9
+ t.string :hostname, null: false
10
+ t.json :headers, null: false
11
+ t.text :filename, null: false
12
+ t.string :downloaded_as, null: false
13
+ t.integer :filesize, null: false
14
+ t.string :mime_type, null: false
15
+ t.text :url, null: false
16
+
17
+ t.timestamps
18
+ end
19
+ end
20
+ end
21
+
22
+ def adapter
23
+ return "postgresql" if Miteru.configuration.database.start_with?("postgresql://", "postgres://")
24
+ return "mysql2" if Miteru.configuration.database.start_with?("mysql2://")
25
+
26
+ "sqlite3"
27
+ end
28
+
29
+ module Miteru
30
+ class Database
31
+ class << self
32
+ def connect
33
+ case adapter
34
+ when "postgresql", "mysql2"
35
+ ActiveRecord::Base.establish_connection(Miteru.configuration.database)
36
+ else
37
+ ActiveRecord::Base.establish_connection(
38
+ adapter: adapter,
39
+ database: Miteru.configuration.database
40
+ )
41
+ end
42
+
43
+ # ActiveRecord::Base.logger = Logger.new STDOUT
44
+ ActiveRecord::Migration.verbose = false
45
+
46
+ InitialSchema.migrate(:up)
47
+ rescue StandardError => _e
48
+ # Do nothing
49
+ end
50
+
51
+ def close
52
+ ActiveRecord::Base.clear_active_connections!
53
+ ActiveRecord::Base.connection.close
54
+ end
55
+
56
+ def destroy!
57
+ return unless ActiveRecord::Base.connected?
58
+
59
+ InitialSchema.migrate(:down)
60
+ end
61
+ end
62
+ end
63
+ end
64
+
65
+ Miteru::Database.connect
@@ -6,13 +6,12 @@ require "uri"
6
6
 
7
7
  module Miteru
8
8
  class Downloader
9
- attr_reader :base_dir
10
- attr_reader :memo
9
+ attr_reader :base_dir, :memo
11
10
 
12
11
  def initialize(base_dir = "/tmp")
13
12
  @base_dir = base_dir
14
13
  @memo = {}
15
- raise ArgumentError, "#{base_dir} is not exist." unless Dir.exist?(base_dir)
14
+ raise ArgumentError, "#{base_dir} doesn't exist." unless Dir.exist?(base_dir)
16
15
  end
17
16
 
18
17
  def download_kits(kits)
@@ -22,25 +21,26 @@ module Miteru
22
21
  private
23
22
 
24
23
  def download_kit(kit)
25
- destination = kit.download_filepath
24
+ destination = kit.filepath_to_download
26
25
  begin
27
- downloaded_filepath = HTTPClient.download(kit.url, destination)
28
- hash = sha256(downloaded_filepath)
26
+ downloaded_as = HTTPClient.download(kit.url, destination)
27
+ hash = sha256(downloaded_as)
28
+
29
+ # Remove a downloaded file if it is not unique
29
30
  if duplicated?(hash)
30
- puts "Do not download #{kit.url} because there is a duplicate file in the directory (SHA256: #{hash})."
31
- FileUtils.rm downloaded_filepath
32
- else
33
- puts "Download #{kit.url} as #{downloaded_filepath}"
31
+ puts "Don't download #{kit.url}. The same hash is already recorded. (SHA256: #{hash})."
32
+ FileUtils.rm downloaded_as
33
+ return
34
34
  end
35
+
36
+ # Record a kit in DB
37
+ Record.create_by_kit_and_hash(kit, hash)
38
+ puts "Download #{kit.url} as #{downloaded_as}"
35
39
  rescue Down::Error => e
36
40
  puts "Failed to download: #{kit.url} (#{e})"
37
41
  end
38
42
  end
39
43
 
40
- def filepath_to_download(filename)
41
- "#{base_dir}/#{filename}"
42
- end
43
-
44
44
  def sha256(path)
45
45
  return memo[path] if memo.key?(path)
46
46
 
@@ -50,12 +50,8 @@ module Miteru
50
50
  hash
51
51
  end
52
52
 
53
- def sha256s
54
- Dir.glob("#{base_dir}/*.{zip,rar,7z,tar,gz}").map { |path| sha256(path) }
55
- end
56
-
57
53
  def duplicated?(hash)
58
- sha256s.count(hash) > 1
54
+ !Record.unique_hash?(hash)
59
55
  end
60
56
  end
61
57
  end