miteru 1.2.2 → 2.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/gem.yml +36 -0
  3. data/.github/workflows/{test.yml → ruby.yml} +4 -13
  4. data/.gitignore +7 -2
  5. data/.rspec +1 -1
  6. data/README.md +7 -17
  7. data/docker-compose.yml +12 -0
  8. data/exe/miteru +3 -3
  9. data/lefthook.yml +9 -0
  10. data/lib/miteru/cli/application.rb +27 -0
  11. data/lib/miteru/cli/base.rb +16 -0
  12. data/lib/miteru/cli/database.rb +11 -0
  13. data/lib/miteru/commands/database.rb +23 -0
  14. data/lib/miteru/commands/main.rb +37 -0
  15. data/lib/miteru/commands/sidekiq.rb +35 -0
  16. data/lib/miteru/commands/web.rb +37 -0
  17. data/lib/miteru/concerns/database_connectable.rb +16 -0
  18. data/lib/miteru/concerns/error_unwrappable.rb +30 -0
  19. data/lib/miteru/config.rb +98 -0
  20. data/lib/miteru/crawler.rb +28 -44
  21. data/lib/miteru/database.rb +50 -38
  22. data/lib/miteru/downloader.rb +52 -41
  23. data/lib/miteru/errors.rb +37 -0
  24. data/lib/miteru/feeds/ayashige.rb +9 -20
  25. data/lib/miteru/feeds/base.rb +141 -0
  26. data/lib/miteru/feeds/phishing_database.rb +11 -10
  27. data/lib/miteru/feeds/urlscan.rb +47 -19
  28. data/lib/miteru/feeds/urlscan_pro.rb +20 -18
  29. data/lib/miteru/http.rb +51 -0
  30. data/lib/miteru/kit.rb +28 -20
  31. data/lib/miteru/mixin.rb +2 -29
  32. data/lib/miteru/notifiers/base.rb +10 -3
  33. data/lib/miteru/notifiers/slack.rb +85 -10
  34. data/lib/miteru/notifiers/urlscan.rb +29 -14
  35. data/lib/miteru/orchestrator.rb +58 -0
  36. data/lib/miteru/record.rb +8 -15
  37. data/lib/miteru/service.rb +28 -0
  38. data/lib/miteru/sidekiq/application.rb +13 -0
  39. data/lib/miteru/sidekiq/jobs.rb +21 -0
  40. data/lib/miteru/version.rb +1 -1
  41. data/lib/miteru/web/application.rb +42 -0
  42. data/lib/miteru/website.rb +48 -48
  43. data/lib/miteru.rb +130 -22
  44. data/miteru.gemspec +49 -38
  45. metadata +262 -97
  46. data/.overcommit.yml +0 -12
  47. data/.standard.yml +0 -4
  48. data/lib/miteru/attachement.rb +0 -74
  49. data/lib/miteru/cli.rb +0 -41
  50. data/lib/miteru/configuration.rb +0 -122
  51. data/lib/miteru/error.rb +0 -7
  52. data/lib/miteru/feeds/feed.rb +0 -53
  53. data/lib/miteru/feeds/phishstats.rb +0 -28
  54. data/lib/miteru/feeds.rb +0 -45
  55. data/lib/miteru/http_client.rb +0 -85
@@ -1,74 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "uri"
4
-
5
- module Miteru
6
- class Attachement
7
- attr_reader :url
8
-
9
- def initialize(url)
10
- @url = url
11
- end
12
-
13
- def to_a
14
- [
15
- {
16
- text: defanged_url,
17
- fallback: "VT & urlscan.io links",
18
- actions: actions
19
- }
20
- ]
21
- end
22
-
23
- private
24
-
25
- def actions
26
- [vt_link, urlscan_link].compact
27
- end
28
-
29
- def vt_link
30
- return nil unless _vt_link
31
-
32
- {
33
- type: "button",
34
- text: "Lookup on VirusTotal",
35
- url: _vt_link
36
- }
37
- end
38
-
39
- def urlscan_link
40
- return nil unless _urlscan_link
41
-
42
- {
43
- type: "button",
44
- text: "Lookup on urlscan.io",
45
- url: _urlscan_link
46
- }
47
- end
48
-
49
- def defanged_url
50
- @defanged_url ||= url.to_s.gsub(/\./, "[.]")
51
- end
52
-
53
- def domain
54
- @domain ||=
55
- [].tap do |out|
56
- out << URI(url).hostname
57
- rescue URI::Error => _e
58
- out << nil
59
- end.first
60
- end
61
-
62
- def _urlscan_link
63
- return nil unless domain
64
-
65
- "https://urlscan.io/domain/#{domain}"
66
- end
67
-
68
- def _vt_link
69
- return nil unless domain
70
-
71
- "https://www.virustotal.com/#/domain/#{domain}"
72
- end
73
- end
74
- end
data/lib/miteru/cli.rb DELETED
@@ -1,41 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "thor"
4
-
5
- module Miteru
6
- class CLI < Thor
7
- method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of phishing kits"
8
- method_option :ayashige, type: :boolean, default: false, desc: "Enable or disable Ayashige(ninoseki/ayashige) feed"
9
- method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
10
- method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
11
- method_option :post_to_slack, type: :boolean, default: false, desc: "Enable or disable Slack notification"
12
- method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results to fetch. (Max: 10,000)"
13
- method_option :threads, type: :numeric, desc: "Number of threads to use"
14
- method_option :verbose, type: :boolean, default: true
15
- desc "execute", "Execute the crawler"
16
- def execute
17
- Miteru.configure do |config|
18
- config.auto_download = options["auto_download"]
19
- config.ayashige = options["ayashige"]
20
- config.directory_traveling = options["directory_traveling"]
21
- config.download_to = options["download_to"]
22
- config.post_to_slack = options["post_to_slack"]
23
- config.size = options["size"]
24
- config.verbose = options["verbose"]
25
-
26
- threads = options["threads"]
27
- config.threads = threads if threads
28
- end
29
-
30
- Crawler.execute
31
- end
32
-
33
- default_command :execute
34
-
35
- class << self
36
- def exit_on_failure?
37
- true
38
- end
39
- end
40
- end
41
- end
@@ -1,122 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "parallel"
4
-
5
- module Miteru
6
- class Configuration
7
- # @return [Boolean]
8
- attr_accessor :auto_download
9
-
10
- # @return [Boolean]
11
- attr_accessor :ayashige
12
-
13
- # @return [Boolean]
14
- attr_accessor :directory_traveling
15
-
16
- # @return [String]
17
- attr_accessor :download_to
18
-
19
- # @return [Boolean]
20
- attr_accessor :post_to_slack
21
-
22
- # @return [Integer]
23
- attr_accessor :size
24
-
25
- # @return [Integer]
26
- attr_accessor :threads
27
-
28
- # @return [Boolean]
29
- attr_accessor :verbose
30
-
31
- # @return [String]
32
- attr_accessor :database
33
-
34
- # @return [String, nil]
35
- attr_accessor :slack_webhook_url
36
-
37
- # @return [String]
38
- attr_accessor :slack_channel
39
-
40
- # @return [String]
41
- attr_accessor :urlscan_api_key
42
-
43
- # @return [String]
44
- attr_accessor :urlscan_submit_visibility
45
-
46
- # @return [Array<String>]
47
- attr_reader :valid_extensions
48
-
49
- # @return [Array<String>]
50
- attr_reader :valid_mime_types
51
-
52
- # @return [Integer]
53
- attr_reader :file_maxsize
54
-
55
- def initialize
56
- @auto_download = false
57
- @ayashige = false
58
- @directory_traveling = false
59
- @download_to = "/tmp"
60
- @post_to_slack = false
61
- @size = 100
62
- @threads = Parallel.processor_count
63
- @verbose = false
64
- @database = ENV.fetch("MITERU_DATABASE", "miteru.db")
65
- @file_maxsize = ENV.fetch("FILE_MAXSIZE", 1024 * 1024 * 100).to_i
66
-
67
- @slack_webhook_url = ENV.fetch("SLACK_WEBHOOK_URL", nil)
68
- @slack_channel = ENV.fetch("SLACK_CHANNEL", "#general")
69
-
70
- @urlscan_api_key = ENV.fetch("URLSCAN_API_KEY", nil)
71
- @urlscan_submit_visibility = ENV.fetch("URLSCAN_SUBMIT_VISIBILITY", "public")
72
-
73
- @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
74
- @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
75
- end
76
-
77
- def auto_download?
78
- @auto_download
79
- end
80
-
81
- def ayashige?
82
- @ayashige
83
- end
84
-
85
- def directory_traveling?
86
- @directory_traveling
87
- end
88
-
89
- def post_to_slack?
90
- @post_to_slack
91
- end
92
-
93
- def verbose?
94
- @verbose
95
- end
96
-
97
- def slack_webhook_url?
98
- !@slack_webhook_url.nil?
99
- end
100
-
101
- def urlscan_api_key?
102
- !@urlscan_api_key.nil?
103
- end
104
- end
105
-
106
- class << self
107
- # @return [Miteru::Configuration] Miteru's current configuration
108
- def configuration
109
- @configuration ||= Configuration.new
110
- end
111
-
112
- # Set Miteru's configuration
113
- # @param config [Miteru::Configuration]
114
- attr_writer :configuration
115
-
116
- # Modify Miteru's current configuration
117
- # @yieldparam [Miteru::Configuration] config current Miteru config
118
- def configure
119
- yield configuration
120
- end
121
- end
122
- end
data/lib/miteru/error.rb DELETED
@@ -1,7 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Miteru
4
- class HTTPResponseError < StandardError; end
5
-
6
- class DownloadError < StandardError; end
7
- end
@@ -1,53 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Miteru
4
- class Feeds
5
- class Feed
6
- include Mixins::URL
7
-
8
- def source
9
- @source ||= self.class.to_s.split("::").last
10
- end
11
-
12
- #
13
- # Return URLs
14
- #
15
- # @return [Array<String>] URLs
16
- #
17
- def urls
18
- raise NotImplementedError, "You must implement #{self.class}##{__method__}"
19
- end
20
-
21
- #
22
- # Return entries
23
- #
24
- # @return [Array<Miteru::Entry>]
25
- #
26
- def entries
27
- breakdowend_urls.map do |url|
28
- Entry.new(url, source)
29
- end
30
- end
31
-
32
- #
33
- # Return breakdowned URLs
34
- #
35
- # @return [Array<String>] Breakdowned URLs
36
- #
37
- def breakdowend_urls
38
- urls.select { |url| url.start_with?("http://", "https://") }.map do |url|
39
- breakdown(url, Miteru.configuration.directory_traveling?)
40
- end.flatten.uniq
41
- end
42
-
43
- private
44
-
45
- def get(url)
46
- res = HTTPClient.get(url)
47
- raise HTTPResponseError if res.code != 200
48
-
49
- res.body.to_s
50
- end
51
- end
52
- end
53
- end
@@ -1,28 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "json"
4
- require "uri"
5
-
6
- module Miteru
7
- class Feeds
8
- class PhishStats < Feed
9
- URL = "https://phishstats.info:2096/api/phishing?_sort=-id&size=100"
10
-
11
- def urls
12
- json = JSON.parse(get(URL))
13
- json.map do |entry|
14
- entry["url"]
15
- end
16
- rescue HTTPResponseError, HTTP::Error, JSON::ParserError => e
17
- Miteru.logger.error "Failed to load PhishStats feed (#{e})"
18
- []
19
- end
20
-
21
- private
22
-
23
- def url_for(path)
24
- URI(URL + path)
25
- end
26
- end
27
- end
28
- end
data/lib/miteru/feeds.rb DELETED
@@ -1,45 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative "./feeds/feed"
4
- require_relative "./feeds/phishing_database"
5
- require_relative "./feeds/phishstats"
6
- require_relative "./feeds/ayashige"
7
- require_relative "./feeds/urlscan"
8
- require_relative "./feeds/urlscan_pro"
9
-
10
- module Miteru
11
- class Entry
12
- # @return [String]
13
- attr_reader :url
14
- # @return [String]
15
- attr_reader :source
16
-
17
- def initialize(url, source)
18
- @url = url
19
- @source = source
20
- end
21
- end
22
-
23
- class Feeds
24
- IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
25
-
26
- def initialize
27
- @feeds = [
28
- PhishingDatabase.new,
29
- PhishStats.new,
30
- UrlScan.new(Miteru.configuration.size),
31
- UrlScanPro.new,
32
- Miteru.configuration.ayashige? ? Ayashige.new : nil
33
- ].compact
34
- end
35
-
36
- #
37
- # Returns a list of suspicious entries
38
- #
39
- # @return [Array<Entry>]
40
- #
41
- def suspicious_entries
42
- @suspicious_entries ||= @feeds.map(&:entries).flatten.uniq(&:url)
43
- end
44
- end
45
- end
@@ -1,85 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "down/http"
4
- require "http"
5
- require "uri"
6
-
7
- module Miteru
8
- class HTTPClient
9
- DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
10
- URLSCAN_UA = "miteru/#{Miteru::VERSION}"
11
-
12
- attr_reader :ssl_context
13
-
14
- def initialize
15
- ctx = OpenSSL::SSL::SSLContext.new
16
- ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
17
- @ssl_context = ctx
18
- end
19
-
20
- def download(url, destination)
21
- down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
22
- down.download(url, destination: destination)
23
- destination
24
- end
25
-
26
- def head(url, options = {})
27
- options = options.merge default_options
28
-
29
- HTTP.follow
30
- .timeout(3)
31
- .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
32
- .head(url, options)
33
- end
34
-
35
- def get(url, options = {})
36
- options = options.merge default_options
37
-
38
- HTTP.follow
39
- .timeout(write: 2, connect: 5, read: 10)
40
- .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
41
- .get(url, options)
42
- end
43
-
44
- def post(url, options = {})
45
- HTTP.post url, options
46
- end
47
-
48
- class << self
49
- def download(url, base_dir = "/tmp")
50
- new.download(url, base_dir)
51
- end
52
-
53
- def get(url, options = {})
54
- new.get url, options
55
- end
56
-
57
- def post(url, options = {})
58
- new.post url, options
59
- end
60
-
61
- def head(url, options = {})
62
- new.head url, options
63
- end
64
- end
65
-
66
- private
67
-
68
- def default_headers
69
- { user_agent: DEFAULT_UA }
70
- end
71
-
72
- def default_options
73
- { ssl_context: ssl_context }
74
- end
75
-
76
- def urlscan_headers
77
- { user_agent: URLSCAN_UA }
78
- end
79
-
80
- def urlscan_url?(url)
81
- uri = URI(url)
82
- uri.hostname == "urlscan.io"
83
- end
84
- end
85
- end