miteru 0.3.2 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/lib/miteru/cli.rb +4 -2
- data/lib/miteru/crawler.rb +11 -5
- data/lib/miteru/http_client.rb +1 -1
- data/lib/miteru/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aa876b402ad3fcfe44fb1a66035bdb7649db11383650faf572d94fb8b4c8f498
|
4
|
+
data.tar.gz: c8ffde9ad933fbe670900d8b6547f5ce35ba7f85bd0a0d1201d3c00518d612bb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 15bc48dc53696348635d637674c5574d537b8cf220939dce354875706f1a58820b95ecab547da0106a72e76d026d3e067b14438bfbef7ce1ce971737491269e2
|
7
|
+
data.tar.gz: 86d0aad06740b2013c35ef3920fa27fc1ac811580c08103273f3c073362d37b0e2c8bd0e49c9d8aac63dd06004556b902c6bccaf57f7833670e01d8e907d72a1
|
data/README.md
CHANGED
@@ -36,6 +36,8 @@ Options:
|
|
36
36
|
[--auto-download], [--no-auto-download] # Enable or disable auto-download of *.zip file(s)
|
37
37
|
[--download-to=DOWNLOAD_TO] # Directory to download file(s)
|
38
38
|
# Default: /tmp
|
39
|
+
[--size=N] # Number of urlscan.io's results. (Max: 100,000)
|
40
|
+
# Default: 100
|
39
41
|
[--post-to-slack], [--no-post-to-slack] # Post a message to Slack if it detects a phishing kit
|
40
42
|
[--verbose], [--no-verbose]
|
41
43
|
# Default: true
|
data/lib/miteru/cli.rb
CHANGED
@@ -8,15 +8,17 @@ module Miteru
|
|
8
8
|
class CLI < Thor
|
9
9
|
method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of *.zip file(s)"
|
10
10
|
method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
|
11
|
+
method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 100,000)"
|
11
12
|
method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
|
12
13
|
method_option :verbose, type: :boolean, default: true
|
13
14
|
desc "execute", "Execute the crawler"
|
14
15
|
def execute
|
15
|
-
websites = Crawler.execute(options[:verbose])
|
16
|
+
websites = Crawler.execute(size: options[:size], verbose: options[:verbose])
|
16
17
|
websites.each do |website|
|
17
18
|
next unless website.has_kit?
|
18
19
|
|
19
|
-
|
20
|
+
message = "#{website.url}: it might contain a phishing kit (#{website.zip_files.join(',')})."
|
21
|
+
puts message.colorize(:light_red)
|
20
22
|
post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
|
21
23
|
download_zip_files(website.url, website.zip_files, options[:download_to]) if options[:auto_download]
|
22
24
|
end
|
data/lib/miteru/crawler.rb
CHANGED
@@ -6,17 +6,23 @@ require "http"
|
|
6
6
|
module Miteru
|
7
7
|
class Crawler
|
8
8
|
attr_reader :threads
|
9
|
-
|
9
|
+
attr_reader :size
|
10
|
+
attr_reader :verbose
|
11
|
+
|
12
|
+
def initialize(size: 100, verbose: false)
|
10
13
|
@threads = 10
|
14
|
+
@size = size
|
15
|
+
@verbose = verbose
|
16
|
+
raise ArgumentError, "size must be less than 100,000" if size > 100_000
|
11
17
|
end
|
12
18
|
|
13
19
|
def suspicous_urls
|
14
|
-
url = "https://urlscan.io/api/v1/search/?q=certstream-suspicious"
|
20
|
+
url = "https://urlscan.io/api/v1/search/?q=certstream-suspicious&size=#{size}"
|
15
21
|
res = JSON.parse(get(url))
|
16
22
|
res["results"].map { |result| result.dig("task", "url") }
|
17
23
|
end
|
18
24
|
|
19
|
-
def execute
|
25
|
+
def execute
|
20
26
|
pool = Thread.pool(threads)
|
21
27
|
websites = []
|
22
28
|
|
@@ -32,8 +38,8 @@ module Miteru
|
|
32
38
|
websites
|
33
39
|
end
|
34
40
|
|
35
|
-
def self.execute(verbose
|
36
|
-
new
|
41
|
+
def self.execute(size: 100, verbose: false)
|
42
|
+
new(size: size, verbose: verbose).execute
|
37
43
|
end
|
38
44
|
|
39
45
|
private
|
data/lib/miteru/http_client.rb
CHANGED
data/lib/miteru/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miteru
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Manabu Niseki
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-09-
|
11
|
+
date: 2018-09-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|