miteru 0.14.2 → 0.14.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: acafc5c390603cb4e035ba592a47291eb5b93f20c1a6f4c12dbf22f40b15f3b4
4
- data.tar.gz: bc8e05d8356ed633c45c1c241abb9972c79a85edab8f5d555cfce740d72f938f
3
+ metadata.gz: c1ae4b442c2963ff387cc1bf3bf6390a393c2f8ba416276f2dc0cc48f2ceb86b
4
+ data.tar.gz: ce3baa2e515837cf722a4bd90650f558a337cde2e2dd2ecef976620f7d20eddb
5
5
  SHA512:
6
- metadata.gz: e191d8815c1eda041a9c64e2ef5c62a16da248ff5a19cbcd419ccdfa956963e0ed1177e83193c45df71971f8717ecd558a8a0b0d69a0d0ac8c7a2a4c9463ba87
7
- data.tar.gz: b098a5efaa9eb18a618a5c3a42a0ce7b584ad37d39c7f154cc5afed9a11f5c0c2750215efa83b7d8a4e0109e32399dc37978da2a1bcf4aa2ff9c21f7454d974f
6
+ metadata.gz: 40afc8ff440ffad5be4e4b7efdb7670c6df1a9e05d503930424e53a6c57fcfd3270239f39a92de2ccac149e8bc6175c87833183ad822d900d526490f067c06dc
7
+ data.tar.gz: 2ad37a6b2ebfaf78451ba35fa70d826ad95202cf077fdba5ed02cbebd8c74104c6f7f2e49da1ea26a9d42890bbd0b4d7b76e06787cab3697ba43f441a5825230
@@ -1,4 +1,4 @@
1
- FROM ruby:2.6-alpine3.10
1
+ FROM ruby:2.7-alpine3.10
2
2
  RUN apk --no-cache add git build-base ruby-dev \
3
3
  && cd /tmp/ \
4
4
  && git clone https://github.com/ninoseki/miteru.git \
data/exe/miteru CHANGED
@@ -5,4 +5,5 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
5
5
 
6
6
  require "miteru"
7
7
 
8
- Miteru::CLI.start
8
+ ARGV.unshift(Miteru::CLI.default_task) unless Miteru::CLI.all_tasks.key?(ARGV[0])
9
+ Miteru::CLI.start(ARGV)
@@ -29,5 +29,13 @@ module Miteru
29
29
 
30
30
  Crawler.execute
31
31
  end
32
+
33
+ default_command :execute
34
+
35
+ class << self
36
+ def exit_on_failure?
37
+ true
38
+ end
39
+ end
32
40
  end
33
41
  end
@@ -28,6 +28,10 @@ module Miteru
28
28
  # @return [Boolean]
29
29
  attr_accessor :verbose
30
30
 
31
+ attr_reader :valid_extensions
32
+
33
+ attr_reader :valid_mime_types
34
+
31
35
  def initialize
32
36
  @auto_download = false
33
37
  @ayashige = false
@@ -37,6 +41,9 @@ module Miteru
37
41
  @size = 100
38
42
  @threads = Parallel.processor_count
39
43
  @verbose = false
44
+
45
+ @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
46
+ @valid_mime_types = ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar", "application/gzip"]
40
47
  end
41
48
 
42
49
  def auto_download?
@@ -10,7 +10,6 @@ require_relative "./feeds/urlscan_pro"
10
10
  module Miteru
11
11
  class Feeds
12
12
  IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
13
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
14
13
 
15
14
  def initialize
16
15
  @feeds = [
@@ -49,18 +48,7 @@ module Miteru
49
48
  segments = uri.path.split("/")
50
49
  return [base] if segments.length.zero?
51
50
 
52
- urls = (0...segments.length).map do |idx|
53
- breakdowned_url = "#{base}#{segments[0..idx].join('/')}"
54
- breakdown = [breakdowned_url]
55
- if idx > 0 && idx < segments.length
56
- next if segments[idx].nil? || invalid_extension?(segments[idx])
57
-
58
- VALID_EXTENSIONS.each do |ext|
59
- breakdown << "#{base}#{segments[0..idx - 1].join('/')}/#{segments[idx]}#{ext}"
60
- end
61
- end
62
- breakdown
63
- end.flatten.compact
51
+ urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
64
52
 
65
53
  urls.reject do |breakdowned_url|
66
54
  # Reject a url which ends with specific extension names
@@ -6,7 +6,7 @@ require "uri"
6
6
  module Miteru
7
7
  class Feeds
8
8
  class PhishingDatabase < Feed
9
- URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-NEW-today.txt"
9
+ URL = "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-links-ACTIVE-NOW.txt"
10
10
 
11
11
  def urls
12
12
  body = get(URL)
@@ -18,7 +18,7 @@ module Miteru
18
18
  end
19
19
 
20
20
  def download(url, destination)
21
- down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
21
+ down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
22
22
  down.download(url, destination: destination)
23
23
  destination
24
24
  end
@@ -5,37 +5,44 @@ require "securerandom"
5
5
 
6
6
  module Miteru
7
7
  class Kit
8
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
8
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
9
+ VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
9
10
 
10
- attr_reader :base_url, :link
11
+ attr_reader :url
11
12
 
12
- def initialize(base_url:, link:)
13
- @base_url = base_url
14
- @link = link.start_with?("/") ? link[1..-1] : link
13
+ attr_reader :status
14
+ attr_reader :content_length
15
+ attr_reader :mime_type
16
+
17
+ def initialize(url)
18
+ @url = url
19
+
20
+ @content_length = nil
21
+ @mime_type = nil
22
+ @status = nil
15
23
  end
16
24
 
17
- def valid?
18
- VALID_EXTENSIONS.include? extname
25
+ def valid?;
26
+ # make a HEAD request for the validation
27
+ before_validation
28
+
29
+ valid_ext? && reachable? && valid_mime_type? && valid_content_length?
19
30
  end
20
31
 
21
32
  def extname
22
- return ".tar.gz" if link.end_with?("tar.gz")
33
+ return ".tar.gz" if url.end_with?("tar.gz")
23
34
 
24
- File.extname(link)
35
+ File.extname(url)
25
36
  end
26
37
 
27
38
  def basename
28
- File.basename(link)
39
+ File.basename(url)
29
40
  end
30
41
 
31
42
  def filename
32
43
  CGI.unescape basename
33
44
  end
34
45
 
35
- def url
36
- "#{base_url}/#{basename}"
37
- end
38
-
39
46
  def download_filepath
40
47
  "#{base_dir}/#{download_filename}"
41
48
  end
@@ -59,7 +66,7 @@ module Miteru
59
66
  end
60
67
 
61
68
  def hostname
62
- URI(base_url).hostname
69
+ URI(url).hostname
63
70
  end
64
71
 
65
72
  def download_filename
@@ -69,5 +76,30 @@ module Miteru
69
76
  def base_dir
70
77
  @base_dir ||= Miteru.configuration.download_to
71
78
  end
79
+
80
+ def valid_ext?
81
+ VALID_EXTENSIONS.include? extname
82
+ end
83
+
84
+ def before_validation
85
+ res = HTTPClient.head(url)
86
+ @content_length = res.content_length
87
+ @mime_type = res.content_type.mime_type.to_s
88
+ @status = res.status
89
+ rescue StandardError
90
+ # do nothing
91
+ end
92
+
93
+ def reachable?
94
+ status&.success?
95
+ end
96
+
97
+ def valid_mime_type?
98
+ VALID_MIME_TYPES.include? mime_type
99
+ end
100
+
101
+ def valid_content_length?
102
+ content_length.to_i > 0
103
+ end
72
104
  end
73
105
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.14.2"
4
+ VERSION = "0.14.7"
5
5
  end
@@ -4,9 +4,10 @@ require "oga"
4
4
 
5
5
  module Miteru
6
6
  class Website
7
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
7
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
8
8
 
9
9
  attr_reader :url
10
+
10
11
  def initialize(url)
11
12
  @url = url
12
13
  end
@@ -16,25 +17,12 @@ module Miteru
16
17
  end
17
18
 
18
19
  def kits
19
- if ext?
20
- return [] unless check(url)
21
-
22
- link = url.split("/").last
23
- base_url = url.split("/")[0..-2].join("/")
24
- kit = Kit.new(base_url: base_url, link: link)
25
- return kit.valid? ? [kit] : []
26
- end
27
-
28
- links.map do |link|
29
- kit = Kit.new(base_url: url, link: link.to_s)
20
+ @kits ||= links.map do |link|
21
+ kit = Kit.new(link)
30
22
  kit.valid? ? kit : nil
31
23
  end.compact
32
24
  end
33
25
 
34
- def ext?
35
- VALID_EXTENSIONS.any? { |ext| url.end_with?(ext) }
36
- end
37
-
38
26
  def ok?
39
27
  response.code == 200
40
28
  end
@@ -48,9 +36,7 @@ module Miteru
48
36
  end
49
37
 
50
38
  def has_kits?
51
- return kits? if ext?
52
-
53
- ok? && index? && kits?
39
+ kits?
54
40
  rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
55
41
  false
56
42
  end
@@ -63,19 +49,16 @@ module Miteru
63
49
  "It might contain #{noun}: #{filename_with_sizes}."
64
50
  end
65
51
 
52
+ def links
53
+ (href_links + possible_file_links).compact.uniq
54
+ end
55
+
66
56
  private
67
57
 
68
58
  def response
69
59
  @response ||= get
70
60
  end
71
61
 
72
- def check(url)
73
- res = HTTPClient.head(url)
74
- res.status.success?
75
- rescue StandardError
76
- false
77
- end
78
-
79
62
  def get
80
63
  HTTPClient.get url
81
64
  end
@@ -90,12 +73,31 @@ module Miteru
90
73
  nil
91
74
  end
92
75
 
93
- def links
94
- if doc
95
- doc.css("a").map { |a| a.get("href") }.compact
76
+ def href_links
77
+ if doc && ok? && index?
78
+ doc.css("a").map { |a| a.get("href") }.compact.map do |href|
79
+ href = href.start_with?("/") ? href : "/#{href}"
80
+ url + href
81
+ end
96
82
  else
97
83
  []
98
84
  end
85
+ rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
86
+ []
87
+ end
88
+
89
+ def possible_file_links
90
+ uri = URI.parse(url)
91
+
92
+ segments = uri.path.split("/")
93
+ return [] if segments.length.zero?
94
+
95
+ last = segments.last
96
+ VALID_EXTENSIONS.map do |ext|
97
+ new_segments = segments[0..-2] + ["#{last}#{ext}"]
98
+ uri.path = new_segments.join("/")
99
+ uri.to_s
100
+ end
99
101
  end
100
102
  end
101
103
  end
@@ -30,14 +30,14 @@ Gem::Specification.new do |spec|
30
30
  spec.add_development_dependency "rake", "~> 13.0"
31
31
  spec.add_development_dependency "rspec", "~> 3.9"
32
32
  spec.add_development_dependency "vcr", "~> 6.0"
33
- spec.add_development_dependency "webmock", "~> 3.8"
33
+ spec.add_development_dependency "webmock", "~> 3.9"
34
34
 
35
35
  spec.add_dependency "colorize", "~> 0.8"
36
- spec.add_dependency "down", "~> 5.1"
36
+ spec.add_dependency "down", "~> 5.2"
37
37
  spec.add_dependency "http", "~> 4.4"
38
- spec.add_dependency "oga", "~> 3.2"
38
+ spec.add_dependency "oga", "~> 3.3"
39
39
  spec.add_dependency "parallel", "~> 1.19"
40
40
  spec.add_dependency "slack-notifier", "~> 2.3"
41
41
  spec.add_dependency "thor", "~> 1.0"
42
- spec.add_dependency "urlscan", "~> 0.5"
42
+ spec.add_dependency "urlscan", "~> 0.6"
43
43
  end
@@ -0,0 +1,5 @@
1
+ {
2
+ "extends": [
3
+ "config:base"
4
+ ]
5
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.2
4
+ version: 0.14.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-06-23 00:00:00.000000000 Z
11
+ date: 2020-10-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -100,14 +100,14 @@ dependencies:
100
100
  requirements:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: '3.8'
103
+ version: '3.9'
104
104
  type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: '3.8'
110
+ version: '3.9'
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: colorize
113
113
  requirement: !ruby/object:Gem::Requirement
@@ -128,14 +128,14 @@ dependencies:
128
128
  requirements:
129
129
  - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '5.1'
131
+ version: '5.2'
132
132
  type: :runtime
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '5.1'
138
+ version: '5.2'
139
139
  - !ruby/object:Gem::Dependency
140
140
  name: http
141
141
  requirement: !ruby/object:Gem::Requirement
@@ -156,14 +156,14 @@ dependencies:
156
156
  requirements:
157
157
  - - "~>"
158
158
  - !ruby/object:Gem::Version
159
- version: '3.2'
159
+ version: '3.3'
160
160
  type: :runtime
161
161
  prerelease: false
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
164
  - - "~>"
165
165
  - !ruby/object:Gem::Version
166
- version: '3.2'
166
+ version: '3.3'
167
167
  - !ruby/object:Gem::Dependency
168
168
  name: parallel
169
169
  requirement: !ruby/object:Gem::Requirement
@@ -212,14 +212,14 @@ dependencies:
212
212
  requirements:
213
213
  - - "~>"
214
214
  - !ruby/object:Gem::Version
215
- version: '0.5'
215
+ version: '0.6'
216
216
  type: :runtime
217
217
  prerelease: false
218
218
  version_requirements: !ruby/object:Gem::Requirement
219
219
  requirements:
220
220
  - - "~>"
221
221
  - !ruby/object:Gem::Version
222
- version: '0.5'
222
+ version: '0.6'
223
223
  description: An experimental phishing kit detector
224
224
  email:
225
225
  - manabu.niseki@gmail.com
@@ -259,12 +259,13 @@ files:
259
259
  - lib/miteru/version.rb
260
260
  - lib/miteru/website.rb
261
261
  - miteru.gemspec
262
+ - renovate.json
262
263
  - screenshots/slack.png
263
264
  homepage: https://github.com/ninoseki/miteru
264
265
  licenses:
265
266
  - MIT
266
267
  metadata: {}
267
- post_install_message:
268
+ post_install_message:
268
269
  rdoc_options: []
269
270
  require_paths:
270
271
  - lib
@@ -280,7 +281,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
280
281
  version: '0'
281
282
  requirements: []
282
283
  rubygems_version: 3.1.2
283
- signing_key:
284
+ signing_key:
284
285
  specification_version: 4
285
286
  summary: An experimental phishing kit detector
286
287
  test_files: []