miteru 0.14.2 → 0.14.3

This diff has not been reviewed by any users.
Log in in order to be able to vote.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: acafc5c390603cb4e035ba592a47291eb5b93f20c1a6f4c12dbf22f40b15f3b4
4
- data.tar.gz: bc8e05d8356ed633c45c1c241abb9972c79a85edab8f5d555cfce740d72f938f
3
+ metadata.gz: 95bccae4b2e34a97963be17cea1b90180ab18278b07be56db5bbc9e1755f673a
4
+ data.tar.gz: 0cf9b0015a178f037c802cd88da68828d6db7618cd35a4fc4e9a6dd9f5030703
5
5
  SHA512:
6
- metadata.gz: e191d8815c1eda041a9c64e2ef5c62a16da248ff5a19cbcd419ccdfa956963e0ed1177e83193c45df71971f8717ecd558a8a0b0d69a0d0ac8c7a2a4c9463ba87
7
- data.tar.gz: b098a5efaa9eb18a618a5c3a42a0ce7b584ad37d39c7f154cc5afed9a11f5c0c2750215efa83b7d8a4e0109e32399dc37978da2a1bcf4aa2ff9c21f7454d974f
6
+ metadata.gz: 65ebef82c94e969c4886a34bab6241aea8c6a495a041a10a8df7af7429d77a0eda03f233df8ae56b904542d649c8a9bb549e6ceeb4007b5f19ad2616407b20ff
7
+ data.tar.gz: 2d145f705bd18c0219695ae83aeff01bccfeb3fc34dc04a0a9dcf9374a9b839bd43762d76501430509baac8598895df088e35dcd24a5c846e5962072ce00ec40
@@ -28,6 +28,8 @@ module Miteru
28
28
  # @return [Boolean]
29
29
  attr_accessor :verbose
30
30
 
31
+ attr_reader :valid_extensions
32
+
31
33
  def initialize
32
34
  @auto_download = false
33
35
  @ayashige = false
@@ -37,6 +39,8 @@ module Miteru
37
39
  @size = 100
38
40
  @threads = Parallel.processor_count
39
41
  @verbose = false
42
+
43
+ @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
40
44
  end
41
45
 
42
46
  def auto_download?
@@ -10,7 +10,6 @@ require_relative "./feeds/urlscan_pro"
10
10
  module Miteru
11
11
  class Feeds
12
12
  IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
13
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
14
13
 
15
14
  def initialize
16
15
  @feeds = [
@@ -49,18 +48,7 @@ module Miteru
49
48
  segments = uri.path.split("/")
50
49
  return [base] if segments.length.zero?
51
50
 
52
- urls = (0...segments.length).map do |idx|
53
- breakdowned_url = "#{base}#{segments[0..idx].join('/')}"
54
- breakdown = [breakdowned_url]
55
- if idx > 0 && idx < segments.length
56
- next if segments[idx].nil? || invalid_extension?(segments[idx])
57
-
58
- VALID_EXTENSIONS.each do |ext|
59
- breakdown << "#{base}#{segments[0..idx - 1].join('/')}/#{segments[idx]}#{ext}"
60
- end
61
- end
62
- breakdown
63
- end.flatten.compact
51
+ urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
64
52
 
65
53
  urls.reject do |breakdowned_url|
66
54
  # Reject a url which ends with specific extension names
@@ -18,7 +18,7 @@ module Miteru
18
18
  end
19
19
 
20
20
  def download(url, destination)
21
- down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
21
+ down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
22
22
  down.download(url, destination: destination)
23
23
  destination
24
24
  end
@@ -5,37 +5,32 @@ require "securerandom"
5
5
 
6
6
  module Miteru
7
7
  class Kit
8
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
8
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
9
9
 
10
- attr_reader :base_url, :link
10
+ attr_reader :url
11
11
 
12
- def initialize(base_url:, link:)
13
- @base_url = base_url
14
- @link = link.start_with?("/") ? link[1..-1] : link
12
+ def initialize(url)
13
+ @url = url
15
14
  end
16
15
 
17
- def valid?
18
- VALID_EXTENSIONS.include? extname
16
+ def valid?;
17
+ valid_ext? && reachable?
19
18
  end
20
19
 
21
20
  def extname
22
- return ".tar.gz" if link.end_with?("tar.gz")
21
+ return ".tar.gz" if url.end_with?("tar.gz")
23
22
 
24
- File.extname(link)
23
+ File.extname(url)
25
24
  end
26
25
 
27
26
  def basename
28
- File.basename(link)
27
+ File.basename(url)
29
28
  end
30
29
 
31
30
  def filename
32
31
  CGI.unescape basename
33
32
  end
34
33
 
35
- def url
36
- "#{base_url}/#{basename}"
37
- end
38
-
39
34
  def download_filepath
40
35
  "#{base_dir}/#{download_filename}"
41
36
  end
@@ -59,7 +54,7 @@ module Miteru
59
54
  end
60
55
 
61
56
  def hostname
62
- URI(base_url).hostname
57
+ URI(url).hostname
63
58
  end
64
59
 
65
60
  def download_filename
@@ -69,5 +64,16 @@ module Miteru
69
64
  def base_dir
70
65
  @base_dir ||= Miteru.configuration.download_to
71
66
  end
67
+
68
+ def valid_ext?
69
+ VALID_EXTENSIONS.include? extname
70
+ end
71
+
72
+ def reachable?
73
+ res = HTTPClient.head(url)
74
+ res.status.success?
75
+ rescue StandardError
76
+ false
77
+ end
72
78
  end
73
79
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.14.2"
4
+ VERSION = "0.14.3"
5
5
  end
@@ -4,9 +4,10 @@ require "oga"
4
4
 
5
5
  module Miteru
6
6
  class Website
7
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
7
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
8
8
 
9
9
  attr_reader :url
10
+
10
11
  def initialize(url)
11
12
  @url = url
12
13
  end
@@ -16,25 +17,12 @@ module Miteru
16
17
  end
17
18
 
18
19
  def kits
19
- if ext?
20
- return [] unless check(url)
21
-
22
- link = url.split("/").last
23
- base_url = url.split("/")[0..-2].join("/")
24
- kit = Kit.new(base_url: base_url, link: link)
25
- return kit.valid? ? [kit] : []
26
- end
27
-
28
- links.map do |link|
29
- kit = Kit.new(base_url: url, link: link.to_s)
20
+ @kits ||= links.map do |link|
21
+ kit = Kit.new(link)
30
22
  kit.valid? ? kit : nil
31
23
  end.compact
32
24
  end
33
25
 
34
- def ext?
35
- VALID_EXTENSIONS.any? { |ext| url.end_with?(ext) }
36
- end
37
-
38
26
  def ok?
39
27
  response.code == 200
40
28
  end
@@ -48,9 +36,7 @@ module Miteru
48
36
  end
49
37
 
50
38
  def has_kits?
51
- return kits? if ext?
52
-
53
- ok? && index? && kits?
39
+ kits?
54
40
  rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
55
41
  false
56
42
  end
@@ -63,19 +49,16 @@ module Miteru
63
49
  "It might contain #{noun}: #{filename_with_sizes}."
64
50
  end
65
51
 
52
+ def links
53
+ (href_links + possible_file_links).compact.uniq
54
+ end
55
+
66
56
  private
67
57
 
68
58
  def response
69
59
  @response ||= get
70
60
  end
71
61
 
72
- def check(url)
73
- res = HTTPClient.head(url)
74
- res.status.success?
75
- rescue StandardError
76
- false
77
- end
78
-
79
62
  def get
80
63
  HTTPClient.get url
81
64
  end
@@ -90,12 +73,31 @@ module Miteru
90
73
  nil
91
74
  end
92
75
 
93
- def links
94
- if doc
95
- doc.css("a").map { |a| a.get("href") }.compact
76
+ def href_links
77
+ if doc && ok? && index?
78
+ doc.css("a").map { |a| a.get("href") }.compact.map do |href|
79
+ href = href.start_with?("/") ? href : "/#{href}"
80
+ url + href
81
+ end
96
82
  else
97
83
  []
98
84
  end
85
+ rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
86
+ []
87
+ end
88
+
89
+ def possible_file_links
90
+ uri = URI.parse(url)
91
+
92
+ segments = uri.path.split("/")
93
+ return [] if segments.length.zero?
94
+
95
+ last = segments.last
96
+ VALID_EXTENSIONS.map do |ext|
97
+ new_segments = segments[0..-2] + ["#{last}#{ext}"]
98
+ uri.path = new_segments.join("/")
99
+ uri.to_s
100
+ end
99
101
  end
100
102
  end
101
103
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.2
4
+ version: 0.14.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-06-23 00:00:00.000000000 Z
11
+ date: 2020-06-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -264,7 +264,7 @@ homepage: https://github.com/ninoseki/miteru
264
264
  licenses:
265
265
  - MIT
266
266
  metadata: {}
267
- post_install_message:
267
+ post_install_message:
268
268
  rdoc_options: []
269
269
  require_paths:
270
270
  - lib
@@ -280,7 +280,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
280
280
  version: '0'
281
281
  requirements: []
282
282
  rubygems_version: 3.1.2
283
- signing_key:
283
+ signing_key:
284
284
  specification_version: 4
285
285
  summary: An experimental phishing kit detector
286
286
  test_files: []