miteru 0.14.2 → 0.14.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: acafc5c390603cb4e035ba592a47291eb5b93f20c1a6f4c12dbf22f40b15f3b4
4
- data.tar.gz: bc8e05d8356ed633c45c1c241abb9972c79a85edab8f5d555cfce740d72f938f
3
+ metadata.gz: 95bccae4b2e34a97963be17cea1b90180ab18278b07be56db5bbc9e1755f673a
4
+ data.tar.gz: 0cf9b0015a178f037c802cd88da68828d6db7618cd35a4fc4e9a6dd9f5030703
5
5
  SHA512:
6
- metadata.gz: e191d8815c1eda041a9c64e2ef5c62a16da248ff5a19cbcd419ccdfa956963e0ed1177e83193c45df71971f8717ecd558a8a0b0d69a0d0ac8c7a2a4c9463ba87
7
- data.tar.gz: b098a5efaa9eb18a618a5c3a42a0ce7b584ad37d39c7f154cc5afed9a11f5c0c2750215efa83b7d8a4e0109e32399dc37978da2a1bcf4aa2ff9c21f7454d974f
6
+ metadata.gz: 65ebef82c94e969c4886a34bab6241aea8c6a495a041a10a8df7af7429d77a0eda03f233df8ae56b904542d649c8a9bb549e6ceeb4007b5f19ad2616407b20ff
7
+ data.tar.gz: 2d145f705bd18c0219695ae83aeff01bccfeb3fc34dc04a0a9dcf9374a9b839bd43762d76501430509baac8598895df088e35dcd24a5c846e5962072ce00ec40
@@ -28,6 +28,8 @@ module Miteru
28
28
  # @return [Boolean]
29
29
  attr_accessor :verbose
30
30
 
31
+ attr_reader :valid_extensions
32
+
31
33
  def initialize
32
34
  @auto_download = false
33
35
  @ayashige = false
@@ -37,6 +39,8 @@ module Miteru
37
39
  @size = 100
38
40
  @threads = Parallel.processor_count
39
41
  @verbose = false
42
+
43
+ @valid_extensions = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
40
44
  end
41
45
 
42
46
  def auto_download?
@@ -10,7 +10,6 @@ require_relative "./feeds/urlscan_pro"
10
10
  module Miteru
11
11
  class Feeds
12
12
  IGNORE_EXTENSIONS = %w(.htm .html .php .asp .aspx .exe .txt).freeze
13
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
14
13
 
15
14
  def initialize
16
15
  @feeds = [
@@ -49,18 +48,7 @@ module Miteru
49
48
  segments = uri.path.split("/")
50
49
  return [base] if segments.length.zero?
51
50
 
52
- urls = (0...segments.length).map do |idx|
53
- breakdowned_url = "#{base}#{segments[0..idx].join('/')}"
54
- breakdown = [breakdowned_url]
55
- if idx > 0 && idx < segments.length
56
- next if segments[idx].nil? || invalid_extension?(segments[idx])
57
-
58
- VALID_EXTENSIONS.each do |ext|
59
- breakdown << "#{base}#{segments[0..idx - 1].join('/')}/#{segments[idx]}#{ext}"
60
- end
61
- end
62
- breakdown
63
- end.flatten.compact
51
+ urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join('/')}" }
64
52
 
65
53
  urls.reject do |breakdowned_url|
66
54
  # Reject a url which ends with specific extension names
@@ -18,7 +18,7 @@ module Miteru
18
18
  end
19
19
 
20
20
  def download(url, destination)
21
- down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
21
+ down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
22
22
  down.download(url, destination: destination)
23
23
  destination
24
24
  end
@@ -5,37 +5,32 @@ require "securerandom"
5
5
 
6
6
  module Miteru
7
7
  class Kit
8
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
8
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
9
9
 
10
- attr_reader :base_url, :link
10
+ attr_reader :url
11
11
 
12
- def initialize(base_url:, link:)
13
- @base_url = base_url
14
- @link = link.start_with?("/") ? link[1..-1] : link
12
+ def initialize(url)
13
+ @url = url
15
14
  end
16
15
 
17
- def valid?
18
- VALID_EXTENSIONS.include? extname
16
+ def valid?;
17
+ valid_ext? && reachable?
19
18
  end
20
19
 
21
20
  def extname
22
- return ".tar.gz" if link.end_with?("tar.gz")
21
+ return ".tar.gz" if url.end_with?("tar.gz")
23
22
 
24
- File.extname(link)
23
+ File.extname(url)
25
24
  end
26
25
 
27
26
  def basename
28
- File.basename(link)
27
+ File.basename(url)
29
28
  end
30
29
 
31
30
  def filename
32
31
  CGI.unescape basename
33
32
  end
34
33
 
35
- def url
36
- "#{base_url}/#{basename}"
37
- end
38
-
39
34
  def download_filepath
40
35
  "#{base_dir}/#{download_filename}"
41
36
  end
@@ -59,7 +54,7 @@ module Miteru
59
54
  end
60
55
 
61
56
  def hostname
62
- URI(base_url).hostname
57
+ URI(url).hostname
63
58
  end
64
59
 
65
60
  def download_filename
@@ -69,5 +64,16 @@ module Miteru
69
64
  def base_dir
70
65
  @base_dir ||= Miteru.configuration.download_to
71
66
  end
67
+
68
+ def valid_ext?
69
+ VALID_EXTENSIONS.include? extname
70
+ end
71
+
72
+ def reachable?
73
+ res = HTTPClient.head(url)
74
+ res.status.success?
75
+ rescue StandardError
76
+ false
77
+ end
72
78
  end
73
79
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Miteru
4
- VERSION = "0.14.2"
4
+ VERSION = "0.14.3"
5
5
  end
@@ -4,9 +4,10 @@ require "oga"
4
4
 
5
5
  module Miteru
6
6
  class Website
7
- VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
7
+ VALID_EXTENSIONS = Miteru.configuration.valid_extensions
8
8
 
9
9
  attr_reader :url
10
+
10
11
  def initialize(url)
11
12
  @url = url
12
13
  end
@@ -16,25 +17,12 @@ module Miteru
16
17
  end
17
18
 
18
19
  def kits
19
- if ext?
20
- return [] unless check(url)
21
-
22
- link = url.split("/").last
23
- base_url = url.split("/")[0..-2].join("/")
24
- kit = Kit.new(base_url: base_url, link: link)
25
- return kit.valid? ? [kit] : []
26
- end
27
-
28
- links.map do |link|
29
- kit = Kit.new(base_url: url, link: link.to_s)
20
+ @kits ||= links.map do |link|
21
+ kit = Kit.new(link)
30
22
  kit.valid? ? kit : nil
31
23
  end.compact
32
24
  end
33
25
 
34
- def ext?
35
- VALID_EXTENSIONS.any? { |ext| url.end_with?(ext) }
36
- end
37
-
38
26
  def ok?
39
27
  response.code == 200
40
28
  end
@@ -48,9 +36,7 @@ module Miteru
48
36
  end
49
37
 
50
38
  def has_kits?
51
- return kits? if ext?
52
-
53
- ok? && index? && kits?
39
+ kits?
54
40
  rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
55
41
  false
56
42
  end
@@ -63,19 +49,16 @@ module Miteru
63
49
  "It might contain #{noun}: #{filename_with_sizes}."
64
50
  end
65
51
 
52
+ def links
53
+ (href_links + possible_file_links).compact.uniq
54
+ end
55
+
66
56
  private
67
57
 
68
58
  def response
69
59
  @response ||= get
70
60
  end
71
61
 
72
- def check(url)
73
- res = HTTPClient.head(url)
74
- res.status.success?
75
- rescue StandardError
76
- false
77
- end
78
-
79
62
  def get
80
63
  HTTPClient.get url
81
64
  end
@@ -90,12 +73,31 @@ module Miteru
90
73
  nil
91
74
  end
92
75
 
93
- def links
94
- if doc
95
- doc.css("a").map { |a| a.get("href") }.compact
76
+ def href_links
77
+ if doc && ok? && index?
78
+ doc.css("a").map { |a| a.get("href") }.compact.map do |href|
79
+ href = href.start_with?("/") ? href : "/#{href}"
80
+ url + href
81
+ end
96
82
  else
97
83
  []
98
84
  end
85
+ rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError, HTTP::Error, LL::ParserError, OpenSSL::SSL::SSLError => _e
86
+ []
87
+ end
88
+
89
+ def possible_file_links
90
+ uri = URI.parse(url)
91
+
92
+ segments = uri.path.split("/")
93
+ return [] if segments.length.zero?
94
+
95
+ last = segments.last
96
+ VALID_EXTENSIONS.map do |ext|
97
+ new_segments = segments[0..-2] + ["#{last}#{ext}"]
98
+ uri.path = new_segments.join("/")
99
+ uri.to_s
100
+ end
99
101
  end
100
102
  end
101
103
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miteru
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.2
4
+ version: 0.14.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Niseki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-06-23 00:00:00.000000000 Z
11
+ date: 2020-06-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -264,7 +264,7 @@ homepage: https://github.com/ninoseki/miteru
264
264
  licenses:
265
265
  - MIT
266
266
  metadata: {}
267
- post_install_message:
267
+ post_install_message:
268
268
  rdoc_options: []
269
269
  require_paths:
270
270
  - lib
@@ -280,7 +280,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
280
280
  version: '0'
281
281
  requirements: []
282
282
  rubygems_version: 3.1.2
283
- signing_key:
283
+ signing_key:
284
284
  specification_version: 4
285
285
  summary: An experimental phishing kit detector
286
286
  test_files: []