miteru 0.12.4 → 0.12.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e9e6b96b39d329d3003d0cec2d06f99814f7f9fb518cb80f8c465999a548e88f
-  data.tar.gz: 5fb646c603f612f09de5c9da2cce8a0c3b379af7d9e3a31994ea2403da37e35a
+  metadata.gz: 652b5f4162ff0a7c62f95123c06b82f5f3c6c469442af3f9087767342550afb1
+  data.tar.gz: ef33a506c434a06bdf7c098b460d4174e76b0b813044b507aba5dccfe31e3438
 SHA512:
-  metadata.gz: 173f0e24dba5aee1e5cfb126bc70d2e7ae0e7ebf13c59a55f60a023ce9705e84e86f645770c27f5026a48029124355ffa54e68f0b0dfc1675b397f7e6c9544ca
-  data.tar.gz: c8c46730a108355c70696b2034cfd4bbf0a71c144e295c774cc214abb01600e97ecfb6747114bcec904a0852eb5db56fbe7b35aa45d4244f2763e4dff0cf0223
+  metadata.gz: b106d1f83405f51116ba9c0bcaaac1a2b7f0b8df524164d829755b3c7d82dd7dd68f2eb0ef4ea5da2d826615428731d92008f4c0313d716c6b674f7e5f905ca6
+  data.tar.gz: 35539e25834188e5ec42124ee7ee00791f08fd6993ede104de88eeaafb7f7f4990558e8672faa918442d77b5fe220924f04252a7033876a592a278a1947e6e14

data/lib/miteru/crawler.rb

@@ -21,7 +21,7 @@ module Miteru
       @threads = threads
       @verbose = verbose
 
-      @feeds = Feeds.new(size, directory_traveling: directory_traveling)
+      @feeds = Feeds.new(size: size, directory_traveling: directory_traveling)
       @notifier = Notifier.new(post_to_slack)
     end
 

data/lib/miteru/downloader.rb

@@ -10,29 +10,31 @@ module Miteru
 
     def initialize(base_dir = "/tmp")
       @base_dir = base_dir
-      raise ArgumentError, "#{base_dir} is not existing." unless Dir.exist?(base_dir)
+      raise ArgumentError, "#{base_dir} is not exist." unless Dir.exist?(base_dir)
     end
 
     def download_kits(kits)
-      kits.each do |kit|
-        filename = download_filename(kit)
-        destination = filepath_to_download(filename)
-        begin
-          downloaded_filepath = HTTPClient.download(kit.url, destination)
-          if duplicated?(downloaded_filepath)
-            puts "Do not download #{kit.url} because there is a file that has a same hash value in the directory (SHA256: #{sha256(downloaded_filepath)})."
-            FileUtils.rm downloaded_filepath
-          else
-            puts "Download #{kit.url} as #{downloaded_filepath}"
-          end
-        rescue Down::Error => e
-          puts "Failed to download: #{kit.url} (#{e})"
-        end
-      end
+      kits.each { |kit| download_kit kit }
     end
 
     private
 
+    def download_kit(kit)
+      filename = download_filename(kit)
+      destination = filepath_to_download(filename)
+      begin
+        downloaded_filepath = HTTPClient.download(kit.url, destination)
+        if duplicated?(downloaded_filepath)
+          puts "Do not download #{kit.url} because there is a duplicate file in the directory (SHA256: #{sha256(downloaded_filepath)})."
+          FileUtils.rm downloaded_filepath
+        else
+          puts "Download #{kit.url} as #{downloaded_filepath}"
+        end
+      rescue Down::Error => e
+        puts "Failed to download: #{kit.url} (#{e})"
+      end
+    end
+
     def download_filename(kit)
       domain = URI(kit.base_url).hostname
 

data/lib/miteru/feeds.rb

@@ -6,16 +6,18 @@ require_relative "./feeds/urlscan"
 
 module Miteru
   class Feeds
-    attr_reader :directory_traveling
-
-    def initialize(urlscan_size = 100, directory_traveling: false)
+    def initialize(size: 100, directory_traveling: false)
       @feeds = [
         Ayashige.new,
-        UrlScan.new(urlscan_size)
+        UrlScan.new(size)
       ]
       @directory_traveling = directory_traveling
     end
 
+    def directory_traveling?
+      @directory_traveling
+    end
+
     def suspicious_urls
       @suspicious_urls ||= [].tap do |arr|
         urls = @feeds.map do |feed|
@@ -34,7 +36,7 @@ module Miteru
       end
 
       base = "#{uri.scheme}://#{uri.hostname}"
-      return [base] unless directory_traveling
+      return [base] unless directory_traveling?
 
       segments = uri.path.split("/")
       return [base] if segments.length.zero?

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.12.4"
+  VERSION = "0.12.5"
 end

data/lib/miteru/website.rb

@@ -34,7 +34,7 @@ module Miteru
 
     def has_kits?
       ok? && index? && kits?
-    rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
+    rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError, Encoding::CompatibilityError, LL::ParserError => _e
       false
     end
 
@@ -42,8 +42,8 @@ module Miteru
       return "It doesn't contain a phishing kit." unless kits?
 
       kit_names = kits.map(&:basename).join(", ")
-      noun = kits.length == 1 ? "kit" : "kits"
-      "It might contain phishing #{noun}: #{kit_names}."
+      noun = kits.length == 1 ? "a phishing kit" : "phishing kits"
+      "It might contain #{noun}: #{kit_names}."
     end
 
     private

data/miteru.gemspec

@@ -29,8 +29,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 12.3"
   spec.add_development_dependency "rspec", "~> 3.8"
-  spec.add_development_dependency "vcr", "~> 4.0"
-  spec.add_development_dependency "webmock", "~> 3.5"
+  spec.add_development_dependency "vcr", "~> 5.0"
+  spec.add_development_dependency "webmock", "~> 3.6"
 
   spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "down", "~> 4.8"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.12.4
+  version: 0.12.5
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-03 00:00:00.000000000 Z
+date: 2019-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,28 +86,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.6'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement