miteru 0.12.2 → 0.12.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d0345bd419d2af64ac26074a1a586ed961a6dce64bcb0043ee142cac6cfe99f7
-  data.tar.gz: 53a828e7e50fcd83b5005eed831b29e1282f743729f2f57319dad21f51a53e59
+  metadata.gz: 92dd51ba5b114cdc84e97aa356c235c9f534332cc1d17c0fd7ebdffc9f9d1d02
+  data.tar.gz: 226ae4c89212677e7cc4a99e86ce978f7ee1b1f1c050732aa89ae307a8c3533b
 SHA512:
-  metadata.gz: 4c43b49597aa13659906b5eb10cc151bfb04ae89e64fa62cd18e682e978f6330bd256cac3e2c168ca6d8c8b43f2b5194d2ea16e123875e3b607c603cb1a15146
-  data.tar.gz: 31099a1446e773873327f1e9d51141f24386a6f0ea25d4502fd707c050a166c8406ca1528bca7cfb7b16670289c5dda689dec8764c5d83191b970118819dd843
+  metadata.gz: c04bc650fdd9adaedafd9a0cd99374966ec94c8bbb1f2e61c0cb175f5836fdeea7f78218480dbef26bcbd73c703695c808d4b4f6e2bf5bd61ba841c86480c8f1
+  data.tar.gz: 2b774f64092138fbe2119b2439a23a3710ae461b628da07b7a59b35c11f67bd9ef6bb035d5c85486e63fb02154c44a84d7557f3eaff26b986f4595edcf25b76f

data/lib/miteru/cli.rb

@@ -9,7 +9,7 @@ module Miteru
     method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
     method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
     method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 10,000)"
-    method_option :threads, type: :numeric, default: 10, desc: "Number of threads to use"
+    method_option :threads, type: :numeric, desc: "Number of threads to use"
     method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"
     def execute

data/lib/miteru/crawler.rb

@@ -6,7 +6,6 @@ require "uri"
 
 module Miteru
   class Crawler
-    attr_reader :auto_download
     attr_reader :directory_traveling
     attr_reader :downloader
     attr_reader :feeds
@@ -14,7 +13,7 @@ module Miteru
     attr_reader :threads
     attr_reader :verbose
 
-    def initialize(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
+    def initialize(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: Parallel.processor_count, verbose: false)
       @auto_download = auto_download
       @directory_traveling = directory_traveling
       @downloader = Downloader.new(download_to)
@@ -27,18 +26,18 @@ module Miteru
     end
 
     def execute
-      puts "Loaded #{feeds.suspicious_urls.length} URLs to crawl." if verbose
+      puts "Loaded #{feeds.suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose
 
       Parallel.each(feeds.suspicious_urls, in_threads: threads) do |url|
         website = Website.new(url)
         downloader.download_kits(website.kits) if website.has_kits? && auto_download?
         notify(website) if verbose || website.has_kits?
-      rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _
+      rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
         next
       end
     end
 
-    def self.execute(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
+    def self.execute(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: Parallel.processor_count, verbose: false)
       new(
         auto_download: auto_download,
         directory_traveling: directory_traveling,

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.12.2"
+  VERSION = "0.12.3"
 end

data/lib/miteru/website.rb

@@ -34,13 +34,16 @@ module Miteru
 
     def has_kits?
       ok? && index? && kits?
-    rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _
+    rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
       false
     end
 
     def message
+      return "It doesn't contain a phishing kit." unless kits?
+
       kit_names = kits.map(&:basename).join(", ")
-      kits? ? "it might contain phishing kit(s): (#{kit_names})." : "it doesn't contain a phishing kit."
+      noun = kits.length == 1 ? "kit" : "kits"
+      "It might contain phishing #{noun}: (#{kit_names})."
     end
 
     private
@@ -59,7 +62,7 @@ module Miteru
 
     def parse_html(html)
       Oga.parse_html(html)
-    rescue ArgumentError, LL::ParserError => _
+    rescue ArgumentError, Encoding::CompatibilityError, LL::ParserError => _e
       nil
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.12.2
+  version: 0.12.3
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-03 00:00:00.000000000 Z
+date: 2019-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler