miteru 0.12.1 → 0.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c06dbaa1d46bc20c70d44bf5a3fda286e45a9f5bdb5046e9c900e3c6aa5e86a5
-  data.tar.gz: c512cddc5e3c53875e9cc0c0c7db5b61e45fb301ce7cac2a0ab5b46167729df4
+  metadata.gz: d0345bd419d2af64ac26074a1a586ed961a6dce64bcb0043ee142cac6cfe99f7
+  data.tar.gz: 53a828e7e50fcd83b5005eed831b29e1282f743729f2f57319dad21f51a53e59
 SHA512:
-  metadata.gz: 7321f78b8d68ce7434f9aaf29b4d781b0cac49ccf48b6a4f9944e6154c55ad0185bc7fe91625a46d7dece574f121d41523df63e761efa264fd4866370c85ccbd
-  data.tar.gz: 4589f29b656ecec1fe9cc03cad0395b49b5f0a9bf674fb36bbcf13b4b2ebfa6cb0601f1d80ea1b94a2fd9f5d33f23b3654c6bc5852a810977da93dcf39f6a2f6
+  metadata.gz: 4c43b49597aa13659906b5eb10cc151bfb04ae89e64fa62cd18e682e978f6330bd256cac3e2c168ca6d8c8b43f2b5194d2ea16e123875e3b607c603cb1a15146
+  data.tar.gz: 31099a1446e773873327f1e9d51141f24386a6f0ea25d4502fd707c050a166c8406ca1528bca7cfb7b16670289c5dda689dec8764c5d83191b970118819dd843

data/README.md

@@ -44,7 +44,7 @@ Usage:
   miteru execute
 
 Options:
-  [--auto-download], [--no-auto-download]              # Enable or disable auto-download of compressed file(s)
+  [--auto-download], [--no-auto-download]              # Enable or disable auto-download of phishing kits
   [--directory-traveling], [--no-directory-traveling]  # Enable or disable directory traveling
   [--download-to=DOWNLOAD_TO]                          # Directory to download file(s)
                                                        # Default: /tmp
@@ -81,9 +81,7 @@ $ docker run -v /tmp:/tmp miteru execute --auto-download
 
 ## Aasciinema cast
 
-[![asciicast](https://asciinema.org/a/ga6ZbwuK1HOLOyELb23QrSvJP.svg)](https://asciinema.org/a/ga6ZbwuK1HOLOyELb23QrSvJP)
-
-Note: In this demo, I stopped the process during the execution because it takes minutes to finish.
+[![asciicast](https://asciinema.org/a/hHpkHhMLiiv17gmdRhVMtZWwM.svg)](https://asciinema.org/a/hHpkHhMLiiv17gmdRhVMtZWwM)
 
 ## Note
 

data/lib/miteru/crawler.rb

@@ -33,7 +33,7 @@ module Miteru
         website = Website.new(url)
         downloader.download_kits(website.kits) if website.has_kits? && auto_download?
         notify(website) if verbose || website.has_kits?
-      rescue OpenSSL::SSL::SSLError, HTTP::Error, LL::ParserError, Addressable::URI::InvalidURIError => _
+      rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _
         next
       end
     end

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "0.12.1"
+  VERSION = "0.12.2"
 end

data/lib/miteru/website.rb

@@ -10,12 +10,11 @@ module Miteru
     end
 
     def title
-      doc.at_css("title")&.text
+      doc&.at_css("title")&.text
     end
 
     def kits
-      @kits ||= doc.css("a").map do |a|
-        link = a.get("href")
+      @kits ||= links.map do |link|
         kit = Kit.new(base_url: url, link: link.to_s)
         kit.valid? ? kit : nil
       end.compact
@@ -35,7 +34,7 @@ module Miteru
 
     def has_kits?
       ok? && index? && kits?
-    rescue OpenSSL::SSL::SSLError, HTTP::Error, LL::ParserError, Addressable::URI::InvalidURIError => _
+    rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _
       false
     end
 
@@ -55,7 +54,21 @@ module Miteru
     end
 
     def doc
-      @doc ||= Oga.parse_html(response.body.to_s)
+      @doc ||= parse_html(response.body.to_s)
+    end
+
+    def parse_html(html)
+      Oga.parse_html(html)
+    rescue ArgumentError, LL::ParserError => _
+      nil
+    end
+
+    def links
+      if doc
+        doc.css("a").map { |a| a.get("href") }.compact
+      else
+        []
+      end
     end
   end
 end

data/miteru.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "down", "~> 4.8"
   spec.add_dependency "http", "~> 4.1"
   spec.add_dependency "oga", "~> 2.15"
-  spec.add_dependency "parallel", "~> 1.16"
+  spec.add_dependency "parallel", "~> 1.17"
   spec.add_dependency "slack-incoming-webhooks", "~> 0.2"
   spec.add_dependency "thor", "~> 0.19"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.12.2
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-30 00:00:00.000000000 Z
+date: 2019-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '1.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '1.17'
 - !ruby/object:Gem::Dependency
   name: slack-incoming-webhooks
   requirement: !ruby/object:Gem::Requirement