web_stat 0.3.7 → 0.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5304507f8ce01a6a98717756927366005db0eceb89b3d22f29e4283c6eedb4d7
-  data.tar.gz: f561221644f98831867c2d853201a0312dcb75674ca4ae247b319b4f2a1ba085
+  metadata.gz: bbf065f5482ee7bf68e14e41af1e48b855f4219b9f44a7e16440834c92f404d5
+  data.tar.gz: a16fc8b8a66cb534bb8ce61c6da28212faff44f2df6324772261f9b6847cfc7f
 SHA512:
-  metadata.gz: bec01871973e80dc46b9246a264791213b3bdd4cb20693fc0d51cc28c2d9bba28b40af65322dc2b0c950e9bb5aedd288ff8fd771adfbd5b2aee1564b98888cb0
-  data.tar.gz: 5cfce031ad779f347fae362ccff2deadb3711e4f9fcc06cafcd232b86e8c18aff63e40c430a330cfdf00d398aa5257457c5dc9f3bee492478359d3801e217f08
+  metadata.gz: 83b8b7d71a1047ec2614a9bfbc3c6305806cf7a616ba6b2568382b72ef91f5b0947e7aefb3d926b33c4970d291d2a34c99a86ba0f0e320a3e7fde9dd2b5bc154
+  data.tar.gz: aa4af6063d7dd81c7c78c0100ff5354d7c54f28d7b2fac7393656cb5bde932b7260c95d1fe08fc560784e85ead7bbf49134c4d56cb108cef522cef753e477032

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    web_stat (0.3.6)
+    web_stat (0.3.7)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
@@ -83,7 +83,7 @@ GEM
       nokogiri (>= 1.6.0)
     rubyzip (2.3.0)
     safe_yaml (1.0.5)
-    sanitize (5.2.0)
+    sanitize (5.2.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)

data/lib/web_stat/fetch.rb

@@ -47,7 +47,7 @@ module WebStat
           break
         end
       end
-      if path.nil? || path.empty?
+      if path.nil? || path.empty? || @nokogiri.at('body').xpath('//img').first
         path = @nokogiri.at('body').xpath('//img').first.attr('src')
       end
       if ! path.nil? && path.match(/^\//)
@@ -78,17 +78,25 @@ module WebStat
     # @param [String] url
     # @param [String] body
     def get_url(url)
-      agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
+      mech = Mechanize.new { |_mech| _mech.user_agent = WebStat::Configure.get["user_agent"] }
       # Enable to read Robots.txt
-      agent.robots = true
+      mech.robots = true
       begin
-        document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
-        if document.class == Mechanize::File
-          body = document.body
+        if mech.agent.robots_disallowed?(url)
+          raise Mechanize::RobotsDisallowedError.new(url)
+        end
+        if WebStat::Configure.get["use_chromedirver"]
+          document = WebStat::WebDriverHelper.get_last_url(url)
+          @status = 200
         else
-          body = document.body.encode('UTF-8', document.encoding)
+          document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})
+          if document.class == Mechanize::File
+            body = document.body
+          else
+            body = document.body.encode('UTF-8', document.encoding)
+          end
+          @status = document.code
         end
-        @status = document.code
       rescue Mechanize::ResponseCodeError => e
         body = e.page.body
         @status = e.page.code

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.3.7"
+  VERSION = "0.3.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.8
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-14 00:00:00.000000000 Z
+date: 2020-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler