web_stat 0.3.8 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bbf065f5482ee7bf68e14e41af1e48b855f4219b9f44a7e16440834c92f404d5
-  data.tar.gz: a16fc8b8a66cb534bb8ce61c6da28212faff44f2df6324772261f9b6847cfc7f
+  metadata.gz: 6021c6bacb394385b1e5634449f4a3adfb27a5c2d799f9d0874084889a3ac66c
+  data.tar.gz: d35180f95d0937b69df93dc5ba051edb8eeb79b52bd6df0e8770c849bc33c511
 SHA512:
-  metadata.gz: 83b8b7d71a1047ec2614a9bfbc3c6305806cf7a616ba6b2568382b72ef91f5b0947e7aefb3d926b33c4970d291d2a34c99a86ba0f0e320a3e7fde9dd2b5bc154
-  data.tar.gz: aa4af6063d7dd81c7c78c0100ff5354d7c54f28d7b2fac7393656cb5bde932b7260c95d1fe08fc560784e85ead7bbf49134c4d56cb108cef522cef753e477032
+  metadata.gz: 812aa33c7a4d8642b1239d05b8a0f2fa6bc938379f48e998732d71dcdd40c5c15d04b2795dde6b7e9cee0f439a12ad5b7259bc58fe741076106108bbc6c16d90
+  data.tar.gz: 3bf21e030a43868ccee4c45fe1e8993c6375b85f05d74ec3b11d68b97fe10a09194e186cfd942c3fe03f9ad2ce009e5ed15edefb06d21830180f9b04bd2fd8ea

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    web_stat (0.3.7)
+    web_stat (0.3.12)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
@@ -83,7 +83,7 @@ GEM
       nokogiri (>= 1.6.0)
     rubyzip (2.3.0)
     safe_yaml (1.0.5)
-    sanitize (5.2.1)
+    sanitize (5.2.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)

data/README.md

@@ -55,12 +55,10 @@ And then execute:
 
 ### spec
 
-  $ bundle exec rake spec
-  
-or
-
-  $ bundle exec rspec
+  $ docker/start -d
+  $ docker/exec ENV=development bundle exec rspec
 
 Test a file
 
-  $ bundle exec rspec spec/web_stat/fetch_spec.rb 
+  $ docker/start -d
+  $ docker/exec ENV=development bundle exec rspec spec/web_stat/fetch_spec.rb 

data/lib/helpers/web_drive_helper.rb

@@ -5,14 +5,29 @@ module WebStat
       # @param [String] url
       # @param [Integer] delay
       def get_last_url(url, delay=nil)
+        driver = get_driver(url, delay)
+        last_url = driver.current_url
+        driver.quit
+        last_url
+      end
+      # Get source of html
+      # @param [String] url
+      # @param [Integer] delay
+      def get_source(url, delay=nil)
+        driver = get_driver(url, delay)
+        source = driver.page_source
+        driver.quit
+        source
+      end
+      
+      private
+      def get_driver(url, delay=nil)
         Selenium::WebDriver.logger.output = File.join("/tmp", "selenium.log")
         Selenium::WebDriver.logger.level = :info
         options = Selenium::WebDriver::Chrome::Options.new(args: [
           'headless',
           'no-sandbox',
-          'disable-gpu',
-          'start-maximized',
-          'window-size=1920,1080'
+          'disable-gpu'
           ])
         driver = Selenium::WebDriver.for(:chrome, options: options)
         driver.manage.timeouts.implicit_wait = 10
@@ -21,9 +36,10 @@ module WebStat
         if delay.is_a?(Integer)
           sleep delay
         end
-        last_url = driver.current_url
+        driver
+      rescue => e
         driver.quit
-        last_url
+        raise e
       end
     end
   end

data/lib/web_stat/configure.rb

@@ -9,7 +9,7 @@ module WebStat
         if defined? Rails
           YAML.load_file(get_configure_path)[Rails.env]
         else
-          YAML.load_file(get_configure_path)["production"]
+          YAML.load_file(get_configure_path)[ENV["ENV"] || "production"]
         end
       end
       

data/lib/web_stat/fetch.rb

@@ -47,8 +47,12 @@ module WebStat
           break
         end
       end
-      if path.nil? || path.empty? || @nokogiri.at('body').xpath('//img').first
-        path = @nokogiri.at('body').xpath('//img').first.attr('src')
+      readability_content = Readability::Document.new(@nokogiri.at('body')).content
+      if (path.nil? || path.empty?) && readability_content.xpath('//img').first
+        path = ::Nokogiri::HTML(readability_content).xpath('//img').first.attr('src')
+      end
+      if (path.nil? || path.empty?) && @nokogiri.xpath('//img').first
+        path = @nokogiri.xpath('//img').first.attr('src')
       end
       if ! path.nil? && path.match(/^\//)
         "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
@@ -86,7 +90,7 @@ module WebStat
           raise Mechanize::RobotsDisallowedError.new(url)
         end
         if WebStat::Configure.get["use_chromedirver"]
-          document = WebStat::WebDriverHelper.get_last_url(url)
+          body = WebStat::WebDriverHelper.get_source(url)
           @status = 200
         else
           document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})

data/lib/web_stat/final_redirect_url.rb

@@ -11,7 +11,7 @@ module WebStat
             redirect_lookup_depth = options[:depth].to_i > 0 ? options[:depth].to_i : 10
             response_uri = get_final_redirect_url(url, redirect_lookup_depth)
             final_url =  url_string_from_uri(response_uri)
-          rescue Exception => ex
+          rescue => e
             # nothing
           end
         end

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.3.8"
+  VERSION = "0.3.13"
 end

data/spec/spec_helper.rb

@@ -7,12 +7,7 @@ require "web_stat"
 require 'webmock'
 include WebMock::API
 WebMock.enable!
-
-WebMock.disable_net_connect!({
-  allow_localhost: true,
-  allow: 'chromedriver.storage.googleapis.com'
-})
-
+  
 RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
   config.example_status_persistence_file_path = ".rspec_status"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.3.13
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-17 00:00:00.000000000 Z
+date: 2020-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler