web_stat 0.3.5 → 0.3.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b35413482a93f316bbd3d99a037f0e97102544a4e04604b5add670ac1a1500a
-  data.tar.gz: 4517b7754b2096901b005c26497fa2addad572b02f08a7aed385f6dc7de2e55a
+  metadata.gz: 29a2c3d0e37310f75c009e696044495e1f1b1f3d9ee8840d08aec711b67b4a11
+  data.tar.gz: 1214f85ed921b82f964cb77f5b882c3f41a47db36188c7ce789208644918fcf7
 SHA512:
-  metadata.gz: a54c666953b0c51e1e5ea8d230069bf608e6284629740070f77f3816468553037852c0cbc6c39c35cd53000435fbced4acee6a7b0ca855f2c241e0fb769d32da
-  data.tar.gz: 3fef9c1c48f272e27c877a4a588ce45f841ad316429bb78fc3aa6de5748d6471b9e5aaed2ee076af0d3fa0779b5696c452c1b0fc2f77644c7914b171377def8e
+  metadata.gz: bad407ab3233b5a3fc11e5bb770862bd6346a42759830c66f04b9b525773c9459972d5ab78d8b86281914dfd78051898ad14c6a77d04ed38392b75dc4a3760e3
+  data.tar.gz: 2854708ae4aec9833f288ca3bece3a03a548aeaf5177582fb1ed569c85c27fd97b7897a9d080d81b8b670590e52d5b1a666eb918606c5643cc9286593ea9fd12

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    web_stat (0.3.5)
+    web_stat (0.3.10)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)

data/lib/helpers/web_drive_helper.rb

@@ -5,14 +5,29 @@ module WebStat
       # @param [String] url
       # @param [Integer] delay
       def get_last_url(url, delay=nil)
+        driver = get_driver(url, delay)
+        last_url = driver.current_url
+        driver.quit
+        last_url
+      end
+      # Get source of html
+      # @param [String] url
+      # @param [Integer] delay
+      def get_source(url, delay=nil)
+        driver = get_driver(url, delay)
+        source = driver.page_source
+        driver.quit
+        source
+      end
+      
+      private
+      def get_driver(url, delay=nil)
         Selenium::WebDriver.logger.output = File.join("/tmp", "selenium.log")
         Selenium::WebDriver.logger.level = :info
         options = Selenium::WebDriver::Chrome::Options.new(args: [
           'headless',
           'no-sandbox',
-          'disable-gpu',
-          'start-maximized',
-          'window-size=1920,1080'
+          'disable-gpu'
           ])
         driver = Selenium::WebDriver.for(:chrome, options: options)
         driver.manage.timeouts.implicit_wait = 10
@@ -21,9 +36,7 @@ module WebStat
         if delay.is_a?(Integer)
           sleep delay
         end
-        last_url = driver.current_url
-        driver.quit
-        last_url
+        driver
       end
     end
   end

data/lib/web_stat/fetch.rb

@@ -47,6 +47,9 @@ module WebStat
           break
         end
       end
+      if path.nil? || path.empty? || @nokogiri.at('body').xpath('//img').first
+        path = @nokogiri.at('body').xpath('//img').first.attr('src')
+      end
       if ! path.nil? && path.match(/^\//)
         "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
       else
@@ -75,17 +78,25 @@ module WebStat
     # @param [String] url
     # @param [String] body
     def get_url(url)
-      agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
+      mech = Mechanize.new { |_mech| _mech.user_agent = WebStat::Configure.get["user_agent"] }
       # Enable to read Robots.txt
-      agent.robots = true
+      mech.robots = true
       begin
-        document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
-        if document.class == Mechanize::File
-          body = document.body
+        if mech.agent.robots_disallowed?(url)
+          raise Mechanize::RobotsDisallowedError.new(url)
+        end
+        if WebStat::Configure.get["use_chromedirver"]
+          body = WebStat::WebDriverHelper.get_source(url)
+          @status = 200
         else
-          body = document.body.encode('UTF-8', document.encoding)
+          document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})
+          if document.class == Mechanize::File
+            body = document.body
+          else
+            body = document.body.encode('UTF-8', document.encoding)
+          end
+          @status = document.code
         end
-        @status = document.code
       rescue Mechanize::ResponseCodeError => e
         body = e.page.body
         @status = e.page.code
@@ -96,7 +107,7 @@ module WebStat
     # Get the informations of @url
     # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
     def stat(userdics: nil)
-      clean_content = content.scrub('').gsub(/[\n\t\r　]/, "").gsub(/\s{2,}/, "\s")
+      clean_content = content.scrub('').gsub(/[\n\t\r　]/, "").gsub(/\s{2,}/, "\s").gsub(URI.regexp, "")
       language_code = CLD.detect_language(clean_content)[:code]
       if userdics && userdics.has_key?(language_code) && File.exists?(userdics[language_code])
         tag = WebStat::Tag.new("#{title} #{content}", userdic: userdics[language_code])

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.3.5"
+  VERSION = "0.3.10"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.10
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-13 00:00:00.000000000 Z
+date: 2020-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler