web_stat 0.3.0 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f62b7ce2720dcd1851a3c0620af8da6e00313e1b5de918e8b1e23ceeeacee89
-  data.tar.gz: 1a52579415bf2a4f96efd39931ef41849751f693658f5cfbcd88202c680a446c
+  metadata.gz: 3b35413482a93f316bbd3d99a037f0e97102544a4e04604b5add670ac1a1500a
+  data.tar.gz: 4517b7754b2096901b005c26497fa2addad572b02f08a7aed385f6dc7de2e55a
 SHA512:
-  metadata.gz: 51b5d4fbc2e0969913cec3c1ae4b745cca8a790b4f9638d9a920d844840500424c69f18e7d3f58d2d870b8391e0817f06781699b302f36b2a68d81467608a867
-  data.tar.gz: 2d4ba4bfac9c5d5c76fcc48ba0202b7dbf223b4f42b7d6a649cce22c0742d6b7605c3386c93be2cb29487308714fc94fd2bf1063faf78bdcd5b7b6c5ab060d8b
+  metadata.gz: a54c666953b0c51e1e5ea8d230069bf608e6284629740070f77f3816468553037852c0cbc6c39c35cd53000435fbced4acee6a7b0ca855f2c241e0fb769d32da
+  data.tar.gz: 3fef9c1c48f272e27c877a4a588ce45f841ad316429bb78fc3aa6de5748d6471b9e5aaed2ee076af0d3fa0779b5696c452c1b0fc2f77644c7914b171377def8e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    web_stat (0.3.0)
+    web_stat (0.3.5)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)

data/lib/helpers/web_drive_helper.rb

@@ -1,28 +1,30 @@
-class WebDriverHelper
-  class << self
-    # Get last url
-    # @param [String] url
-    # @param [Integer] delay
-    def get_last_url(url, delay=nil)
-      Selenium::WebDriver.logger.output = File.join("/tmp", "selenium.log")
-      Selenium::WebDriver.logger.level = :info
-      options = Selenium::WebDriver::Chrome::Options.new(args: [
-        'headless',
-        'no-sandbox',
-        'disable-gpu',
-        'start-maximized',
-        'window-size=1920,1080'
-        ])
-      driver = Selenium::WebDriver.for(:chrome, options: options)
-      driver.manage.timeouts.implicit_wait = 10
-      Selenium::WebDriver::Wait.new(timeout: 10)
-      driver.get(url)
-      if delay.is_a?(Integer)
-        sleep delay
+module WebStat
+  class WebDriverHelper
+    class << self
+      # Get last url
+      # @param [String] url
+      # @param [Integer] delay
+      def get_last_url(url, delay=nil)
+        Selenium::WebDriver.logger.output = File.join("/tmp", "selenium.log")
+        Selenium::WebDriver.logger.level = :info
+        options = Selenium::WebDriver::Chrome::Options.new(args: [
+          'headless',
+          'no-sandbox',
+          'disable-gpu',
+          'start-maximized',
+          'window-size=1920,1080'
+          ])
+        driver = Selenium::WebDriver.for(:chrome, options: options)
+        driver.manage.timeouts.implicit_wait = 10
+        Selenium::WebDriver::Wait.new(timeout: 10)
+        driver.get(url)
+        if delay.is_a?(Integer)
+          sleep delay
+        end
+        last_url = driver.current_url
+        driver.quit
+        last_url
       end
-      last_url = driver.current_url
-      driver.quit
-      last_url
     end
   end
 end

data/lib/web_stat/config/web_stat.yml

@@ -1,14 +1,21 @@
-# Minimum number of characters to detect meta title
-min_length_of_meta_title: 10
-# Split regular expression for titles 
-regex_to_sprit_title: '\||-|:|｜|：|〜|\~| – '
-# User Agent
-user_agent: "web_stat gem agent"
-# Eyecatch image xpaths
-eyecatch_image_xpaths:
-  - '/html/head/meta[@property="twitter:image"]/@content'
-  - '/html/head/meta[@property="og:image"]/@content'
-  - '//img[@class="attachment-post-thumbnail"]/@src'
-  - '//div[@id="content"]//img/@src'
-  - '//img/@src'
-userdic: ""
+development: &development
+  # Minimum number of characters to detect meta title
+  min_length_of_meta_title: 10
+  # Split regular expression for titles 
+  regex_to_sprit_title: '\||-|:|｜|：|〜|\~| – '
+  # User Agent
+  user_agent: "web_stat gem agent"
+  # Eyecatch image xpaths
+  eyecatch_image_xpaths:
+    - '/html/head/meta[@property="twitter:image"]/@content'
+    - '/html/head/meta[@property="og:image"]/@content'
+    - '//img[@class="attachment-post-thumbnail"]/@src'
+    - '//div[@id="content"]//img/@src'
+    - '//img/@src'
+  userdic: ""
+  use_chromedirver: false
+test:
+  <<: *development
+production:
+  <<: *development
+  use_chromedirver: true

data/lib/web_stat/configure.rb

@@ -3,31 +3,37 @@ module WebStat
   class Configure
     DEFAULT_CONFIG_FILE_PATH = 'config/web_stat.yml'
     
-    # Get yaml
-    def self.get
-      YAML.load_file(self.get_configure_path)
-    end
-    
-    # Get configure path
-    def self.get_configure_path
-      if File.exists?(self.get_custom_configure_path)
-        self.get_custom_configure_path
-      else
-        self.get_default_configure_path
+    class << self
+      # Get yaml
+      def get
+        if defined? Rails
+          YAML.load_file(get_configure_path)[Rails.env]
+        else
+          YAML.load_file(get_configure_path)["production"]
+        end
       end
-    end
-    
-    # Get default configure path
-    def self.get_default_configure_path
-      File.join(File.expand_path("../", __FILE__), DEFAULT_CONFIG_FILE_PATH)
-    end
-    
-    # Get custom configure path
-    def self.get_custom_configure_path
-      if defined? Rails
-	File.join(Rails.root, DEFAULT_CONFIG_FILE_PATH)
-      else
-        File.join(Bundler.root, DEFAULT_CONFIG_FILE_PATH)
+      
+      # Get configure path
+      def get_configure_path
+        if File.exists?(get_custom_configure_path)
+          get_custom_configure_path
+        else
+          get_default_configure_path
+        end
+      end
+      
+      # Get default configure path
+      def get_default_configure_path
+        File.join(File.expand_path("../", __FILE__), DEFAULT_CONFIG_FILE_PATH)
+      end
+      
+      # Get custom configure path
+      def get_custom_configure_path
+        if defined? Rails
+  	      File.join(Rails.root, DEFAULT_CONFIG_FILE_PATH)
+        else
+          File.join(Bundler.root, DEFAULT_CONFIG_FILE_PATH)
+        end
       end
     end
   end

data/lib/web_stat/fetch.rb

@@ -57,7 +57,7 @@ module WebStat
     # Get local path to save url
     # @param [String] url
     def save_local_path(url)
-      return nil if url.nil?
+      return nil if url.nil? || ! url.match(%{^http})
       tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
       agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
       image = agent.get(url)
@@ -122,7 +122,7 @@ module WebStat
     # Get original url
     # @param [String] url
     def original_url(url)
-      last_url = FinalRedirectUrl.final_redirect_url(url)
+      last_url = WebStat::FinalRedirectUrl.final_redirect_url(url)
       unless last_url.nil? || last_url.scrub('').empty?
         last_url
       else

data/lib/web_stat/final_redirect_url.rb

@@ -1,50 +1,54 @@
 # ref) https://github.com/indyarocks/final_redirect_url
 # customize
 #  Changed 
-
-module FinalRedirectUrl
-
-  def self.final_redirect_url(url, options={})
-    final_url = ''
-    if is_valid_url?(url)
-      begin
-        redirect_lookup_depth = options[:depth].to_i > 0 ? options[:depth].to_i : 10
-        response_uri = get_final_redirect_url(url, redirect_lookup_depth)
-        final_url =  url_string_from_uri(response_uri)
-      rescue Exception => ex
-        # nothing
+module WebStat
+  class FinalRedirectUrl
+    class << self
+      def final_redirect_url(url, options={})
+        final_url = ''
+        if is_valid_url?(url)
+          begin
+            redirect_lookup_depth = options[:depth].to_i > 0 ? options[:depth].to_i : 10
+            response_uri = get_final_redirect_url(url, redirect_lookup_depth)
+            final_url =  url_string_from_uri(response_uri)
+          rescue Exception => ex
+            # nothing
+          end
+        end
+        final_url
       end
-    end
-    final_url
-  end
-
-  private
-  def self.is_valid_url?(url)
-    url.to_s.match? URI::regexp(['http', 'https'])
-  end
-
-  def self.get_final_redirect_url(url, limit = 10)
-    return url if limit <= 0
-    uri = URI.parse(url)
-    response = ::Net::HTTP.get_response(uri)
-    if response.class == Net::HTTPOK
-      return URI.parse(WebDriverHelper.get_last_url(uri))
-    else
-      redirect_location = response['location']
-      location_uri = URI.parse(redirect_location)
-      if location_uri.host.nil?
-        redirect_location = uri.scheme + '://' + uri.host + redirect_location
+    
+      private
+      def is_valid_url?(url)
+        url.to_s.match? URI::regexp(['http', 'https'])
+      end
+      def get_final_redirect_url(url, limit = 10)
+        return url if limit <= 0
+        uri = URI.parse(url)
+        response = ::Net::HTTP.get_response(uri)
+        if response.class == Net::HTTPOK
+          if WebStat::Configure.get["use_chromedirver"]
+            return URI.parse(WebStat::WebDriverHelper.get_last_url(uri))
+          else
+            return URI.parse(uri)
+          end
+        else
+          redirect_location = response['location']
+          location_uri = URI.parse(redirect_location)
+          if location_uri.host.nil?
+            redirect_location = uri.scheme + '://' + uri.host + redirect_location
+          end
+          warn "redirected to #{redirect_location}"
+          get_final_redirect_url(redirect_location, limit - 1)
+        end
+      end
+      def url_string_from_uri(uri)
+        url_str = "#{uri.scheme}://#{uri.host}#{uri.request_uri}"
+        if uri.fragment
+          url_str = url_str + "##{uri.fragment}"
+        end
+        url_str
       end
-      warn "redirected to #{redirect_location}"
-      get_final_redirect_url(redirect_location, limit - 1)
-    end
-  end
-
-  def self.url_string_from_uri(uri)
-    url_str = "#{uri.scheme}://#{uri.host}#{uri.request_uri}"
-    if uri.fragment
-      url_str = url_str + "##{uri.fragment}"
     end
-    url_str
   end
-end
+end

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.3.0"
+  VERSION = "0.3.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.5
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-11 00:00:00.000000000 Z
+date: 2020-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler