web_stat 0.3.1 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf6e7c6f1461adb76d15e934bbcb4866350dbef5ee45dc36ab5bac44cac88ce5
-  data.tar.gz: 3f788539423a476ba805b5cc023320fbd9e16b5ef3cffe476f8c7c9a9c6bcedf
+  metadata.gz: 2ed82b635d751fd0e40eeeef77348ebfc02636f1a9f2d86ca0018c572c4b1f2d
+  data.tar.gz: 5034dcc4ba993f6d084228eb3e7e12fe03b92e8ff9e0119d3db8f7ad81521f4a
 SHA512:
-  metadata.gz: 77787d2811e56d7e7db5d78c378204ff83fb44252a38c41e5843e8abf98e5c06dc5e5763f5e877202557294ea480c065f541f05ae9bd944e28433d9f46be5520
-  data.tar.gz: ed612a3a5bb29330c08876736e834a166f658db6f4ca75c7d93c120a60ddeb25695f98416efaeb5d42693b2278b74ca22ab83b77235135b962ce0f9ed956fd6a
+  metadata.gz: d1a20f17519854baf5cea62ff55236b0ea63f9ab61b3c97df2f276ad7e64694d7d6387824957bee717c50a042734b37dcd26a6761591fa3d6550de8266454ac6
+  data.tar.gz: 1669e18b93b126bc3d382eba1e25de554fc23751f0d51635ec42c89fa8c05392d61901bb4fa510e27b1048133e96dba153e213cab3c3de90a2af1461cca88bf2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    web_stat (0.3.0)
+    web_stat (0.3.5)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)

data/lib/web_stat/config/web_stat.yml

@@ -1,14 +1,21 @@
-# Minimum number of characters to detect meta title
-min_length_of_meta_title: 10
-# Split regular expression for titles 
-regex_to_sprit_title: '\||-|:|｜|：|〜|\~| – '
-# User Agent
-user_agent: "web_stat gem agent"
-# Eyecatch image xpaths
-eyecatch_image_xpaths:
-  - '/html/head/meta[@property="twitter:image"]/@content'
-  - '/html/head/meta[@property="og:image"]/@content'
-  - '//img[@class="attachment-post-thumbnail"]/@src'
-  - '//div[@id="content"]//img/@src'
-  - '//img/@src'
-userdic: ""
+development: &development
+  # Minimum number of characters to detect meta title
+  min_length_of_meta_title: 10
+  # Split regular expression for titles 
+  regex_to_sprit_title: '\||-|:|｜|：|〜|\~| – '
+  # User Agent
+  user_agent: "web_stat gem agent"
+  # Eyecatch image xpaths
+  eyecatch_image_xpaths:
+    - '/html/head/meta[@property="twitter:image"]/@content'
+    - '/html/head/meta[@property="og:image"]/@content'
+    - '//img[@class="attachment-post-thumbnail"]/@src'
+    - '//div[@id="content"]//img/@src'
+    - '//img/@src'
+  userdic: ""
+  use_chromedirver: false
+test:
+  <<: *development
+production:
+  <<: *development
+  use_chromedirver: true

data/lib/web_stat/configure.rb

@@ -3,31 +3,37 @@ module WebStat
   class Configure
     DEFAULT_CONFIG_FILE_PATH = 'config/web_stat.yml'
     
-    # Get yaml
-    def self.get
-      YAML.load_file(self.get_configure_path)
-    end
-    
-    # Get configure path
-    def self.get_configure_path
-      if File.exists?(self.get_custom_configure_path)
-        self.get_custom_configure_path
-      else
-        self.get_default_configure_path
+    class << self
+      # Get yaml
+      def get
+        if defined? Rails
+          YAML.load_file(get_configure_path)[Rails.env]
+        else
+          YAML.load_file(get_configure_path)["production"]
+        end
       end
-    end
-    
-    # Get default configure path
-    def self.get_default_configure_path
-      File.join(File.expand_path("../", __FILE__), DEFAULT_CONFIG_FILE_PATH)
-    end
-    
-    # Get custom configure path
-    def self.get_custom_configure_path
-      if defined? Rails
-	File.join(Rails.root, DEFAULT_CONFIG_FILE_PATH)
-      else
-        File.join(Bundler.root, DEFAULT_CONFIG_FILE_PATH)
+      
+      # Get configure path
+      def get_configure_path
+        if File.exists?(get_custom_configure_path)
+          get_custom_configure_path
+        else
+          get_default_configure_path
+        end
+      end
+      
+      # Get default configure path
+      def get_default_configure_path
+        File.join(File.expand_path("../", __FILE__), DEFAULT_CONFIG_FILE_PATH)
+      end
+      
+      # Get custom configure path
+      def get_custom_configure_path
+        if defined? Rails
+  	      File.join(Rails.root, DEFAULT_CONFIG_FILE_PATH)
+        else
+          File.join(Bundler.root, DEFAULT_CONFIG_FILE_PATH)
+        end
       end
     end
   end

data/lib/web_stat/fetch.rb

@@ -47,6 +47,12 @@ module WebStat
           break
         end
       end
+      if path.nil?
+        path = @nokogiri.at('body').xpath('//img').first.attr('src')
+      end
+      if path.nil?
+        Readability::Document.new(@nokogiri.at('body')).content
+      end
       if ! path.nil? && path.match(/^\//)
         "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
       else
@@ -57,7 +63,7 @@ module WebStat
     # Get local path to save url
     # @param [String] url
     def save_local_path(url)
-      return nil if url.nil?
+      return nil if url.nil? || ! url.match(%{^http})
       tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
       agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
       image = agent.get(url)
@@ -96,7 +102,7 @@ module WebStat
     # Get the informations of @url
     # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
     def stat(userdics: nil)
-      clean_content = content.scrub('').gsub(/[\n\t\r　]/, "").gsub(/\s{2,}/, "\s")
+      clean_content = content.scrub('').gsub(/[\n\t\r　]/, "").gsub(/\s{2,}/, "\s").gsub(URI.regexp, "")
       language_code = CLD.detect_language(clean_content)[:code]
       if userdics && userdics.has_key?(language_code) && File.exists?(userdics[language_code])
         tag = WebStat::Tag.new("#{title} #{content}", userdic: userdics[language_code])

data/lib/web_stat/final_redirect_url.rb

@@ -27,7 +27,11 @@ module WebStat
         uri = URI.parse(url)
         response = ::Net::HTTP.get_response(uri)
         if response.class == Net::HTTPOK
-          return URI.parse(WebStat::WebDriverHelper.get_last_url(uri))
+          if WebStat::Configure.get["use_chromedirver"]
+            return URI.parse(WebStat::WebDriverHelper.get_last_url(uri))
+          else
+            return URI.parse(uri)
+          end
         else
           redirect_location = response['location']
           location_uri = URI.parse(redirect_location)

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.3.1"
+  VERSION = "0.3.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.6
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-12 00:00:00.000000000 Z
+date: 2020-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler