web_stat 0.4.1 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fc357cc98a214af7a391fe834ec927b43649ca8d8c5b1105e7fb1947dd7127a
-  data.tar.gz: f7c5f6d28101ff677aaa4db5b9b486d067798b0a8461f91352db252f68374513
+  metadata.gz: 234586f7faedd6315e64b118b3f63532d2ea64f5a56fa046c78d56ae3be52935
+  data.tar.gz: 71b73587d78fa05da29ffe15c2c381e3fccd31b76025060bbeedcff1f5ea5cf5
 SHA512:
-  metadata.gz: 1616c4f1a64ebc693ee4e822b06150028783736448850d8aca80c90909ca879580a82f871f95375904b38813b10155105fd5d1121ed0321ae9cc898a4b95549e
-  data.tar.gz: 1637fb0b6b6f8ee83b133d95e3ab31a6c4ad2a64399f4ac19173f215ec7a79eac851c1348d9efa1efbfdc88a7ac8e5d6ecd9f9c067c4295b0a82f2662ef4c448
+  metadata.gz: 34b7f3a68413b53865c2c5e57dc78ab2627689e656ed6ebc9d69938804cbbbf490e4148f2e0e44afbd8da4c9052423427bd053f90bf24fe65195c4ded0ede57d
+  data.tar.gz: ae11480e79de83e5c082e3663fa6570a64e85da5739b3c009a500ad4bd88bbd95f9fa7045362f211aff014f3e3aed9d363158f91e33ab87d60fc09e9dc26b3d3

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    web_stat (0.4.1)
+    web_stat (0.4.7)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
-      mechanize (>= 2.7)
+      mechanize (>= 2.7.7)
       natto (>= 1.1.2)
       nokogiri (>= 1.10.4)
       pdf-reader (= 2.4.0)
@@ -26,10 +26,11 @@ GEM
     cld (0.8.0)
       ffi
     coderay (1.1.3)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
+    connection_pool (2.2.3)
+    crack (0.4.5)
+      rexml
     crass (1.0.6)
-    diff-lcs (1.3)
+    diff-lcs (1.4.4)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     ffi (1.14.2)
@@ -38,7 +39,7 @@ GEM
     hashery (2.1.2)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    mechanize (2.7.6)
+    mechanize (2.7.7)
       domain_name (~> 0.5, >= 0.5.1)
       http-cookie (~> 1.0)
       mime-types (>= 1.17.2)
@@ -46,15 +47,17 @@ GEM
       net-http-persistent (>= 2.5.2)
       nokogiri (~> 1.6)
       ntlm-http (~> 0.1, >= 0.1.1)
+      webrick (~> 1.7)
       webrobots (>= 0.0.9, < 0.2)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.1104)
+    mime-types-data (3.2021.0225)
     natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (2.9.4)
+    net-http-persistent (4.0.1)
+      connection_pool (~> 2.2)
     nokogiri (1.11.1-x86_64-linux)
       racc (~> 1.4)
     nokogumbo (2.0.4)
@@ -72,30 +75,29 @@ GEM
     pry-byebug (3.9.0)
       byebug (~> 11.0)
       pry (~> 0.13.0)
-    public_suffix (4.0.5)
+    public_suffix (4.0.6)
     racc (1.5.2)
-    rake (13.0.1)
+    rake (13.0.3)
     rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
     ruby-rc4 (0.1.5)
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
     rubyzip (2.3.0)
-    safe_yaml (1.0.5)
-    sanitize (5.2.2)
+    sanitize (5.2.3)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
@@ -106,7 +108,7 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
-    webmock (3.8.3)
+    webmock (3.11.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)

data/lib/web_stat/fetch.rb

@@ -1,6 +1,6 @@
 module WebStat
   class Fetch
-    attr_accessor :url, :html, :nokogiri, :userdic, :status
+    attr_accessor :url, :html, :nokogiri, :userdic, :status, :header
     # Get title
     # @return [String] title
     def title
@@ -82,6 +82,8 @@ module WebStat
         end
       end
       tmp_file
+    rescue
+      false
     end
 
     # Get url
@@ -95,20 +97,13 @@ module WebStat
         if mech.agent.robots_disallowed?(url)
           raise Mechanize::RobotsDisallowedError.new(url)
         end
-        if WebStat::Configure.get["use_chromedirver"]
-          begin
-            body = WebStat::WebDriverHelper.get_source(url)
-          rescue Selenium::WebDriver::Error::UnknownError => e
-            document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
-            if document.class == Mechanize::File
-              body = document.body
-            else
-              body = document.body.encode('UTF-8', document.encoding)
-            end
-          end
+        document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})
+        @header = document.header
+        begin
+          raise 'not_use_chromedirver' unless WebStat::Configure.get["use_chromedirver"]
+          body = WebStat::WebDriverHelper.get_source(url)
           @status = 200
-        else
-          document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})
+        rescue
           if document.class == Mechanize::File
             body = document.body
           else
@@ -122,6 +117,24 @@ module WebStat
       end
       body
     end
+    
+    # Return Date or last modified header.
+    # @param [String] url
+    # @return DataTime
+    def get_last_modified
+      @header = @header || {}
+      if @header.has_key?("date") && @header.has_key?("last-modified")
+        if DateTime.parse(@header["date"]) >= DateTime.parse(@header["last-modified"])
+          DateTime.parse(@header["date"])
+        else
+          DateTime.parse(@header["last-modified"])
+        end
+      elsif @header.has_key?("date")
+        DateTime.parse(@header["date"])
+      elsif @header.has_key?("last-modified")
+        DateTime.parse(@header["last-modified"])
+      end
+    end
 
     # Get the informations of @url
     # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
@@ -142,6 +155,7 @@ module WebStat
         language_code: language_code,
         status: @status,
         url: @url,
+        last_modified_at: get_last_modified,
         eyecatch_image_path: save_local_path(eyecatch_image_path),
         tags: tag.nouns
       }

data/lib/web_stat/fetch/fetch_as_web.rb

@@ -5,7 +5,7 @@ module WebStat
     # initialize class
     # @param [String] url
     def initialize(url)
-      unless url_valid?(url)
+      unless FetchAsWeb.url_valid?(url)
         raise WebStat::INVALID_URL, url
       end
       @url = original_url(url)
@@ -36,11 +36,12 @@ module WebStat
       end
       @nokogiri = ::Nokogiri::HTML(@html)
     end
-    
-    # Validation url
-    def url_valid?(url)
-      regexp = Regexp.new("^https?://([a-zA-Z0-9][a-zA-Z0-9\\\-\.]{1,61}[a-zA-Z0-9])\\\.([a-zA-Z]{2,})(.*)?$", Regexp::IGNORECASE)
-      regexp.match?(url)
-    end 
+    class << self
+      # Validation url
+      def url_valid?(url)
+        regexp = Regexp.new("^https?://([a-z0-9][a-z0-9\\\-\.]{0,61})\\\.([a-z]{2,})(.*)?$", Regexp::IGNORECASE)
+        regexp.match?(url)
+      end
+    end
   end
 end

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.4.1"
+  VERSION = "0.4.7"
 end

data/spec/spec_helper.rb

@@ -102,4 +102,10 @@ WebMock.stub_request(:get, "https://cdn.newsdict.jp/assets/newsdict-5d8601394c3f
     .to_return(
         status: 200,
         body: File.new(File.join(File.dirname(__FILE__), "fixtures", "images", "newsdict-5d8601394c3f4eea2d7161ab92ab327ac7099e22214c853327011b3a71859b8e.png")),
-        headers: {content_type: 'application/html; charset=utf-8'})
+        headers: {content_type: 'application/html; charset=utf-8'})
+        
+WebMock.stub_request(:get, "https://newsdict.blog/last_modified_at")
+  .to_return(
+    status: 200,
+    body: "ok",
+    headers: {content_type: 'application/html; charset=utf-8', date: "Tue, 05 Apr 2016 07:43:08 GMT", "Last-Modified": "Tue, 05 Apr 2020 07:43:08 JST"})

data/spec/web_stat/fetch_spec.rb

@@ -197,11 +197,16 @@ RSpec.describe WebStat::Fetch do
   end
   
   it "valid url" do
-    web_stat_fetch_web_class = WebStat::FetchAsWeb.new("https://newsdict.blog/content/images/size/w100/2019/03/facebook-3.jpg")
-    expect(web_stat_fetch_web_class.url_valid?("http://status.aws.amazon.com/#cloudfront_12345")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://findy-code.io?h=NWsZey5UgJ51u&t=omikuji-22")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://www.meetup.com/pro/docker")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://gxyt4.app.goo.gl/Mn64U")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://status.cloud.google.com/incident/cloud-functions/19010")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("http://status.aws.amazon.com/#cloudfront_12345")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://findy-code.io?h=NWsZey5UgJ51u&t=omikuji-22")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://www.meetup.com/pro/docker")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://gxyt4.app.goo.gl/Mn64U")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://status.cloud.google.com/incident/cloud-functions/19010")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("http://g.co/arts/SK1jZHJpT8N1BGaM7")).to be true
+  end
+    
+  it "get_last_modified" do
+    web_stat = WebStat::FetchAsWeb.new("https://newsdict.blog/last_modified_at")
+    web_stat.stat[:last_modified_at] === DateTime.parse("Tue, 05 Apr 2020 07:43:08 JST")
   end
 end

data/web_stat.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   
   spec.add_runtime_dependency "bundler", ">= 2.0.2"
   spec.add_runtime_dependency "nokogiri", ">= 1.10.4"
-  spec.add_runtime_dependency "mechanize", ">= 2.7"
+  spec.add_runtime_dependency "mechanize", ">= 2.7.7"
   spec.add_runtime_dependency "ruby-readability", ">= 0.7"
   spec.add_runtime_dependency "natto", ">= 1.1.2"
   spec.add_runtime_dependency "sanitize", ">= 5.0.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.7
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-31 00:00:00.000000000 Z
+date: 2021-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 2.7.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 2.7.7
 - !ruby/object:Gem::Dependency
   name: ruby-readability
   requirement: !ruby/object:Gem::Requirement