web_stat 0.4.0 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 29d04d8379978d1e295b829193a330fd9bab8cb92326e83aab39d066059dcb02
-  data.tar.gz: c2df9896f21ec9d777dea101a1c4c4be513ad28c11428fc1104ccefe08f98b40
+  metadata.gz: 3ce60bcf4a31f90024abc35cf1ecc57e32626dfb5ecf4bc4f2280bd72931ff34
+  data.tar.gz: 575f805a63a995b2d0e3bc909978dcd4e3b4f15462f717cf83d6aac00c96078d
 SHA512:
-  metadata.gz: 728bc129ddced4cc58081ca4d8c02b761ba77e4adf277b736115ce51b1e7293f7b208adc5742fe50ba4e406f2a7c3f65af19d5c55f0cf0188a4a4b25704719bd
-  data.tar.gz: ef8ee476834a6fb75fed33476d070e1b00044f9115b85eaf3148f1901af10e4cd32f79ef3b369e59366a56ab7e8560180da3def8562e2b396f2b8b6bfc26196b
+  metadata.gz: 7af1262b25163205eabdfa26e1671f95d53963387eceec6c2c99da0a3a17359b77aaa097ec9556010b8000ecc0772d1dc19a67d128310c8d6bdd8379d008a913
+  data.tar.gz: 855706ad8525609e8a2a50ca64091081794940d69187f951a79c02c66acfa5be9f53cdeacf0f99f6c0431ad41a71ce616691590d40bce0a56576275fba96c453

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    web_stat (0.3.19)
+    web_stat (0.4.6)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
-      mechanize (>= 2.7)
+      mechanize (>= 2.7.7)
       natto (>= 1.1.2)
       nokogiri (>= 1.10.4)
       pdf-reader (= 2.4.0)
@@ -26,10 +26,11 @@ GEM
     cld (0.8.0)
       ffi
     coderay (1.1.3)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
+    connection_pool (2.2.3)
+    crack (0.4.5)
+      rexml
     crass (1.0.6)
-    diff-lcs (1.3)
+    diff-lcs (1.4.4)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     ffi (1.14.2)
@@ -38,7 +39,7 @@ GEM
     hashery (2.1.2)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    mechanize (2.7.6)
+    mechanize (2.7.7)
       domain_name (~> 0.5, >= 0.5.1)
       http-cookie (~> 1.0)
       mime-types (>= 1.17.2)
@@ -46,16 +47,20 @@ GEM
       net-http-persistent (>= 2.5.2)
       nokogiri (~> 1.6)
       ntlm-http (~> 0.1, >= 0.1.1)
+      webrick (~> 1.7)
       webrobots (>= 0.0.9, < 0.2)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.1104)
+    mime-types-data (3.2021.0212)
+    mini_portile2 (2.5.0)
     natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (2.9.4)
-    nokogiri (1.11.1-x86_64-linux)
+    net-http-persistent (4.0.1)
+      connection_pool (~> 2.2)
+    nokogiri (1.11.1)
+      mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
     nokogumbo (2.0.4)
       nokogiri (~> 1.8, >= 1.8.4)
@@ -72,30 +77,29 @@ GEM
     pry-byebug (3.9.0)
       byebug (~> 11.0)
       pry (~> 0.13.0)
-    public_suffix (4.0.5)
+    public_suffix (4.0.6)
     racc (1.5.2)
-    rake (13.0.1)
+    rake (13.0.3)
     rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
     ruby-rc4 (0.1.5)
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
     rubyzip (2.3.0)
-    safe_yaml (1.0.5)
-    sanitize (5.2.2)
+    sanitize (5.2.3)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
@@ -106,7 +110,7 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
-    webmock (3.8.3)
+    webmock (3.11.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)

data/lib/web_stat/fetch.rb

@@ -1,6 +1,6 @@
 module WebStat
   class Fetch
-    attr_accessor :url, :html, :nokogiri, :userdic, :status
+    attr_accessor :url, :html, :nokogiri, :userdic, :status, :header
     # Get title
     # @return [String] title
     def title
@@ -95,11 +95,13 @@ module WebStat
         if mech.agent.robots_disallowed?(url)
           raise Mechanize::RobotsDisallowedError.new(url)
         end
-        if WebStat::Configure.get["use_chromedirver"]
+        document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})
+        @header = document.header
+        begin
+          raise 'not_use_chromedirver' unless WebStat::Configure.get["use_chromedirver"]
           body = WebStat::WebDriverHelper.get_source(url)
           @status = 200
-        else
-          document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})
+        rescue
           if document.class == Mechanize::File
             body = document.body
           else
@@ -113,6 +115,24 @@ module WebStat
       end
       body
     end
+    
+    # Return Date or last modified header.
+    # @param [String] url
+    # @return DataTime
+    def get_last_modified
+      @header = @header || {}
+      if @header.has_key?("date") && @header.has_key?("last-modified")
+        if DateTime.parse(@header["date"]) >= DateTime.parse(@header["last-modified"])
+          DateTime.parse(@header["date"])
+        else
+          DateTime.parse(@header["last-modified"])
+        end
+      elsif @header.has_key?("date")
+        DateTime.parse(@header["date"])
+      elsif @header.has_key?("last-modified")
+        DateTime.parse(@header["last-modified"])
+      end
+    end
 
     # Get the informations of @url
     # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
@@ -133,6 +153,7 @@ module WebStat
         language_code: language_code,
         status: @status,
         url: @url,
+        last_modified_at: get_last_modified,
         eyecatch_image_path: save_local_path(eyecatch_image_path),
         tags: tag.nouns
       }

data/lib/web_stat/fetch/fetch_as_web.rb

@@ -5,7 +5,7 @@ module WebStat
     # initialize class
     # @param [String] url
     def initialize(url)
-      unless url_valid?(url)
+      unless FetchAsWeb.url_valid?(url)
         raise WebStat::INVALID_URL, url
       end
       @url = original_url(url)
@@ -36,11 +36,12 @@ module WebStat
       end
       @nokogiri = ::Nokogiri::HTML(@html)
     end
-    
-    # Validation url
-    def url_valid?(url)
-      regexp = Regexp.new("^https?://([a-zA-Z0-9][a-zA-Z0-9\\\-\.]{1,61}[a-zA-Z0-9])\\\.([a-zA-Z]{2,})(.*)?$", Regexp::IGNORECASE)
-      regexp.match?(url)
-    end 
+    class << self
+      # Validation url
+      def url_valid?(url)
+        regexp = Regexp.new("^https?://([a-z0-9][a-z0-9\\\-\.]{0,61})\\\.([a-z]{2,})(.*)?$", Regexp::IGNORECASE)
+        regexp.match?(url)
+      end
+    end
   end
 end

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.4.0"
-end
+  VERSION = "0.4.6"
+end

data/spec/spec_helper.rb

@@ -102,4 +102,10 @@ WebMock.stub_request(:get, "https://cdn.newsdict.jp/assets/newsdict-5d8601394c3f
     .to_return(
         status: 200,
         body: File.new(File.join(File.dirname(__FILE__), "fixtures", "images", "newsdict-5d8601394c3f4eea2d7161ab92ab327ac7099e22214c853327011b3a71859b8e.png")),
-        headers: {content_type: 'application/html; charset=utf-8'})
+        headers: {content_type: 'application/html; charset=utf-8'})
+        
+WebMock.stub_request(:get, "https://newsdict.blog/last_modified_at")
+  .to_return(
+    status: 200,
+    body: "ok",
+    headers: {content_type: 'application/html; charset=utf-8', date: "Tue, 05 Apr 2016 07:43:08 GMT", "Last-Modified": "Tue, 05 Apr 2020 07:43:08 JST"})

data/spec/web_stat/fetch_spec.rb

@@ -197,11 +197,16 @@ RSpec.describe WebStat::Fetch do
   end
   
   it "valid url" do
-    web_stat_fetch_web_class = WebStat::FetchAsWeb.new("https://newsdict.blog/content/images/size/w100/2019/03/facebook-3.jpg")
-    expect(web_stat_fetch_web_class.url_valid?("http://status.aws.amazon.com/#cloudfront_12345")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://findy-code.io?h=NWsZey5UgJ51u&t=omikuji-22")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://www.meetup.com/pro/docker")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://gxyt4.app.goo.gl/Mn64U")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://status.cloud.google.com/incident/cloud-functions/19010")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("http://status.aws.amazon.com/#cloudfront_12345")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://findy-code.io?h=NWsZey5UgJ51u&t=omikuji-22")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://www.meetup.com/pro/docker")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://gxyt4.app.goo.gl/Mn64U")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://status.cloud.google.com/incident/cloud-functions/19010")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("http://g.co/arts/SK1jZHJpT8N1BGaM7")).to be true
+  end
+    
+  it "get_last_modified" do
+    web_stat = WebStat::FetchAsWeb.new("https://newsdict.blog/last_modified_at")
+    web_stat.stat[:last_modified_at] === DateTime.parse("Tue, 05 Apr 2020 07:43:08 JST")
   end
 end

data/web_stat.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   
   spec.add_runtime_dependency "bundler", ">= 2.0.2"
   spec.add_runtime_dependency "nokogiri", ">= 1.10.4"
-  spec.add_runtime_dependency "mechanize", ">= 2.7"
+  spec.add_runtime_dependency "mechanize", ">= 2.7.7"
   spec.add_runtime_dependency "ruby-readability", ">= 0.7"
   spec.add_runtime_dependency "natto", ">= 1.1.2"
   spec.add_runtime_dependency "sanitize", ">= 5.0.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.6
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-09 00:00:00.000000000 Z
+date: 2021-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 2.7.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 2.7.7
 - !ruby/object:Gem::Dependency
   name: ruby-readability
   requirement: !ruby/object:Gem::Requirement