web_stat 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 871e9eb97dc238635bd6a46571e86f6a9548c0ca32b6bf3576017ad71c81394f
-  data.tar.gz: 7ed7e9750fc2030f486c14f4de9441ba2c2c98a8357565bf708e6d43f7ae620e
+  metadata.gz: d3f6f4f9692846a2c50a365125efb42dfd4a0b44fddef1c108621d94fd6dcde4
+  data.tar.gz: 2f6d400bce014144c940edcafbad9dc4e8c164fce356c70bfd7dbd1ad0c2f862
 SHA512:
-  metadata.gz: 963dca0c991935d084ba40d2ff35de87c55d1584f9d39f3c54c7df0d89aa701ddef21014c1ea6b4a0a83d118b9abd6daf98cd084663992d0a3ae09c9b92e4267
-  data.tar.gz: df4c99e70693d004d312219dd2550cd7a3b39e4eb64ac101cd0654a886ee0d6a4d667a82dbdb15574f825aa4cc71fb3d34d9e10f080a977fe026c6f7b7d6006e
+  metadata.gz: 6518aaed72267de7612257c43067a762b933d2e1a64d04defb07d7471eacd8bd1a4ca1cadc96dad5b279cc09a4bbc5a3453abfe10715bde5dc614d08d1953098
+  data.tar.gz: 3cd03188f34030da0c9ead6633bd1c67550524c22368b5c52dd69132e54be69edff6a28d7a91b6ff76ad02a6f3bb2becebb2df1e03db72fd7f3f16262d8137e2

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    web_stat (0.4.2)
+    web_stat (0.5.0)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
-      mechanize (>= 2.7)
+      google-api-client (>= 0.53.0)
+      mechanize (>= 2.7.7)
       natto (>= 1.1.2)
       nokogiri (>= 1.10.4)
       pdf-reader (= 2.4.0)
@@ -18,6 +19,12 @@ GEM
   remote: https://rubygems.org/
   specs:
     Ascii85 (1.0.3)
+    activesupport (6.1.3.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     afm (0.2.2)
@@ -26,40 +33,99 @@ GEM
     cld (0.8.0)
       ffi
     coderay (1.1.3)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
+    crack (0.4.5)
+      rexml
     crass (1.0.6)
-    diff-lcs (1.3)
+    declarative (0.0.20)
+    diff-lcs (1.4.4)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.14.2)
+    faraday (1.4.2)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.1.0)
+    ffi (1.15.3)
+    gems (1.2.0)
+    google-api-client (0.53.0)
+      google-apis-core (~> 0.1)
+      google-apis-generator (~> 0.1)
+    google-apis-core (0.3.0)
+      addressable (~> 2.5, >= 2.5.1)
+      googleauth (~> 0.14)
+      httpclient (>= 2.8.1, < 3.0)
+      mini_mime (~> 1.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.0)
+      rexml
+      signet (~> 0.14)
+      webrick
+    google-apis-discovery_v1 (0.4.0)
+      google-apis-core (~> 0.1)
+    google-apis-generator (0.3.0)
+      activesupport (>= 5.0)
+      gems (~> 1.2)
+      google-apis-core (~> 0.1)
+      google-apis-discovery_v1 (~> 0.0)
+      thor (>= 0.20, < 2.a)
+    googleauth (0.16.2)
+      faraday (>= 0.17.3, < 2.0)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.16)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (~> 0.14)
     guess_html_encoding (0.0.11)
     hashdiff (1.0.1)
     hashery (2.1.2)
-    http-cookie (1.0.3)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
-    mechanize (2.7.6)
-      domain_name (~> 0.5, >= 0.5.1)
-      http-cookie (~> 1.0)
-      mime-types (>= 1.17.2)
-      net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (>= 2.5.2)
-      nokogiri (~> 1.6)
-      ntlm-http (~> 0.1, >= 0.1.1)
-      webrobots (>= 0.0.9, < 0.2)
+    httpclient (2.8.3)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    jwt (2.2.3)
+    mechanize (2.8.1)
+      addressable (~> 2.7)
+      domain_name (~> 0.5, >= 0.5.20190701)
+      http-cookie (~> 1.0, >= 1.0.3)
+      mime-types (~> 3.0)
+      net-http-digest_auth (~> 1.4, >= 1.4.1)
+      net-http-persistent (>= 2.5.2, < 5.0.dev)
+      nokogiri (~> 1.11, >= 1.11.2)
+      rubyntlm (~> 0.6, >= 0.6.3)
+      webrick (~> 1.7)
+      webrobots (~> 0.1.2)
+    memoist (0.16.2)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.1104)
+    mime-types-data (3.2021.0225)
+    mini_mime (1.1.0)
+    mini_portile2 (2.5.3)
+    minitest (5.14.4)
+    multi_json (1.15.0)
+    multipart-post (2.1.1)
     natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (2.9.4)
-    nokogiri (1.11.1-x86_64-linux)
+    net-http-persistent (4.0.1)
+      connection_pool (~> 2.2)
+    nokogiri (1.11.7)
+      mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
-    nokogumbo (2.0.4)
+    nokogumbo (2.0.5)
       nokogiri (~> 1.8, >= 1.8.4)
-    ntlm-http (0.1.1)
+    os (1.1.1)
     pdf-reader (2.4.0)
       Ascii85 (~> 1.0.0)
       afm (~> 0.2.1)
@@ -72,46 +138,63 @@ GEM
     pry-byebug (3.9.0)
       byebug (~> 11.0)
       pry (~> 0.13.0)
-    public_suffix (4.0.5)
+    public_suffix (4.0.6)
     racc (1.5.2)
-    rake (13.0.1)
+    rake (13.0.3)
+    representable (3.1.1)
+      declarative (< 0.1.0)
+      trailblazer-option (>= 0.1.1, < 0.2.0)
+      uber (< 0.2.0)
+    retriable (3.1.2)
     rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
     ruby-rc4 (0.1.5)
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
+    ruby2_keywords (0.0.4)
+    rubyntlm (0.6.3)
     rubyzip (2.3.0)
-    safe_yaml (1.0.5)
-    sanitize (5.2.2)
+    sanitize (5.2.3)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
     selenium-webdriver (3.142.7)
       childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
+    signet (0.15.0)
+      addressable (~> 2.3)
+      faraday (>= 0.17.3, < 2.0)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
+    thor (1.1.0)
+    trailblazer-option (0.1.1)
     ttfunk (1.7.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    uber (0.1.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
-    webmock (3.8.3)
+    webmock (3.11.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.7.0)
     webrobots (0.1.2)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby

data/lib/web_stat.rb

@@ -11,6 +11,7 @@ require 'net/http'
 require 'pdf/reader'
 require 'ruby-readability'
 require 'selenium-webdriver'
+require 'google/apis/youtube_v3'
 
 require "helpers/web_drive_helper"
 require "web_stat/final_redirect_url"
@@ -18,6 +19,7 @@ require "web_stat/categorize"
 require "web_stat/configure"
 require "web_stat/errors"
 require "web_stat/fetch"
+
 require "web_stat/tag"
 require "web_stat/version"
 require "web_stat/fetch/fetch_as_html"

data/lib/web_stat/config/web_stat.yml

@@ -14,10 +14,12 @@ development: &development
     - '//img/@src'
   userdic: ""
   use_chromedirver: false
+  id_extraction_regexs:
+    youtube: '^https://www.youtube.com/watch\?v=([^&]+)'
   thumbnail_regex:
-    youtube:
-      - '%r{^https://www.youtube.com/watch\?v=([^&]+)}'
-      - 'http://img.youtube.com/vi/\1/default.jpg'
+    youtube: 'http://img.youtube.com/vi/\1/default.jpg'
+  api_keys:
+    youtube: "dummy-key"
 test:
   <<: *development
 production:

data/lib/web_stat/fetch.rb

@@ -1,6 +1,6 @@
 module WebStat
   class Fetch
-    attr_accessor :url, :html, :nokogiri, :userdic, :status
+    attr_accessor :url, :html, :nokogiri, :userdic, :status, :header
     # Get title
     # @return [String] title
     def title
@@ -34,7 +34,23 @@ module WebStat
     end
     # Get main section
     def content
-      Sanitize.clean(Readability::Document.new(@nokogiri.at('body').to_s).content)
+      if @url.match(WebStat::Configure.get["id_extraction_regexs"]["youtube"])
+        youtube_decscription
+      else
+        Sanitize.clean(Readability::Document.new(@nokogiri.at('body').to_s).content)
+      end
+    end
+    
+    # Get describe of youtube movie.
+    def youtube_decscription
+      regex_string = WebStat::Configure.get["id_extraction_regexs"]["youtube"]
+      if @url.match(regex_string)
+        id = @url.gsub(%r{#{regex_string}}, '\1')
+        youtube = Google::Apis::YoutubeV3::YouTubeService.new
+        youtube.key = WebStat::Configure.get["api_keys"]["youtube"]
+        response = youtube.list_videos(:snippet, id: id)
+        response.items.first.snippet.description
+      end
     end
 
     # Get temporary path of image
@@ -48,9 +64,9 @@ module WebStat
         end
       end
       # If there is a thumbnail rule, apply it.
-      WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
-        if @url.match(v[0])
-          return @url.gsub(v[0], v[1])
+      WebStat::Configure.get["id_extraction_regexs"].each do |provider, regex_string|
+        if @url.match(regex_string)
+          return @url.gsub(%r{#{regex_string}}, WebStat::Configure.get["thumbnail_regex"][provider])
         end
       end
       readability_content = ::Nokogiri::HTML(Readability::Document.new(@nokogiri.at('body').to_s).content)
@@ -82,6 +98,8 @@ module WebStat
         end
       end
       tmp_file
+    rescue
+      false
     end
 
     # Get url
@@ -95,20 +113,13 @@ module WebStat
         if mech.agent.robots_disallowed?(url)
           raise Mechanize::RobotsDisallowedError.new(url)
         end
-        if WebStat::Configure.get["use_chromedirver"]
-          begin
-            body = WebStat::WebDriverHelper.get_source(url)
-          rescue Selenium::WebDriver::Error::UnknownError => e
-            document = mech.agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
-            if document.class == Mechanize::File
-              body = document.body
-            else
-              body = document.body.encode('UTF-8', document.encoding)
-            end
-          end
+        document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})
+        @header = document.header
+        begin
+          raise 'not_use_chromedirver' unless WebStat::Configure.get["use_chromedirver"]
+          body = WebStat::WebDriverHelper.get_source(url)
           @status = 200
-        else
-          document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})
+        rescue
           if document.class == Mechanize::File
             body = document.body
           else
@@ -122,6 +133,24 @@ module WebStat
       end
       body
     end
+    
+    # Return Date or last modified header.
+    # @param [String] url
+    # @return DataTime
+    def get_last_modified
+      @header = @header || {}
+      if @header.has_key?("date") && @header.has_key?("last-modified")
+        if DateTime.parse(@header["date"]) >= DateTime.parse(@header["last-modified"])
+          DateTime.parse(@header["date"])
+        else
+          DateTime.parse(@header["last-modified"])
+        end
+      elsif @header.has_key?("date")
+        DateTime.parse(@header["date"])
+      elsif @header.has_key?("last-modified")
+        DateTime.parse(@header["last-modified"])
+      end
+    end
 
     # Get the informations of @url
     # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
@@ -142,6 +171,7 @@ module WebStat
         language_code: language_code,
         status: @status,
         url: @url,
+        last_modified_at: get_last_modified,
         eyecatch_image_path: save_local_path(eyecatch_image_path),
         tags: tag.nouns
       }

data/lib/web_stat/fetch/fetch_as_web.rb

@@ -5,7 +5,7 @@ module WebStat
     # initialize class
     # @param [String] url
     def initialize(url)
-      unless url_valid?(url)
+      unless FetchAsWeb.url_valid?(url)
         raise WebStat::INVALID_URL, url
       end
       @url = original_url(url)
@@ -36,11 +36,12 @@ module WebStat
       end
       @nokogiri = ::Nokogiri::HTML(@html)
     end
-    
-    # Validation url
-    def url_valid?(url)
-      regexp = Regexp.new("^https?://([a-zA-Z0-9][a-zA-Z0-9\\\-\.]{1,61}[a-zA-Z0-9])\\\.([a-zA-Z]{2,})(.*)?$", Regexp::IGNORECASE)
-      regexp.match?(url)
-    end 
+    class << self
+      # Validation url
+      def url_valid?(url)
+        regexp = Regexp.new("^https?://([a-z0-9][a-z0-9\\\-\.]{0,61})\\\.([a-z]{2,})(.*)?$", Regexp::IGNORECASE)
+        regexp.match?(url)
+      end
+    end
   end
 end

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

data/spec/spec_helper.rb

@@ -102,4 +102,10 @@ WebMock.stub_request(:get, "https://cdn.newsdict.jp/assets/newsdict-5d8601394c3f
     .to_return(
         status: 200,
         body: File.new(File.join(File.dirname(__FILE__), "fixtures", "images", "newsdict-5d8601394c3f4eea2d7161ab92ab327ac7099e22214c853327011b3a71859b8e.png")),
-        headers: {content_type: 'application/html; charset=utf-8'})
+        headers: {content_type: 'application/html; charset=utf-8'})
+        
+WebMock.stub_request(:get, "https://newsdict.blog/last_modified_at")
+  .to_return(
+    status: 200,
+    body: "ok",
+    headers: {content_type: 'application/html; charset=utf-8', date: "Tue, 05 Apr 2016 07:43:08 GMT", "Last-Modified": "Tue, 05 Apr 2020 07:43:08 JST"})

data/spec/web_stat/configure_spec.rb

@@ -14,14 +14,15 @@ RSpec.describe WebStat::Configure do
   it "Get thumbnail_regex.youtube." do
     config = WebStat::Configure.get
     expect(config["thumbnail_regex"]["yotube"].nil?).to eq true
-    expect(config["thumbnail_regex"]["youtube"].count).to eq 2
+    expect(config["id_extraction_regexs"]["youtube"]).to be_a String
+    expect(config["thumbnail_regex"]["youtube"]).to be_a String
   end
 
   it "Match youtube url." do
     sample_url = "https://www.youtube.com/watch?v=aChpsuUffUM"
-    WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
-      if sample_url.match(v[0])
-        expect(sample_url.gsub(v[0], v[1])).to eq 'http://img.youtube.com/vi/aChpsuUffUM/default.jpg'
+    WebStat::Configure.get["id_extraction_regexs"].each do |provider, regex_string|
+      if sample_url.match(regex_string)
+        expect(sample_url.gsub(%r{#{regex_string}}, WebStat::Configure.get["thumbnail_regex"][provider])).to eq 'http://img.youtube.com/vi/aChpsuUffUM/default.jpg'
       end
     end
   end

data/spec/web_stat/fetch_spec.rb

@@ -197,11 +197,16 @@ RSpec.describe WebStat::Fetch do
   end
   
   it "valid url" do
-    web_stat_fetch_web_class = WebStat::FetchAsWeb.new("https://newsdict.blog/content/images/size/w100/2019/03/facebook-3.jpg")
-    expect(web_stat_fetch_web_class.url_valid?("http://status.aws.amazon.com/#cloudfront_12345")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://findy-code.io?h=NWsZey5UgJ51u&t=omikuji-22")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://www.meetup.com/pro/docker")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://gxyt4.app.goo.gl/Mn64U")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://status.cloud.google.com/incident/cloud-functions/19010")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("http://status.aws.amazon.com/#cloudfront_12345")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://findy-code.io?h=NWsZey5UgJ51u&t=omikuji-22")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://www.meetup.com/pro/docker")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://gxyt4.app.goo.gl/Mn64U")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://status.cloud.google.com/incident/cloud-functions/19010")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("http://g.co/arts/SK1jZHJpT8N1BGaM7")).to be true
+  end
+    
+  it "get_last_modified" do
+    web_stat = WebStat::FetchAsWeb.new("https://newsdict.blog/last_modified_at")
+    web_stat.stat[:last_modified_at] === DateTime.parse("Tue, 05 Apr 2020 07:43:08 JST")
   end
 end

data/web_stat.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   
   spec.add_runtime_dependency "bundler", ">= 2.0.2"
   spec.add_runtime_dependency "nokogiri", ">= 1.10.4"
-  spec.add_runtime_dependency "mechanize", ">= 2.7"
+  spec.add_runtime_dependency "mechanize", ">= 2.7.7"
   spec.add_runtime_dependency "ruby-readability", ">= 0.7"
   spec.add_runtime_dependency "natto", ">= 1.1.2"
   spec.add_runtime_dependency "sanitize", ">= 5.0.0"
@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "pdf-reader", "2.4.0"
   spec.add_runtime_dependency "webrick", ">= 1.7.0"
   spec.add_runtime_dependency "rexml", ">= 3.2.4"
+  spec.add_runtime_dependency "google-api-client", ">= 0.53.0" 
 
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", ">= 3.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-31 00:00:00.000000000 Z
+date: 2021-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 2.7.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 2.7.7
 - !ruby/object:Gem::Dependency
   name: ruby-readability
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.2.4
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.53.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.53.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -238,7 +252,7 @@ description: Fetch the web pages and stat.
 email:
 - yube@newsdict.jp
 executables:
-- fetch_as_html
+- fetch_as_url
 extensions: []
 extra_rdoc_files: []
 files:
@@ -253,7 +267,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/fetch_as_html
+- bin/fetch_as_url
 - docker-compose.yml
 - docker/exec
 - docker/start