web_stat 0.4.7 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 234586f7faedd6315e64b118b3f63532d2ea64f5a56fa046c78d56ae3be52935
-  data.tar.gz: 71b73587d78fa05da29ffe15c2c381e3fccd31b76025060bbeedcff1f5ea5cf5
+  metadata.gz: d3f6f4f9692846a2c50a365125efb42dfd4a0b44fddef1c108621d94fd6dcde4
+  data.tar.gz: 2f6d400bce014144c940edcafbad9dc4e8c164fce356c70bfd7dbd1ad0c2f862
 SHA512:
-  metadata.gz: 34b7f3a68413b53865c2c5e57dc78ab2627689e656ed6ebc9d69938804cbbbf490e4148f2e0e44afbd8da4c9052423427bd053f90bf24fe65195c4ded0ede57d
-  data.tar.gz: ae11480e79de83e5c082e3663fa6570a64e85da5739b3c009a500ad4bd88bbd95f9fa7045362f211aff014f3e3aed9d363158f91e33ab87d60fc09e9dc26b3d3
+  metadata.gz: 6518aaed72267de7612257c43067a762b933d2e1a64d04defb07d7471eacd8bd1a4ca1cadc96dad5b279cc09a4bbc5a3453abfe10715bde5dc614d08d1953098
+  data.tar.gz: 3cd03188f34030da0c9ead6633bd1c67550524c22368b5c52dd69132e54be69edff6a28d7a91b6ff76ad02a6f3bb2becebb2df1e03db72fd7f3f16262d8137e2

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    web_stat (0.4.7)
+    web_stat (0.5.0)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
+      google-api-client (>= 0.53.0)
       mechanize (>= 2.7.7)
       natto (>= 1.1.2)
       nokogiri (>= 1.10.4)
@@ -18,6 +19,12 @@ GEM
   remote: https://rubygems.org/
   specs:
     Ascii85 (1.0.3)
+    activesupport (6.1.3.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     afm (0.2.2)
@@ -26,43 +33,99 @@ GEM
     cld (0.8.0)
       ffi
     coderay (1.1.3)
-    connection_pool (2.2.3)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
     crack (0.4.5)
       rexml
     crass (1.0.6)
+    declarative (0.0.20)
     diff-lcs (1.4.4)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.14.2)
+    faraday (1.4.2)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.1.0)
+    ffi (1.15.3)
+    gems (1.2.0)
+    google-api-client (0.53.0)
+      google-apis-core (~> 0.1)
+      google-apis-generator (~> 0.1)
+    google-apis-core (0.3.0)
+      addressable (~> 2.5, >= 2.5.1)
+      googleauth (~> 0.14)
+      httpclient (>= 2.8.1, < 3.0)
+      mini_mime (~> 1.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.0)
+      rexml
+      signet (~> 0.14)
+      webrick
+    google-apis-discovery_v1 (0.4.0)
+      google-apis-core (~> 0.1)
+    google-apis-generator (0.3.0)
+      activesupport (>= 5.0)
+      gems (~> 1.2)
+      google-apis-core (~> 0.1)
+      google-apis-discovery_v1 (~> 0.0)
+      thor (>= 0.20, < 2.a)
+    googleauth (0.16.2)
+      faraday (>= 0.17.3, < 2.0)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.16)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (~> 0.14)
     guess_html_encoding (0.0.11)
     hashdiff (1.0.1)
     hashery (2.1.2)
-    http-cookie (1.0.3)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
-    mechanize (2.7.7)
-      domain_name (~> 0.5, >= 0.5.1)
-      http-cookie (~> 1.0)
-      mime-types (>= 1.17.2)
-      net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (>= 2.5.2)
-      nokogiri (~> 1.6)
-      ntlm-http (~> 0.1, >= 0.1.1)
+    httpclient (2.8.3)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    jwt (2.2.3)
+    mechanize (2.8.1)
+      addressable (~> 2.7)
+      domain_name (~> 0.5, >= 0.5.20190701)
+      http-cookie (~> 1.0, >= 1.0.3)
+      mime-types (~> 3.0)
+      net-http-digest_auth (~> 1.4, >= 1.4.1)
+      net-http-persistent (>= 2.5.2, < 5.0.dev)
+      nokogiri (~> 1.11, >= 1.11.2)
+      rubyntlm (~> 0.6, >= 0.6.3)
       webrick (~> 1.7)
-      webrobots (>= 0.0.9, < 0.2)
+      webrobots (~> 0.1.2)
+    memoist (0.16.2)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2021.0225)
+    mini_mime (1.1.0)
+    mini_portile2 (2.5.3)
+    minitest (5.14.4)
+    multi_json (1.15.0)
+    multipart-post (2.1.1)
     natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
     net-http-persistent (4.0.1)
       connection_pool (~> 2.2)
-    nokogiri (1.11.1-x86_64-linux)
+    nokogiri (1.11.7)
+      mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
-    nokogumbo (2.0.4)
+    nokogumbo (2.0.5)
       nokogiri (~> 1.8, >= 1.8.4)
-    ntlm-http (0.1.1)
+    os (1.1.1)
     pdf-reader (2.4.0)
       Ascii85 (~> 1.0.0)
       afm (~> 0.2.1)
@@ -78,6 +141,11 @@ GEM
     public_suffix (4.0.6)
     racc (1.5.2)
     rake (13.0.3)
+    representable (3.1.1)
+      declarative (< 0.1.0)
+      trailblazer-option (>= 0.1.1, < 0.2.0)
+      uber (< 0.2.0)
+    retriable (3.1.2)
     rexml (3.2.4)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -96,6 +164,8 @@ GEM
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
+    ruby2_keywords (0.0.4)
+    rubyntlm (0.6.3)
     rubyzip (2.3.0)
     sanitize (5.2.3)
       crass (~> 1.0.2)
@@ -104,7 +174,17 @@ GEM
     selenium-webdriver (3.142.7)
       childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
+    signet (0.15.0)
+      addressable (~> 2.3)
+      faraday (>= 0.17.3, < 2.0)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
+    thor (1.1.0)
+    trailblazer-option (0.1.1)
     ttfunk (1.7.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    uber (0.1.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
@@ -114,6 +194,7 @@ GEM
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.7.0)
     webrobots (0.1.2)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby

data/lib/web_stat.rb

@@ -11,6 +11,7 @@ require 'net/http'
 require 'pdf/reader'
 require 'ruby-readability'
 require 'selenium-webdriver'
+require 'google/apis/youtube_v3'
 
 require "helpers/web_drive_helper"
 require "web_stat/final_redirect_url"
@@ -18,6 +19,7 @@ require "web_stat/categorize"
 require "web_stat/configure"
 require "web_stat/errors"
 require "web_stat/fetch"
+
 require "web_stat/tag"
 require "web_stat/version"
 require "web_stat/fetch/fetch_as_html"

data/lib/web_stat/config/web_stat.yml

@@ -14,10 +14,12 @@ development: &development
     - '//img/@src'
   userdic: ""
   use_chromedirver: false
+  id_extraction_regexs:
+    youtube: '^https://www.youtube.com/watch\?v=([^&]+)'
   thumbnail_regex:
-    youtube:
-      - '%r{^https://www.youtube.com/watch\?v=([^&]+)}'
-      - 'http://img.youtube.com/vi/\1/default.jpg'
+    youtube: 'http://img.youtube.com/vi/\1/default.jpg'
+  api_keys:
+    youtube: "dummy-key"
 test:
   <<: *development
 production:

data/lib/web_stat/fetch.rb

@@ -34,7 +34,23 @@ module WebStat
     end
     # Get main section
     def content
-      Sanitize.clean(Readability::Document.new(@nokogiri.at('body').to_s).content)
+      if @url.match(WebStat::Configure.get["id_extraction_regexs"]["youtube"])
+        youtube_decscription
+      else
+        Sanitize.clean(Readability::Document.new(@nokogiri.at('body').to_s).content)
+      end
+    end
+    
+    # Get describe of youtube movie.
+    def youtube_decscription
+      regex_string = WebStat::Configure.get["id_extraction_regexs"]["youtube"]
+      if @url.match(regex_string)
+        id = @url.gsub(%r{#{regex_string}}, '\1')
+        youtube = Google::Apis::YoutubeV3::YouTubeService.new
+        youtube.key = WebStat::Configure.get["api_keys"]["youtube"]
+        response = youtube.list_videos(:snippet, id: id)
+        response.items.first.snippet.description
+      end
     end
 
     # Get temporary path of image
@@ -48,9 +64,9 @@ module WebStat
         end
       end
       # If there is a thumbnail rule, apply it.
-      WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
-        if @url.match(v[0])
-          return @url.gsub(v[0], v[1])
+      WebStat::Configure.get["id_extraction_regexs"].each do |provider, regex_string|
+        if @url.match(regex_string)
+          return @url.gsub(%r{#{regex_string}}, WebStat::Configure.get["thumbnail_regex"][provider])
         end
       end
       readability_content = ::Nokogiri::HTML(Readability::Document.new(@nokogiri.at('body').to_s).content)

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.4.7"
+  VERSION = "0.5.0"
 end

data/spec/web_stat/configure_spec.rb

@@ -14,14 +14,15 @@ RSpec.describe WebStat::Configure do
   it "Get thumbnail_regex.youtube." do
     config = WebStat::Configure.get
     expect(config["thumbnail_regex"]["yotube"].nil?).to eq true
-    expect(config["thumbnail_regex"]["youtube"].count).to eq 2
+    expect(config["id_extraction_regexs"]["youtube"]).to be_a String
+    expect(config["thumbnail_regex"]["youtube"]).to be_a String
   end
 
   it "Match youtube url." do
     sample_url = "https://www.youtube.com/watch?v=aChpsuUffUM"
-    WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
-      if sample_url.match(v[0])
-        expect(sample_url.gsub(v[0], v[1])).to eq 'http://img.youtube.com/vi/aChpsuUffUM/default.jpg'
+    WebStat::Configure.get["id_extraction_regexs"].each do |provider, regex_string|
+      if sample_url.match(regex_string)
+        expect(sample_url.gsub(%r{#{regex_string}}, WebStat::Configure.get["thumbnail_regex"][provider])).to eq 'http://img.youtube.com/vi/aChpsuUffUM/default.jpg'
       end
     end
   end

data/web_stat.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "pdf-reader", "2.4.0"
   spec.add_runtime_dependency "webrick", ">= 1.7.0"
   spec.add_runtime_dependency "rexml", ">= 3.2.4"
+  spec.add_runtime_dependency "google-api-client", ">= 0.53.0" 
 
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", ">= 3.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.4.7
+  version: 0.5.0
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-28 00:00:00.000000000 Z
+date: 2021-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.2.4
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.53.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.53.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -238,7 +252,7 @@ description: Fetch the web pages and stat.
 email:
 - yube@newsdict.jp
 executables:
-- fetch_as_html
+- fetch_as_url
 extensions: []
 extra_rdoc_files: []
 files:
@@ -253,7 +267,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/fetch_as_html
+- bin/fetch_as_url
 - docker-compose.yml
 - docker/exec
 - docker/start