web_stat 0.4.6 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ce60bcf4a31f90024abc35cf1ecc57e32626dfb5ecf4bc4f2280bd72931ff34
-  data.tar.gz: 575f805a63a995b2d0e3bc909978dcd4e3b4f15462f717cf83d6aac00c96078d
+  metadata.gz: e09b39af40b8307541b0c9be774a603c893fa15a1e3dbf0ef4cc3b75912a7c56
+  data.tar.gz: d0f291a214b07adeba7679c82c1a710a60797eb4a3924badcd2d2fee96d750a2
 SHA512:
-  metadata.gz: 7af1262b25163205eabdfa26e1671f95d53963387eceec6c2c99da0a3a17359b77aaa097ec9556010b8000ecc0772d1dc19a67d128310c8d6bdd8379d008a913
-  data.tar.gz: 855706ad8525609e8a2a50ca64091081794940d69187f951a79c02c66acfa5be9f53cdeacf0f99f6c0431ad41a71ce616691590d40bce0a56576275fba96c453
+  metadata.gz: 3a150d48af70feb1840c1aeaebe5720c8ea8b0600c2571427130de835414d96d0fa2a14b7cff85e36484ed801102e394d96971b2c681725db6eb3d3333a3f641
+  data.tar.gz: 708a5bcce8bd1313d05ace67ad5db3351e54f7f7e6c1798061256699c6c99129e2a7c3d9e05ad7052dbdd497c15b8fb0d8afe091484697fe1b294731fb3fcc24

data/lib/web_stat.rb

@@ -2,6 +2,7 @@ require "bundler"
 
 require 'cld'
 require 'uri'
+require 'erb'
 require 'digest'
 require 'logger'
 require 'sanitize'
@@ -11,6 +12,7 @@ require 'net/http'
 require 'pdf/reader'
 require 'ruby-readability'
 require 'selenium-webdriver'
+require 'google/apis/youtube_v3'
 
 require "helpers/web_drive_helper"
 require "web_stat/final_redirect_url"
@@ -18,6 +20,7 @@ require "web_stat/categorize"
 require "web_stat/configure"
 require "web_stat/errors"
 require "web_stat/fetch"
+
 require "web_stat/tag"
 require "web_stat/version"
 require "web_stat/fetch/fetch_as_html"

data/lib/web_stat/config/web_stat.yml

@@ -14,10 +14,13 @@ development: &development
     - '//img/@src'
   userdic: ""
   use_chromedirver: false
+  id_extraction_regexs:
+    youtube: '^https://www.youtube.com/watch\?v=([^&]+)'
   thumbnail_regex:
-    youtube:
-      - '%r{^https://www.youtube.com/watch\?v=([^&]+)}'
-      - 'http://img.youtube.com/vi/\1/default.jpg'
+    youtube: 'http://img.youtube.com/vi/\1/default.jpg'
+  api_keys:
+    youtube: "dummy-key"
+  lang: <%= ENV['LANG'] %>
 test:
   <<: *development
 production:

data/lib/web_stat/configure.rb

@@ -7,9 +7,9 @@ module WebStat
       # Get yaml
       def get
         if defined? Rails
-          YAML.load_file(get_configure_path)[Rails.env]
+          YAML.load(ERB.new(File.read(get_configure_path)).result)[Rails.env]
         else
-          YAML.load_file(get_configure_path)[ENV["ENV"] || "production"]
+          YAML.load(ERB.new(File.read(get_configure_path)).result)[ENV["ENV"] || "production"]
         end
       end
       

data/lib/web_stat/fetch.rb

@@ -34,7 +34,23 @@ module WebStat
     end
     # Get main section
     def content
-      Sanitize.clean(Readability::Document.new(@nokogiri.at('body').to_s).content)
+      if @url&.match(WebStat::Configure.get["id_extraction_regexs"]["youtube"])
+        youtube_decscription
+      else
+        Sanitize.clean(Readability::Document.new(@nokogiri.at('body').to_s).content)
+      end
+    end
+    
+    # Get describe of youtube movie.
+    def youtube_decscription
+      regex_string = WebStat::Configure.get["id_extraction_regexs"]["youtube"]
+      if @url.match(regex_string)
+        id = @url.gsub(%r{#{regex_string}.*$}, '\1')
+        youtube = Google::Apis::YoutubeV3::YouTubeService.new
+        youtube.key = WebStat::Configure.get["api_keys"]["youtube"]
+        response = youtube.list_videos(:snippet, id: id)
+        response.items.first.snippet.description
+      end
     end
 
     # Get temporary path of image
@@ -48,9 +64,9 @@ module WebStat
         end
       end
       # If there is a thumbnail rule, apply it.
-      WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
-        if @url.match(v[0])
-          return @url.gsub(v[0], v[1])
+      WebStat::Configure.get["id_extraction_regexs"].each do |provider, regex_string|
+        if @url.match(regex_string)
+          return @url.gsub(%r{#{regex_string}.*$}, WebStat::Configure.get["thumbnail_regex"][provider])
         end
       end
       readability_content = ::Nokogiri::HTML(Readability::Document.new(@nokogiri.at('body').to_s).content)
@@ -82,6 +98,8 @@ module WebStat
         end
       end
       tmp_file
+    rescue
+      false
     end
 
     # Get url

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.4.6"
-end
+  VERSION = "0.5.3"
+end

data/spec/web_stat/configure_spec.rb

@@ -14,15 +14,20 @@ RSpec.describe WebStat::Configure do
   it "Get thumbnail_regex.youtube." do
     config = WebStat::Configure.get
     expect(config["thumbnail_regex"]["yotube"].nil?).to eq true
-    expect(config["thumbnail_regex"]["youtube"].count).to eq 2
+    expect(config["id_extraction_regexs"]["youtube"]).to be_a String
+    expect(config["thumbnail_regex"]["youtube"]).to be_a String
   end
 
   it "Match youtube url." do
     sample_url = "https://www.youtube.com/watch?v=aChpsuUffUM"
-    WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
-      if sample_url.match(v[0])
-        expect(sample_url.gsub(v[0], v[1])).to eq 'http://img.youtube.com/vi/aChpsuUffUM/default.jpg'
+    WebStat::Configure.get["id_extraction_regexs"].each do |provider, regex_string|
+      if sample_url.match(regex_string)
+        expect(sample_url.gsub(%r{#{regex_string}}, WebStat::Configure.get["thumbnail_regex"][provider])).to eq 'http://img.youtube.com/vi/aChpsuUffUM/default.jpg'
       end
     end
   end
+  
+  it "Environment variables can be read on YAML" do
+    expect(WebStat::Configure.get["lang"]).to eq "C.UTF-8"
+  end
 end

data/web_stat.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "pdf-reader", "2.4.0"
   spec.add_runtime_dependency "webrick", ">= 1.7.0"
   spec.add_runtime_dependency "rexml", ">= 3.2.4"
+  spec.add_runtime_dependency "google-api-client", ">= 0.53.0" 
 
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", ">= 3.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.5.3
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-24 00:00:00.000000000 Z
+date: 2021-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.2.4
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.53.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.53.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -238,7 +252,7 @@ description: Fetch the web pages and stat.
 email:
 - yube@newsdict.jp
 executables:
-- fetch_as_html
+- fetch_as_url
 extensions: []
 extra_rdoc_files: []
 files:
@@ -249,11 +263,10 @@ files:
 - CODE_OF_CONDUCT.md
 - Dockerfile
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/fetch_as_html
+- bin/fetch_as_url
 - docker-compose.yml
 - docker/exec
 - docker/start

data/Gemfile.lock

@@ -1,132 +0,0 @@
-PATH
-  remote: .
-  specs:
-    web_stat (0.4.6)
-      bundler (>= 2.0.2)
-      cld (>= 0.8.0)
-      mechanize (>= 2.7.7)
-      natto (>= 1.1.2)
-      nokogiri (>= 1.10.4)
-      pdf-reader (= 2.4.0)
-      rexml (>= 3.2.4)
-      ruby-readability (>= 0.7)
-      sanitize (>= 5.0.0)
-      selenium-webdriver (= 3.142.7)
-      webrick (>= 1.7.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    Ascii85 (1.0.3)
-    addressable (2.7.0)
-      public_suffix (>= 2.0.2, < 5.0)
-    afm (0.2.2)
-    byebug (11.1.3)
-    childprocess (3.0.0)
-    cld (0.8.0)
-      ffi
-    coderay (1.1.3)
-    connection_pool (2.2.3)
-    crack (0.4.5)
-      rexml
-    crass (1.0.6)
-    diff-lcs (1.4.4)
-    domain_name (0.5.20190701)
-      unf (>= 0.0.5, < 1.0.0)
-    ffi (1.14.2)
-    guess_html_encoding (0.0.11)
-    hashdiff (1.0.1)
-    hashery (2.1.2)
-    http-cookie (1.0.3)
-      domain_name (~> 0.5)
-    mechanize (2.7.7)
-      domain_name (~> 0.5, >= 0.5.1)
-      http-cookie (~> 1.0)
-      mime-types (>= 1.17.2)
-      net-http-digest_auth (~> 1.1, >= 1.1.1)
-      net-http-persistent (>= 2.5.2)
-      nokogiri (~> 1.6)
-      ntlm-http (~> 0.1, >= 0.1.1)
-      webrick (~> 1.7)
-      webrobots (>= 0.0.9, < 0.2)
-    method_source (1.0.0)
-    mime-types (3.3.1)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0212)
-    mini_portile2 (2.5.0)
-    natto (1.2.0)
-      ffi (>= 1.9.0)
-    net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.1)
-      connection_pool (~> 2.2)
-    nokogiri (1.11.1)
-      mini_portile2 (~> 2.5.0)
-      racc (~> 1.4)
-    nokogumbo (2.0.4)
-      nokogiri (~> 1.8, >= 1.8.4)
-    ntlm-http (0.1.1)
-    pdf-reader (2.4.0)
-      Ascii85 (~> 1.0.0)
-      afm (~> 0.2.1)
-      hashery (~> 2.0)
-      ruby-rc4
-      ttfunk
-    pry (0.13.1)
-      coderay (~> 1.1)
-      method_source (~> 1.0)
-    pry-byebug (3.9.0)
-      byebug (~> 11.0)
-      pry (~> 0.13.0)
-    public_suffix (4.0.6)
-    racc (1.5.2)
-    rake (13.0.3)
-    rexml (3.2.4)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    ruby-rc4 (0.1.5)
-    ruby-readability (0.7.0)
-      guess_html_encoding (>= 0.0.4)
-      nokogiri (>= 1.6.0)
-    rubyzip (2.3.0)
-    sanitize (5.2.3)
-      crass (~> 1.0.2)
-      nokogiri (>= 1.8.0)
-      nokogumbo (~> 2.0)
-    selenium-webdriver (3.142.7)
-      childprocess (>= 0.5, < 4.0)
-      rubyzip (>= 1.2.2)
-    ttfunk (1.7.0)
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.7.7)
-    webmock (3.11.2)
-      addressable (>= 2.3.6)
-      crack (>= 0.3.2)
-      hashdiff (>= 0.4.0, < 2.0.0)
-    webrick (1.7.0)
-    webrobots (0.1.2)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  pry (>= 0.13.1)
-  pry-byebug (= 3.9.0)
-  rake (>= 10.0)
-  rspec (>= 3.0)
-  web_stat!
-  webmock (>= 3.8.3)
-
-BUNDLED WITH
-   2.2.4