web_stat 0.3.15 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82732f779004a5a2ef1f7259ce40126de8f51dfb07e46cd4972aafd3ab386ac3
-  data.tar.gz: 17c2b99bdeb5db8c134d107fc9d4957dab985c7e680b1c92060bf7090315477b
+  metadata.gz: 29d04d8379978d1e295b829193a330fd9bab8cb92326e83aab39d066059dcb02
+  data.tar.gz: c2df9896f21ec9d777dea101a1c4c4be513ad28c11428fc1104ccefe08f98b40
 SHA512:
-  metadata.gz: d6691a57b0498fcfbb609042cdb7e12b4ac453b5cedaf6dd39671090c97fc19958a4162ba50947392116320ba78c9a21491acab27be2a41287825d3d4d2194d1
-  data.tar.gz: 1e2e3d33f5c232532f442bbd16540c47810c93caef436a351124165b5294ddc7354d766ea66aa24ebc052c69432a639a6d5ef7b61a7cde896f6dc4676f95650d
+  metadata.gz: 728bc129ddced4cc58081ca4d8c02b761ba77e4adf277b736115ce51b1e7293f7b208adc5742fe50ba4e406f2a7c3f65af19d5c55f0cf0188a4a4b25704719bd
+  data.tar.gz: ef8ee476834a6fb75fed33476d070e1b00044f9115b85eaf3148f1901af10e4cd32f79ef3b369e59366a56ab7e8560180da3def8562e2b396f2b8b6bfc26196b

data/.ruby-version

@@ -1 +1 @@
-2.7.1
+3.0.0

data/Dockerfile

@@ -1,5 +1,5 @@
 # Define base image, you can use --build-arg
-ARG base_image="newsdict/rails:ubuntu20.10_nvmv0.35.2_nodev14.3.0_rubyv2.7.1_sasscv2.3.0_ffiv1.13.1_chromedriver"
+ARG base_image="newsdict/rails:ubuntu20.10_nvmv0.37.0_nodev15.2.1_rubyv3.0.0_sasscv2.4.0_ffiv1.13.1_chromedriver"
 FROM $base_image
 
 # Set locale

data/Gemfile.lock

@@ -1,16 +1,18 @@
 PATH
   remote: .
   specs:
-    web_stat (0.3.15)
+    web_stat (0.3.19)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
       natto (>= 1.1.2)
       nokogiri (>= 1.10.4)
       pdf-reader (= 2.4.0)
+      rexml (>= 3.2.4)
       ruby-readability (>= 0.7)
       sanitize (>= 5.0.0)
       selenium-webdriver (= 3.142.7)
+      webrick (>= 1.7.0)
 
 GEM
   remote: https://rubygems.org/
@@ -24,14 +26,13 @@ GEM
     cld (0.8.0)
       ffi
     coderay (1.1.3)
-    connection_pool (2.2.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     crass (1.0.6)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.13.1)
+    ffi (1.14.2)
     guess_html_encoding (0.0.11)
     hashdiff (1.0.1)
     hashery (2.1.2)
@@ -49,16 +50,14 @@ GEM
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
-    mini_portile2 (2.4.0)
+    mime-types-data (3.2020.1104)
     natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.0)
-      connection_pool (~> 2.2)
-    nokogiri (1.10.9)
-      mini_portile2 (~> 2.4.0)
-    nokogumbo (2.0.2)
+    net-http-persistent (2.9.4)
+    nokogiri (1.11.1-x86_64-linux)
+      racc (~> 1.4)
+    nokogumbo (2.0.4)
       nokogiri (~> 1.8, >= 1.8.4)
     ntlm-http (0.1.1)
     pdf-reader (2.4.0)
@@ -74,7 +73,9 @@ GEM
       byebug (~> 11.0)
       pry (~> 0.13.0)
     public_suffix (4.0.5)
+    racc (1.5.2)
     rake (13.0.1)
+    rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -94,14 +95,14 @@ GEM
       nokogiri (>= 1.6.0)
     rubyzip (2.3.0)
     safe_yaml (1.0.5)
-    sanitize (5.2.1)
+    sanitize (5.2.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
     selenium-webdriver (3.142.7)
       childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
-    ttfunk (1.6.2.1)
+    ttfunk (1.7.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
@@ -109,6 +110,7 @@ GEM
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    webrick (1.7.0)
     webrobots (0.1.2)
 
 PLATFORMS
@@ -123,4 +125,4 @@ DEPENDENCIES
   webmock (>= 3.8.3)
 
 BUNDLED WITH
-   2.1.4
+   2.2.4

data/README.md

@@ -1,30 +1,7 @@
-# !!!!! Precautions when using with Rails !!!!!
-
-Write this line your Gemfile.
-```
-gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url"
-```
-
 # WebStat
 
 Fetch the web pages and stat.
 
-## Requirements
-
--  [MeCab _0.996_](http://taku910.github.io/mecab/#download)
-- add runtime dependency
- - "bundler", "~> 2.0"
- - "nokogiri", "~> 1.10"
- - "mechanize", "~> 2.7"
- - "ruby-readability", "~> 0.7"
- - "final_redirect_url", "~> 0.1.0"
- - "natto", "~> 1.1.2"
-- add development dependency
- - "rake", "~> 10.0"
- - "rspec", "~> 3.0"
-- "rake", "~> 10.0"
-- "rspec", "~> 3.0"
-
 ### Install mecab
 
     $ sudo apt install mecab-ipadic-utf8 libmecab

data/lib/web_stat/config/web_stat.yml

@@ -1,7 +1,7 @@
 development: &development
   # Minimum number of characters to detect meta title
   min_length_of_meta_title: 10
-  # Split regular expression for titles 
+  # Split regular expression for titles
   regex_to_sprit_title: '\||-|:|｜|：|〜|\~| – '
   # User Agent
   user_agent: "web_stat gem agent"
@@ -14,6 +14,10 @@ development: &development
     - '//img/@src'
   userdic: ""
   use_chromedirver: false
+  thumbnail_regex:
+    youtube:
+      - '%r{^https://www.youtube.com/watch\?v=([^&]+)}'
+      - 'http://img.youtube.com/vi/\1/default.jpg'
 test:
   <<: *development
 production:

data/lib/web_stat/fetch.rb

@@ -1,7 +1,6 @@
 module WebStat
   class Fetch
     attr_accessor :url, :html, :nokogiri, :userdic, :status
-
     # Get title
     # @return [String] title
     def title
@@ -19,7 +18,8 @@ module WebStat
         title.strip
       end
     end
-    # Get name of domain 
+
+    # Get name of domain
     def site_name
       begin
         site_name = @nokogiri.title.split(/#{WebStat::Configure.get["regex_to_sprit_title"]}/, 2).last
@@ -34,9 +34,9 @@ module WebStat
     end
     # Get main section
     def content
-      Sanitize.clean(Readability::Document.new(@nokogiri.at('body')).content)
+      Sanitize.clean(Readability::Document.new(@nokogiri.at('body').to_s).content)
     end
-      
+
     # Get temporary path of image
     def eyecatch_image_path
       # Reuse `path` in this method
@@ -47,9 +47,15 @@ module WebStat
           break
         end
       end
-      readability_content = ::Nokogiri::HTML(Readability::Document.new(@nokogiri.at('body')).content)
+      # If there is a thumbnail rule, apply it.
+      WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
+        if @url.match(v[0])
+          return @url.gsub(v[0], v[1])
+        end
+      end
+      readability_content = ::Nokogiri::HTML(Readability::Document.new(@nokogiri.at('body').to_s).content)
       if (path.nil? || path.empty?) && readability_content.xpath('//img').first
-        path = readability_content.xpath('//img').first.attr('src')
+        path =  readability_content.xpath('//img').first.attr('src')
       end
       if (path.nil? || path.empty?) && @nokogiri.xpath('//img').first
         path = @nokogiri.xpath('//img').first.attr('src')
@@ -60,7 +66,7 @@ module WebStat
         path
       end
     end
-    
+
     # Get local path to save url
     # @param [String] url
     def save_local_path(url)
@@ -71,13 +77,13 @@ module WebStat
       File.open(tmp_file, "w+b") do |_file|
         if image.class == Mechanize::File
           _file.puts(image.body)
-        else
+        elsif image.respond_to?(:body_io)
           _file.puts(image.body_io.read)
         end
       end
       tmp_file
     end
-    
+
     # Get url
     # @param [String] url
     # @param [String] body
@@ -107,7 +113,7 @@ module WebStat
       end
       body
     end
-    
+
     # Get the informations of @url
     # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
     def stat(userdics: nil)

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.3.15"
+  VERSION = "0.4.0"
 end

data/spec/web_stat/configure_spec.rb

@@ -3,11 +3,26 @@ RSpec.describe WebStat::Configure do
     configure = WebStat::Configure.get
     expect(configure).not_to eq nil
   end
-  
+
   it "Readable Config" do
     config = WebStat::Configure.get
-    
+
     expect(config["min_length_of_meta_title"]).to eq 10
     expect(config["regex_to_sprit_title"]).to eq '\||-|:|｜|：|〜|\~| – '
   end
+
+  it "Get thumbnail_regex.youtube." do
+    config = WebStat::Configure.get
+    expect(config["thumbnail_regex"]["yotube"].nil?).to eq true
+    expect(config["thumbnail_regex"]["youtube"].count).to eq 2
+  end
+
+  it "Match youtube url." do
+    sample_url = "https://www.youtube.com/watch?v=aChpsuUffUM"
+    WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
+      if sample_url.match(v[0])
+        expect(sample_url.gsub(v[0], v[1])).to eq 'http://img.youtube.com/vi/aChpsuUffUM/default.jpg'
+      end
+    end
+  end
 end

data/web_stat.gemspec

@@ -29,6 +29,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "cld", ">= 0.8.0"
   spec.add_runtime_dependency "selenium-webdriver", "= 3.142.7"
   spec.add_runtime_dependency "pdf-reader", "2.4.0"
+  spec.add_runtime_dependency "webrick", ">= 1.7.0"
+  spec.add_runtime_dependency "rexml", ">= 3.2.4"
 
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", ">= 3.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.3.15
+  version: 0.4.0
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-03 00:00:00.000000000 Z
+date: 2021-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.4.0
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -274,7 +302,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
 summary: Get the status of  the web pages.