web_stat 0.3.16 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c332bb9cf67262c2b5d8e3c30c861107bd6ac8e3d86136730864e49b10dabcce
-  data.tar.gz: 712f39109989e917e8af6b84fe40b64ce84009a9de4b3647c03177ecb69bc58d
+  metadata.gz: 9fc357cc98a214af7a391fe834ec927b43649ca8d8c5b1105e7fb1947dd7127a
+  data.tar.gz: f7c5f6d28101ff677aaa4db5b9b486d067798b0a8461f91352db252f68374513
 SHA512:
-  metadata.gz: 3607fbe1e76018e3e523ed26a2777837c539a80f5c68da195996616d3566f6034125f0d49e96db9338ae40aeda5dac000968f6e089a7d96c19a412f14d2acbd2
-  data.tar.gz: 027d3b911cbee0dfdd7a8f85b32d24e4b8c9e2f916a47a2d3ea31b34fe0e2f07b452e00372d2028d511d39fba7df8f26347789fd795103422711d0e979b1d8a2
+  metadata.gz: 1616c4f1a64ebc693ee4e822b06150028783736448850d8aca80c90909ca879580a82f871f95375904b38813b10155105fd5d1121ed0321ae9cc898a4b95549e
+  data.tar.gz: 1637fb0b6b6f8ee83b133d95e3ab31a6c4ad2a64399f4ac19173f215ec7a79eac851c1348d9efa1efbfdc88a7ac8e5d6ecd9f9c067c4295b0a82f2662ef4c448

data/.ruby-version

@@ -1 +1 @@
-2.7.1
+3.0.0

data/Dockerfile

@@ -1,5 +1,5 @@
 # Define base image, you can use --build-arg
-ARG base_image="newsdict/rails:ubuntu20.10_nvmv0.35.2_nodev14.3.0_rubyv2.7.1_sasscv2.3.0_ffiv1.13.1_chromedriver"
+ARG base_image="newsdict/rails:ubuntu20.10_nvmv0.37.0_nodev15.2.1_rubyv3.0.0_sasscv2.4.0_ffiv1.13.1_chromedriver"
 FROM $base_image
 
 # Set locale

data/Gemfile.lock

@@ -1,16 +1,18 @@
 PATH
   remote: .
   specs:
-    web_stat (0.3.15)
+    web_stat (0.4.1)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
       natto (>= 1.1.2)
       nokogiri (>= 1.10.4)
       pdf-reader (= 2.4.0)
+      rexml (>= 3.2.4)
       ruby-readability (>= 0.7)
       sanitize (>= 5.0.0)
       selenium-webdriver (= 3.142.7)
+      webrick (>= 1.7.0)
 
 GEM
   remote: https://rubygems.org/
@@ -24,14 +26,13 @@ GEM
     cld (0.8.0)
       ffi
     coderay (1.1.3)
-    connection_pool (2.2.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     crass (1.0.6)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.13.1)
+    ffi (1.14.2)
     guess_html_encoding (0.0.11)
     hashdiff (1.0.1)
     hashery (2.1.2)
@@ -49,16 +50,14 @@ GEM
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
-    mini_portile2 (2.4.0)
+    mime-types-data (3.2020.1104)
     natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.0)
-      connection_pool (~> 2.2)
-    nokogiri (1.10.9)
-      mini_portile2 (~> 2.4.0)
-    nokogumbo (2.0.2)
+    net-http-persistent (2.9.4)
+    nokogiri (1.11.1-x86_64-linux)
+      racc (~> 1.4)
+    nokogumbo (2.0.4)
       nokogiri (~> 1.8, >= 1.8.4)
     ntlm-http (0.1.1)
     pdf-reader (2.4.0)
@@ -74,7 +73,9 @@ GEM
       byebug (~> 11.0)
       pry (~> 0.13.0)
     public_suffix (4.0.5)
+    racc (1.5.2)
     rake (13.0.1)
+    rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -94,14 +95,14 @@ GEM
       nokogiri (>= 1.6.0)
     rubyzip (2.3.0)
     safe_yaml (1.0.5)
-    sanitize (5.2.1)
+    sanitize (5.2.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
     selenium-webdriver (3.142.7)
       childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
-    ttfunk (1.6.2.1)
+    ttfunk (1.7.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
@@ -109,6 +110,7 @@ GEM
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    webrick (1.7.0)
     webrobots (0.1.2)
 
 PLATFORMS
@@ -123,4 +125,4 @@ DEPENDENCIES
   webmock (>= 3.8.3)
 
 BUNDLED WITH
-   2.1.4
+   2.2.4

data/README.md

@@ -1,30 +1,7 @@
-# !!!!! Precautions when using with Rails !!!!!
-
-Write this line your Gemfile.
-```
-gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url"
-```
-
 # WebStat
 
 Fetch the web pages and stat.
 
-## Requirements
-
--  [MeCab _0.996_](http://taku910.github.io/mecab/#download)
-- add runtime dependency
- - "bundler", "~> 2.0"
- - "nokogiri", "~> 1.10"
- - "mechanize", "~> 2.7"
- - "ruby-readability", "~> 0.7"
- - "final_redirect_url", "~> 0.1.0"
- - "natto", "~> 1.1.2"
-- add development dependency
- - "rake", "~> 10.0"
- - "rspec", "~> 3.0"
-- "rake", "~> 10.0"
-- "rspec", "~> 3.0"
-
 ### Install mecab
 
     $ sudo apt install mecab-ipadic-utf8 libmecab

data/lib/web_stat/config/web_stat.yml

@@ -1,7 +1,7 @@
 development: &development
   # Minimum number of characters to detect meta title
   min_length_of_meta_title: 10
-  # Split regular expression for titles 
+  # Split regular expression for titles
   regex_to_sprit_title: '\||-|:|｜|：|〜|\~| – '
   # User Agent
   user_agent: "web_stat gem agent"
@@ -14,6 +14,10 @@ development: &development
     - '//img/@src'
   userdic: ""
   use_chromedirver: false
+  thumbnail_regex:
+    youtube:
+      - '%r{^https://www.youtube.com/watch\?v=([^&]+)}'
+      - 'http://img.youtube.com/vi/\1/default.jpg'
 test:
   <<: *development
 production:

data/lib/web_stat/fetch.rb

@@ -1,11 +1,5 @@
 module WebStat
   class Fetch
-    THUMBNAIL_REGEXS = {
-      :youtube => [
-        %r{^https://www.youtube.com/watch\?v=([^&]+)},
-        'http://img.youtube.com/vi/\1/default.jpg'
-        ]
-    }
     attr_accessor :url, :html, :nokogiri, :userdic, :status
     # Get title
     # @return [String] title
@@ -40,7 +34,7 @@ module WebStat
     end
     # Get main section
     def content
-      Sanitize.clean(Readability::Document.new(@nokogiri.at('body')).content)
+      Sanitize.clean(Readability::Document.new(@nokogiri.at('body').to_s).content)
     end
 
     # Get temporary path of image
@@ -54,12 +48,12 @@ module WebStat
         end
       end
       # If there is a thumbnail rule, apply it.
-      THUMBNAIL_REGEXS.each do |provider, v|
+      WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
         if @url.match(v[0])
           return @url.gsub(v[0], v[1])
         end
       end
-      readability_content = ::Nokogiri::HTML(Readability::Document.new(@nokogiri.at('body')).content)
+      readability_content = ::Nokogiri::HTML(Readability::Document.new(@nokogiri.at('body').to_s).content)
       if (path.nil? || path.empty?) && readability_content.xpath('//img').first
         path =  readability_content.xpath('//img').first.attr('src')
       end
@@ -83,7 +77,7 @@ module WebStat
       File.open(tmp_file, "w+b") do |_file|
         if image.class == Mechanize::File
           _file.puts(image.body)
-        else
+        elsif image.respond_to?(:body_io)
           _file.puts(image.body_io.read)
         end
       end
@@ -102,7 +96,16 @@ module WebStat
           raise Mechanize::RobotsDisallowedError.new(url)
         end
         if WebStat::Configure.get["use_chromedirver"]
-          body = WebStat::WebDriverHelper.get_source(url)
+          begin
+            body = WebStat::WebDriverHelper.get_source(url)
+          rescue Selenium::WebDriver::Error::UnknownError => e
+            document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
+            if document.class == Mechanize::File
+              body = document.body
+            else
+              body = document.body.encode('UTF-8', document.encoding)
+            end
+          end
           @status = 200
         else
           document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.3.16"
+  VERSION = "0.4.1"
 end

data/spec/web_stat/configure_spec.rb

@@ -3,11 +3,26 @@ RSpec.describe WebStat::Configure do
     configure = WebStat::Configure.get
     expect(configure).not_to eq nil
   end
-  
+
   it "Readable Config" do
     config = WebStat::Configure.get
-    
+
     expect(config["min_length_of_meta_title"]).to eq 10
     expect(config["regex_to_sprit_title"]).to eq '\||-|:|｜|：|〜|\~| – '
   end
+
+  it "Get thumbnail_regex.youtube." do
+    config = WebStat::Configure.get
+    expect(config["thumbnail_regex"]["yotube"].nil?).to eq true
+    expect(config["thumbnail_regex"]["youtube"].count).to eq 2
+  end
+
+  it "Match youtube url." do
+    sample_url = "https://www.youtube.com/watch?v=aChpsuUffUM"
+    WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
+      if sample_url.match(v[0])
+        expect(sample_url.gsub(v[0], v[1])).to eq 'http://img.youtube.com/vi/aChpsuUffUM/default.jpg'
+      end
+    end
+  end
 end

data/web_stat.gemspec

@@ -29,6 +29,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "cld", ">= 0.8.0"
   spec.add_runtime_dependency "selenium-webdriver", "= 3.142.7"
   spec.add_runtime_dependency "pdf-reader", "2.4.0"
+  spec.add_runtime_dependency "webrick", ">= 1.7.0"
+  spec.add_runtime_dependency "rexml", ">= 3.2.4"
 
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", ">= 3.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.3.16
+  version: 0.4.1
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-07 00:00:00.000000000 Z
+date: 2021-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.4.0
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -274,7 +302,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
 summary: Get the status of  the web pages.