web_stat 0.3.18 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1577e388af881db0fb90000e62a9768711d6098fb256235a7976ac317a2f2f2
-  data.tar.gz: 06d1a79f46b436e20fc0730232ff4997eec11eccd939a1d7d28311780f496273
+  metadata.gz: beb43ff474210a7d97855aa4f0df4c0083b8560302b20debd27ecce87434e6af
+  data.tar.gz: ef3a1f8077801a2e0f55dda383f03bbea94b7fadaf1d83b3bc120582180794ad
 SHA512:
-  metadata.gz: 15f4eca65d1fdc2d3c4bc5d1e06817e78896774abfe384419b8416a0ff73ff6b423c15b029632e28708e58e868f0c126e86f56295b618cc4ea4d93ac6cc7d26d
-  data.tar.gz: e15ee2f11d221b1d6997a6f7aad45571c6bf9456b813e95465f5c029617b701e4f18e3b4b7ec35246d0fddefca1f7648d07bff5a585eaf18cc9859362ceb5ca9
+  metadata.gz: 3408516abee8fb1e422dcbfc29af4c2ad31d0a1a80dc15fdd893073f39408a3e797ce080bc2a28557031d64acd72431b89e82586ded7e1d64f90656c36793e61
+  data.tar.gz: 1003e6bb4d6824ae72e6ada48040b84758dc7002ce7258828f5764ae13224fc0818dc52fda533a2cd58c11c1194f31f807a37ae4e1237adc7d67038a6108dbba

data/.ruby-version

@@ -1 +1 @@
-2.7.1
+3.0.0

data/Dockerfile

@@ -1,5 +1,5 @@
 # Define base image, you can use --build-arg
-ARG base_image="newsdict/rails:ubuntu20.10_nvmv0.35.2_nodev14.3.0_rubyv2.7.1_sasscv2.3.0_ffiv1.13.1_chromedriver"
+ARG base_image="newsdict/rails:ubuntu20.10_nvmv0.37.0_nodev15.2.1_rubyv3.0.0_sasscv2.4.0_ffiv1.13.1_chromedriver"
 FROM $base_image
 
 # Set locale

data/Gemfile.lock

@@ -1,16 +1,18 @@
 PATH
   remote: .
   specs:
-    web_stat (0.3.18)
+    web_stat (0.4.3)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
       natto (>= 1.1.2)
       nokogiri (>= 1.10.4)
       pdf-reader (= 2.4.0)
+      rexml (>= 3.2.4)
       ruby-readability (>= 0.7)
       sanitize (>= 5.0.0)
       selenium-webdriver (= 3.142.7)
+      webrick (>= 1.7.0)
 
 GEM
   remote: https://rubygems.org/
@@ -24,14 +26,13 @@ GEM
     cld (0.8.0)
       ffi
     coderay (1.1.3)
-    connection_pool (2.2.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     crass (1.0.6)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.13.1)
+    ffi (1.14.2)
     guess_html_encoding (0.0.11)
     hashdiff (1.0.1)
     hashery (2.1.2)
@@ -49,16 +50,14 @@ GEM
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
-    mini_portile2 (2.4.0)
+    mime-types-data (3.2020.1104)
     natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.0)
-      connection_pool (~> 2.2)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    nokogumbo (2.0.2)
+    net-http-persistent (2.9.4)
+    nokogiri (1.11.1-x86_64-linux)
+      racc (~> 1.4)
+    nokogumbo (2.0.4)
       nokogiri (~> 1.8, >= 1.8.4)
     ntlm-http (0.1.1)
     pdf-reader (2.4.0)
@@ -74,7 +73,9 @@ GEM
       byebug (~> 11.0)
       pry (~> 0.13.0)
     public_suffix (4.0.5)
+    racc (1.5.2)
     rake (13.0.1)
+    rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -94,14 +95,14 @@ GEM
       nokogiri (>= 1.6.0)
     rubyzip (2.3.0)
     safe_yaml (1.0.5)
-    sanitize (5.2.1)
+    sanitize (5.2.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
     selenium-webdriver (3.142.7)
       childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
-    ttfunk (1.6.2.1)
+    ttfunk (1.7.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
@@ -109,6 +110,7 @@ GEM
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    webrick (1.7.0)
     webrobots (0.1.2)
 
 PLATFORMS
@@ -123,4 +125,4 @@ DEPENDENCIES
   webmock (>= 3.8.3)
 
 BUNDLED WITH
-   2.1.4
+   2.2.4

data/README.md

@@ -1,30 +1,7 @@
-# !!!!! Precautions when using with Rails !!!!!
-
-Write this line your Gemfile.
-```
-gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url"
-```
-
 # WebStat
 
 Fetch the web pages and stat.
 
-## Requirements
-
--  [MeCab _0.996_](http://taku910.github.io/mecab/#download)
-- add runtime dependency
- - "bundler", "~> 2.0"
- - "nokogiri", "~> 1.10"
- - "mechanize", "~> 2.7"
- - "ruby-readability", "~> 0.7"
- - "final_redirect_url", "~> 0.1.0"
- - "natto", "~> 1.1.2"
-- add development dependency
- - "rake", "~> 10.0"
- - "rspec", "~> 3.0"
-- "rake", "~> 10.0"
-- "rspec", "~> 3.0"
-
 ### Install mecab
 
     $ sudo apt install mecab-ipadic-utf8 libmecab

data/lib/web_stat/config/web_stat.yml

@@ -1,7 +1,7 @@
 development: &development
   # Minimum number of characters to detect meta title
   min_length_of_meta_title: 10
-  # Split regular expression for titles 
+  # Split regular expression for titles
   regex_to_sprit_title: '\||-|:|｜|：|〜|\~| – '
   # User Agent
   user_agent: "web_stat gem agent"
@@ -14,6 +14,10 @@ development: &development
     - '//img/@src'
   userdic: ""
   use_chromedirver: false
+  thumbnail_regex:
+    youtube:
+      - '%r{^https://www.youtube.com/watch\?v=([^&]+)}'
+      - 'http://img.youtube.com/vi/\1/default.jpg'
 test:
   <<: *development
 production:

data/lib/web_stat/fetch.rb

@@ -1,11 +1,5 @@
 module WebStat
   class Fetch
-    THUMBNAIL_REGEXS = {
-      :youtube => [
-        %r{^https://www.youtube.com/watch\?v=([^&]+)},
-        'http://img.youtube.com/vi/\1/default.jpg'
-        ]
-    }
     attr_accessor :url, :html, :nokogiri, :userdic, :status
     # Get title
     # @return [String] title
@@ -40,7 +34,7 @@ module WebStat
     end
     # Get main section
     def content
-      Sanitize.clean(Readability::Document.new(@nokogiri.at('body')).content)
+      Sanitize.clean(Readability::Document.new(@nokogiri.at('body').to_s).content)
     end
 
     # Get temporary path of image
@@ -54,12 +48,12 @@ module WebStat
         end
       end
       # If there is a thumbnail rule, apply it.
-      THUMBNAIL_REGEXS.each do |provider, v|
+      WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
         if @url.match(v[0])
           return @url.gsub(v[0], v[1])
         end
       end
-      readability_content = ::Nokogiri::HTML(Readability::Document.new(@nokogiri.at('body')).content)
+      readability_content = ::Nokogiri::HTML(Readability::Document.new(@nokogiri.at('body').to_s).content)
       if (path.nil? || path.empty?) && readability_content.xpath('//img').first
         path =  readability_content.xpath('//img').first.attr('src')
       end
@@ -102,7 +96,16 @@ module WebStat
           raise Mechanize::RobotsDisallowedError.new(url)
         end
         if WebStat::Configure.get["use_chromedirver"]
-          body = WebStat::WebDriverHelper.get_source(url)
+          begin
+            body = WebStat::WebDriverHelper.get_source(url)
+          rescue Selenium::WebDriver::Error::UnknownError => e
+            document = mech.agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
+            if document.class == Mechanize::File
+              body = document.body
+            else
+              body = document.body.encode('UTF-8', document.encoding)
+            end
+          end
           @status = 200
         else
           document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})

data/lib/web_stat/fetch/fetch_as_web.rb

@@ -5,7 +5,7 @@ module WebStat
     # initialize class
     # @param [String] url
     def initialize(url)
-      unless url_valid?(url)
+      unless FetchAsWeb.url_valid?(url)
         raise WebStat::INVALID_URL, url
       end
       @url = original_url(url)
@@ -36,11 +36,12 @@ module WebStat
       end
       @nokogiri = ::Nokogiri::HTML(@html)
     end
-    
-    # Validation url
-    def url_valid?(url)
-      regexp = Regexp.new("^https?://([a-zA-Z0-9][a-zA-Z0-9\\\-\.]{1,61}[a-zA-Z0-9])\\\.([a-zA-Z]{2,})(.*)?$", Regexp::IGNORECASE)
-      regexp.match?(url)
-    end 
+    class << self
+      # Validation url
+      def url_valid?(url)
+        regexp = Regexp.new("^https?://([a-z0-9][a-z0-9\\\-\.]{0,61})\\\.([a-z]{2,})(.*)?$", Regexp::IGNORECASE)
+        regexp.match?(url)
+      end
+    end
   end
 end

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.3.18"
-end
+  VERSION = "0.4.3"
+end

data/spec/web_stat/configure_spec.rb

@@ -3,11 +3,26 @@ RSpec.describe WebStat::Configure do
     configure = WebStat::Configure.get
     expect(configure).not_to eq nil
   end
-  
+
   it "Readable Config" do
     config = WebStat::Configure.get
-    
+
     expect(config["min_length_of_meta_title"]).to eq 10
     expect(config["regex_to_sprit_title"]).to eq '\||-|:|｜|：|〜|\~| – '
   end
+
+  it "Get thumbnail_regex.youtube." do
+    config = WebStat::Configure.get
+    expect(config["thumbnail_regex"]["yotube"].nil?).to eq true
+    expect(config["thumbnail_regex"]["youtube"].count).to eq 2
+  end
+
+  it "Match youtube url." do
+    sample_url = "https://www.youtube.com/watch?v=aChpsuUffUM"
+    WebStat::Configure.get["thumbnail_regex"].each do |provider, v|
+      if sample_url.match(v[0])
+        expect(sample_url.gsub(v[0], v[1])).to eq 'http://img.youtube.com/vi/aChpsuUffUM/default.jpg'
+      end
+    end
+  end
 end

data/spec/web_stat/fetch_spec.rb

@@ -197,11 +197,11 @@ RSpec.describe WebStat::Fetch do
   end
   
   it "valid url" do
-    web_stat_fetch_web_class = WebStat::FetchAsWeb.new("https://newsdict.blog/content/images/size/w100/2019/03/facebook-3.jpg")
-    expect(web_stat_fetch_web_class.url_valid?("http://status.aws.amazon.com/#cloudfront_12345")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://findy-code.io?h=NWsZey5UgJ51u&t=omikuji-22")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://www.meetup.com/pro/docker")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://gxyt4.app.goo.gl/Mn64U")).to be true
-    expect(web_stat_fetch_web_class.url_valid?("https://status.cloud.google.com/incident/cloud-functions/19010")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("http://status.aws.amazon.com/#cloudfront_12345")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://findy-code.io?h=NWsZey5UgJ51u&t=omikuji-22")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://www.meetup.com/pro/docker")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://gxyt4.app.goo.gl/Mn64U")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("https://status.cloud.google.com/incident/cloud-functions/19010")).to be true
+    expect(WebStat::FetchAsWeb.url_valid?("http://g.co/arts/SK1jZHJpT8N1BGaM7")).to be true
   end
 end

data/web_stat.gemspec

@@ -29,6 +29,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "cld", ">= 0.8.0"
   spec.add_runtime_dependency "selenium-webdriver", "= 3.142.7"
   spec.add_runtime_dependency "pdf-reader", "2.4.0"
+  spec.add_runtime_dependency "webrick", ">= 1.7.0"
+  spec.add_runtime_dependency "rexml", ">= 3.2.4"
 
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", ">= 3.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.3.18
+  version: 0.4.3
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-11 00:00:00.000000000 Z
+date: 2021-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.4.0
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -274,7 +302,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
 summary: Get the status of  the web pages.