web_stat 0.2.9 → 0.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6851adcf7a19adb66dbfc7f241cd2b659878ef954836e5656d42d3540a117fd8
-  data.tar.gz: '013188fbf26bd0ccce741afc7500060e40a33650e89fd7c6a1dc4d2ad216fcf3'
+  metadata.gz: 5821645b40a23008360e4e9717ec9f420fdd5adc5a7b4fdfea6d08de111af27f
+  data.tar.gz: c78ddd43e9ad8691dd05481e2854e4280bfcda857cb3fa97624c947c6233af75
 SHA512:
-  metadata.gz: a1bf10c3439fbe6e93b1368970725838f6b649854b5eac8e03bfd3e3b1e16f2e99b5a5bfe324980f51073e75e74a890b6a9e8d49735f28ca638d48eb36c0696f
-  data.tar.gz: 5fc04389537b37d597200c50de8691010e46308063804a29bbbfd1ce2406c592f9d11678752e748606c45061b3d55e2b694ad590af1f486cfc166f646b6ec6c9
+  metadata.gz: 439ad9a5e969057996d802374739bb587648d6fbc27a9b2fe0da71fdc36345c87b2b483fe474f8d499769d107cab6c335cf41f5575363b2a25edc2c1f88f7cd2
+  data.tar.gz: 0e6942690a71294c2402111e21605715d3191bae076167febf2df7277d50da3c8ac2682ef319e570b7f3763ce9515cb7410243f901fb9d3d6a5f86bb67177537

data/Gemfile.lock

@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    web_stat (0.2.9)
+    web_stat (0.2.10)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
@@ -27,13 +27,13 @@ GEM
     connection_pool (2.2.2)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    crass (1.0.5)
+    crass (1.0.6)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.11.3)
+    ffi (1.12.2)
     guess_html_encoding (0.0.11)
-    hashdiff (1.0.0)
+    hashdiff (1.0.1)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     mechanize (2.7.6)
@@ -45,39 +45,39 @@ GEM
       nokogiri (~> 1.6)
       ntlm-http (~> 0.1, >= 0.1.1)
       webrobots (>= 0.0.9, < 0.2)
-    method_source (0.9.2)
+    method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0425)
     mini_portile2 (2.4.0)
-    natto (1.1.2)
+    natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (3.1.0)
+    net-http-persistent (4.0.0)
       connection_pool (~> 2.2)
-    nokogiri (1.10.7)
+    nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
     nokogumbo (2.0.2)
       nokogiri (~> 1.8, >= 1.8.4)
     ntlm-http (0.1.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    public_suffix (4.0.2)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    public_suffix (4.0.4)
     rake (13.0.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.1)
-      rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.2)
+    rspec-support (3.9.3)
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
@@ -88,8 +88,8 @@ GEM
       nokogumbo (~> 2.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
-    webmock (3.7.6)
+    unf_ext (0.0.7.7)
+    webmock (3.8.3)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
@@ -107,4 +107,4 @@ DEPENDENCIES
   webmock (>= 3.6.0)
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -25,6 +25,10 @@ Fetch the web pages and stat.
 - "rake", "~> 10.0"
 - "rspec", "~> 3.0"
 
+### Install mecab
+
+    $ sudo apt install mecab-ipadic-utf8 libmecab
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/lib/web_stat/fetch.rb

@@ -1,6 +1,6 @@
 module WebStat
   class Fetch
-    attr_accessor :url, :html, :nokogiri, :userdic
+    attr_accessor :url, :html, :nokogiri, :userdic, :status
 
     # Get title
     # @return [String] title
@@ -67,16 +67,24 @@ module WebStat
     
     # Get url
     # @param [String] url
+    # @param [String] body
     def get_url(url)
       agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
       # Enable to read Robots.txt
       agent.robots = true
-      document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
-      if document.class == Mechanize::File
-        document.body
-      else
-        document.body.encode('UTF-8', document.encoding)
+      begin
+        document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
+        if document.class == Mechanize::File
+          body = document.body
+        else
+          body = document.body.encode('UTF-8', document.encoding)
+        end
+        @status = document.code
+      rescue Mechanize::ResponseCodeError => e
+        body = e.page.body
+        @status = e.page.code
       end
+      body
     end
     
     # Get the informations of @url
@@ -96,6 +104,7 @@ module WebStat
         site_name: site_name,
         content: clean_content,
         language_code: language_code,
+        status: @status,
         url: @url,
         eyecatch_image_path: save_local_path(eyecatch_image_path),
         tags: tag.nouns
@@ -107,8 +116,9 @@ module WebStat
     # Get original url
     # @param [String] url
     def original_url(url)
-      if url.match(/^http/)
-        FinalRedirectUrl.final_redirect_url(url)
+      last_url = FinalRedirectUrl.final_redirect_url(url)
+      unless last_url.nil? || last_url.scrub('').empty?
+        last_url
       else
         url
       end

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.2.9"
-end
+  VERSION = "0.2.10"
+end

data/spec/spec_helper.rb

@@ -59,9 +59,10 @@ end
 
 # Set webmock
 WebStatTestHelper.scheme_and_files.each do |url|
+  status = [200, 404, 503].sample
   WebMock.stub_request(:get, url)
     .to_return(
-      status: 200,
+      status: status,
       body: File.new(File.join(File.dirname(__FILE__), "fixtures", "htmls", File.basename(url))),
       headers: {content_type: 'application/html; charset=utf-8'})
 end

data/spec/web_stat/fetch_spec.rb

@@ -83,6 +83,7 @@ RSpec.describe WebStat::Fetch do
       expect(web_stat[:title]).to eq "gem作成でついまずいたところ"
       expect(web_stat[:site_name]).to eq "newsdict.blog"
       expect(web_stat[:content]).not_to eq nil
+      expect(web_stat[:status]).to eq("200").or eq("404").or eq("503")
       expect(Sanitize.clean(web_stat[:content]).length).to eq web_stat[:content].length
       expect(web_stat[:eyecatch_image_path]).to be_tmp_file_or_nil
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.2.10
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-08 00:00:00.000000000 Z
+date: 2020-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -224,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Get the status of  the web pages.