web_stat 0.2.6 → 0.2.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f508dae86c4163037aa8ca2dd299be2f98ad01a2b4e61cff8c20b7562cb91e4
-  data.tar.gz: 2a4e70f10396243d5917decf75714302ded5959cebb517fca6d27bc3c45db5eb
+  metadata.gz: bcfaeb202076ea30cae6205877d0b0ad9060eb84116b84ef7bd3580cc4349aac
+  data.tar.gz: 98e586440c8f3aed29e38a003f6d1afc96b26b2ae4c21e43d47e1dfaf6aaa16a
 SHA512:
-  metadata.gz: c6e4b4066d42003bb4e07ac9bfd4184e73dfea3a3d4e3d7a5b239c227f009a631e01cfb7b5d17c0240453d5f943b9e1f49c846d6d2167dd9ad2122372e709fe3
-  data.tar.gz: 597cef76252fdcc77beefcc5d24e68625c93fa5293bed0d7a40f26847b0db6a0b981c83a8e6f3beafee93778c1ff2bf16899a29c6df1866bf6b929b5d29c2975
+  metadata.gz: dfbe6264256f08550ebb42244d92bd81c976dd5dd72a6a4dabffdf6a1366a8f010e8a20527d0bbe7a8334cdb028e86ecae0c63d6cc4368741abe815f1fcb3092
+  data.tar.gz: 9098d904f26dfdfe14c87352cb47ca3f0333f5424ef26d2cd8232c088c2cf8d7dcbeb07012a9294a5c4e4949b51fe71e94d0995465400d87cf7c037fd09ba978

data/Gemfile

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
-gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url.git"
+gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url"
 
 # Specify your gem's dependencies in web_stat.gemspec
 gemspec

data/Gemfile.lock

@@ -1,13 +1,13 @@
 GIT
-  remote: git@github.com:yubele/final_redirect_url.git
-  revision: a8e8fb256f044606313d9726278516a0f24449d4
+  remote: git@github.com:yubele/final_redirect_url
+  revision: 45df878ec9495ebbfa06dc0a60cc5043c2519e16
   specs:
-    final_redirect_url (0.1.0)
+    final_redirect_url (0.1.1)
 
 PATH
   remote: .
   specs:
-    web_stat (0.2.6)
+    web_stat (0.2.11)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
@@ -27,13 +27,13 @@ GEM
     connection_pool (2.2.2)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    crass (1.0.5)
+    crass (1.0.6)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.11.3)
+    ffi (1.12.2)
     guess_html_encoding (0.0.11)
-    hashdiff (1.0.0)
+    hashdiff (1.0.1)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     mechanize (2.7.6)
@@ -45,39 +45,39 @@ GEM
       nokogiri (~> 1.6)
       ntlm-http (~> 0.1, >= 0.1.1)
       webrobots (>= 0.0.9, < 0.2)
-    method_source (0.9.2)
+    method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0425)
     mini_portile2 (2.4.0)
-    natto (1.1.2)
+    natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (3.1.0)
+    net-http-persistent (4.0.0)
       connection_pool (~> 2.2)
-    nokogiri (1.10.7)
+    nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
     nokogumbo (2.0.2)
       nokogiri (~> 1.8, >= 1.8.4)
     ntlm-http (0.1.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    public_suffix (4.0.2)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    public_suffix (4.0.4)
     rake (13.0.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.1)
-      rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.2)
+    rspec-support (3.9.3)
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
@@ -88,8 +88,8 @@ GEM
       nokogumbo (~> 2.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
-    webmock (3.7.6)
+    unf_ext (0.0.7.7)
+    webmock (3.8.3)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
@@ -107,4 +107,4 @@ DEPENDENCIES
   webmock (>= 3.6.0)
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -1,3 +1,10 @@
+# !!!!! Precautions when using with Rails !!!!!
+
+Write this line your Gemfile.
+```
+gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url"
+```
+
 # WebStat
 
 Fetch the web pages and stat.
@@ -18,6 +25,10 @@ Fetch the web pages and stat.
 - "rake", "~> 10.0"
 - "rspec", "~> 3.0"
 
+### Install mecab
+
+    $ sudo apt install mecab-ipadic-utf8 libmecab
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/lib/web_stat.rb

@@ -21,24 +21,26 @@ require "web_stat/fetch/fetch_as_web"
 module WebStat
   class << self
     # Get web page's stat by url
-    def stat_by_web(url)
+    # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
+    def stat_by_web(url, userdics: nil)
       web_stat = WebStat::FetchAsWeb.new(url)
-      web_stat.stat
+      web_stat.stat(userdics: userdics)
     end
 
     # Get web page's stat by url
     # @param String url
-    def stat_by_url(url)
-      stat_by_web(url)
+    def stat_by_url(url, userdics: nil)
+      stat_by_web(url, userdics: userdics)
     end
     
     # Get web page's stat by html
     # @param String html
     # @param [String] url
-    def stat_by_html(html, url=nil)
+    # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
+    def stat_by_html(html, url=nil, userdics: nil)
       web_stat = WebStat::FetchAsHtml.new(html)
       web_stat.url = url unless url.nil?
-      web_stat.stat
+      web_stat.stat(userdics: userdics)
     end
   end
 end

data/lib/web_stat/fetch.rb

@@ -1,6 +1,6 @@
 module WebStat
   class Fetch
-    attr_accessor :url, :html, :nokogiri, :userdic
+    attr_accessor :url, :html, :nokogiri, :userdic, :status
 
     # Get title
     # @return [String] title
@@ -13,7 +13,11 @@ module WebStat
       rescue
         title = @nokogiri.title
       end
-      title.strip
+      if title.nil?
+        "No Title"
+      else
+        title.strip
+      end
     end
     
     # Get name of domain 
@@ -23,7 +27,11 @@ module WebStat
       rescue
         site_name = @nokogiri.title
       end
-      site_name.strip
+      if site_name.nil?
+        "No Sitename"
+      else
+        site_name.strip
+      end
     end
       []
     # Get main section
@@ -41,7 +49,7 @@ module WebStat
           break
         end
       end
-      if path.match(/^\//)
+      if ! path.nil? && path.match(/^\//)
         "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
       else
         path
@@ -53,9 +61,13 @@ module WebStat
     def save_local_path(url)
       return nil if url.nil?
       tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
-      URI.open(original_url(url)) do |remote_file|
-        File.open(tmp_file, "w+b") do |_file|
-          _file.puts(remote_file.read)
+      agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
+      image = agent.get(url)
+      File.open(tmp_file, "w+b") do |_file|
+        if image.class == Mechanize::File
+          _file.puts(image.body)
+        else
+          _file.puts(image.body_io.read)
         end
       end
       tmp_file
@@ -63,23 +75,44 @@ module WebStat
     
     # Get url
     # @param [String] url
+    # @param [String] body
     def get_url(url)
       agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
       # Enable to read Robots.txt
       agent.robots = true
-      document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
-      document.body.encode('UTF-8', document.encoding)
+      begin
+        document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
+        if document.class == Mechanize::File
+          body = document.body
+        else
+          body = document.body.encode('UTF-8', document.encoding)
+        end
+        @status = document.code
+      rescue Mechanize::ResponseCodeError => e
+        body = e.page.body
+        @status = e.page.code
+      end
+      body
     end
     
     # Get the informations of @url
-    def stat
-      clean_content = content.scrub('').gsub(/[\s　]/, "")
-      tag = WebStat::Tag.new(content, userdic: WebStat::Configure.get["userdic"])
+    # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
+    def stat(userdics: nil)
+      clean_content = content.scrub('').gsub(/[\n\t\r　]/, "").gsub(/\s{2,}/, "\s")
+      language_code = CLD.detect_language(clean_content)[:code]
+      if userdics && userdics.has_key?(language_code) && File.exists?(userdics[language_code])
+        tag = WebStat::Tag.new("#{title} #{content}", userdic: userdics[language_code])
+      elsif userdics && userdics.has_key?("other") && File.exists?(userdics["other"])
+        tag = WebStat::Tag.new("#{title} #{content}", userdic: userdics["other"])
+      else
+        tag = WebStat::Tag.new("#{title} #{content}", userdic: WebStat::Configure.get["userdic"])
+      end
       {
         title: title,
         site_name: site_name,
         content: clean_content,
-        language_code: CLD.detect_language(clean_content)[:code],
+        language_code: language_code,
+        status: @status,
         url: @url,
         eyecatch_image_path: save_local_path(eyecatch_image_path),
         tags: tag.nouns
@@ -91,8 +124,9 @@ module WebStat
     # Get original url
     # @param [String] url
     def original_url(url)
-      if url.match(/^http/)
-        FinalRedirectUrl.final_redirect_url(url)
+      last_url = FinalRedirectUrl.final_redirect_url(url)
+      unless last_url.nil? || last_url.scrub('').empty?
+        last_url
       else
         url
       end

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.2.6"
-end
+  VERSION = "0.2.11"
+end

data/spec/spec_helper.rb

@@ -59,9 +59,10 @@ end
 
 # Set webmock
 WebStatTestHelper.scheme_and_files.each do |url|
+  status = [200, 404, 503].sample
   WebMock.stub_request(:get, url)
     .to_return(
-      status: 200,
+      status: status,
       body: File.new(File.join(File.dirname(__FILE__), "fixtures", "htmls", File.basename(url))),
       headers: {content_type: 'application/html; charset=utf-8'})
 end

data/spec/web_stat/fetch_spec.rb

@@ -83,6 +83,7 @@ RSpec.describe WebStat::Fetch do
       expect(web_stat[:title]).to eq "gem作成でついまずいたところ"
       expect(web_stat[:site_name]).to eq "newsdict.blog"
       expect(web_stat[:content]).not_to eq nil
+      expect(web_stat[:status]).to eq("200").or eq("404").or eq("503")
       expect(Sanitize.clean(web_stat[:content]).length).to eq web_stat[:content].length
       expect(web_stat[:eyecatch_image_path]).to be_tmp_file_or_nil
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.11
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-05 00:00:00.000000000 Z
+date: 2020-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -224,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Get the status of  the web pages.