web_stat 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd18b717b3887b0c500f924eee2e913e8390c07076d8f81afa352f5fd9bc0eaf
-  data.tar.gz: e584604f5bd27878c29115b3796022551003436977f924341d8b1366538df717
+  metadata.gz: 84692200a3ace269882025b441ea34bc24ef92e1d61e695db2af2612458d89a6
+  data.tar.gz: 85eb291966b9813c62ab1abdfb0bfc41b823602e86ca3fd3fcd44d8107634aab
 SHA512:
-  metadata.gz: 9050e789b40f41e66c97e29a6a0d16d96fdee7617e03c60fea754ce024dbbf2b45df8c4dc245b9f7ca1639219565db101da066e3aba44c2a0a641026be9229c4
-  data.tar.gz: da47faf8dcff5e99dbb64f2cccaea998e121f7d6693c78f1fa4456e348c40b45347ee540667a91d5aff349a5517ae1b5ce4f60d2b3df65fcc1cb104b3ecf665d
+  metadata.gz: 59df286b73a5d95cacc3b9cbe56fd5cb9c28df8220ceb6fb302a77f8ebe53892d69e4b4b0b4788522db2e19fc1e899ab5f0638e0ce714811c32db12c4adec977
+  data.tar.gz: 945fdd95331fe8def50a2db97e9e45dd659af782b2c169ad6ae45eb9ccd8ea077d0ba32f263e54c49dfebe8c53247bc5c1077d0d63a61661d60ec0870ea56f4b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    web_stat (0.1.6)
+    web_stat (0.1.8)
       bundler (~> 2.0)
       final_redirect_url (~> 0.1.0)
       mechanize (~> 2.7)
@@ -13,15 +13,20 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.6.0)
+      public_suffix (>= 2.0.2, < 4.0)
     coderay (1.1.2)
     connection_pool (2.2.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
     crass (1.0.4)
     diff-lcs (1.3)
-    domain_name (0.5.20180417)
+    domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     ffi (1.11.1)
     final_redirect_url (0.1.0)
     guess_html_encoding (0.0.11)
+    hashdiff (1.0.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     mechanize (2.7.6)
@@ -41,7 +46,7 @@ GEM
     natto (1.1.2)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (3.0.1)
+    net-http-persistent (3.1.0)
       connection_pool (~> 2.2)
     nokogiri (1.10.3)
       mini_portile2 (~> 2.4.0)
@@ -51,6 +56,7 @@ GEM
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
+    public_suffix (3.1.1)
     rake (10.5.0)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
@@ -68,6 +74,7 @@ GEM
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
+    safe_yaml (1.0.5)
     sanitize (5.0.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
@@ -75,6 +82,10 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.6)
+    webmock (3.6.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
     webrobots (0.1.2)
 
 PLATFORMS
@@ -85,6 +96,7 @@ DEPENDENCIES
   rake (~> 10.0)
   rspec (~> 3.0)
   web_stat!
+  webmock (~> 3.6.0)
 
 BUNDLED WITH
    2.0.1

data/README.md

@@ -45,3 +45,7 @@ And then execute:
 ### spec
 
   $ rake spec
+
+Test a file
+
+  $ bundle exec rspec  spec/web_stat/fetch_spec.rb 

data/lib/web_stat/fetch/fetch_as_html.rb

@@ -8,4 +8,4 @@ module WebStat
       @nokogiri = ::Nokogiri::HTML(@html)
     end
   end
-end
+end

data/lib/web_stat/fetch/fetch_as_web.rb

@@ -1,8 +1,6 @@
 require 'mechanize'
 module WebStat
   class FetchAsWeb < Fetch
-    attr_accessor :url
-    
     # initialize class
     # @param [String] url
     def initialize(url)
@@ -11,4 +9,4 @@ module WebStat
       @nokogiri = ::Nokogiri::HTML(@html)
     end
   end
-end
+end

data/lib/web_stat/fetch.rb

@@ -7,8 +7,8 @@ require 'ruby-readability'
 require 'final_redirect_url'
 module WebStat
   class Fetch
-    attr_accessor :html, :nokogiri
-    
+    attr_accessor :url, :html, :nokogiri
+
     # Get title
     # @return [String] title
     def title
@@ -40,6 +40,7 @@ module WebStat
       
     # Get temporary path of image
     def eyecatch_image_path
+      # Reuse `path` in this method
       path = nil
       WebStat::Configure.get["eyecatch_image_xpaths"].each do |xpath|
         if @nokogiri.xpath(xpath).first.respond_to?(:value)
@@ -47,20 +48,15 @@ module WebStat
           break
         end
       end
-      if @url && path.is_a?(String) && !path.match(/^http/) 
-        if path.match(/^\//)
-          path = "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
-        else
-          path = "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}/#{URI.parse(@url).path}/#{path}"
-        end
+      if path.match(/^\//)
+        "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
       end
-      path
     end
     
     # Get local path to save url
     # @param [String] url
     def save_local_path(url)
-      return nil if url.nil? || !url.match(/^http/)
+      return nil if url.nil?
       tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
       open(original_url(url)) do |remote_file|
         File.open(tmp_file, "w+b") do |_file|
@@ -104,4 +100,4 @@ module WebStat
       end
     end
   end
-end
+end

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.1.7"
+  VERSION = "0.1.8"
 end

data/lib/web_stat.rb

@@ -17,13 +17,17 @@ module WebStat
     end
 
     # Get web page's stat by url
+    # @param String url
     def stat_by_url(url)
       stat_by_web(url)
     end
     
     # Get web page's stat by html
-    def stat_by_html(html)
+    # @param String html
+    # @param [String] url
+    def stat_by_html(html, url=nil)
       web_stat = WebStat::FetchAsHtml.new(html)
+      web_stat.url = url unless url.nil?
       web_stat.stat
     end
   end

data/spec/fixtures/htmls/h1-title.html

@@ -7,5 +7,8 @@
 </head>
 <body class="post-template tag-rubygems">
 			<h1 class="post-title">gem作成でついまずいたところ</h1>
+			<p class="profile"><a href="https://newsdict.blog">
+				                                                                <img src="/content/images/size/w100/2019/03/facebook-3.jpg" alt="Yusuke Abe" class="avatar" /></a></p>
+
 </body>
 </html>

data/spec/spec_helper.rb

@@ -3,6 +3,10 @@ require "bundler/setup"
 require 'pry'
 require "web_stat"
 
+require 'webmock'
+include WebMock::API
+WebMock.enable!
+
 RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
   config.example_status_persistence_file_path = ".rspec_status"
@@ -47,8 +51,32 @@ module WebStatTestHelper
     # Get htmls of fixture
     def scheme_and_files
       Dir.glob(File.join(File.dirname(__FILE__), "fixtures", "htmls", "*.html")).map do |file|
-        "file://#{file}"
+	"https://newsdict.blog/#{File.basename(file)}"
       end
     end
   end
-end
+end
+
+# Set webmock
+WebStatTestHelper.scheme_and_files.each do |url|
+  WebMock.stub_request(:get, url)
+    .to_return(
+      status: 200,
+      body: File.new(File.join(File.dirname(__FILE__), "fixtures", "htmls", File.basename(url))),
+      headers: {content_type: 'application/html; charset=utf-8'})
+end
+
+WebMock.stub_request(:get, "https://newsdict.blog/robots.txt")
+    .to_return(
+      status: 200,
+      body: "User-agent: *
+Sitemap: https://newsdict.blog/sitemap.xml
+Disallow: /ghost/
+Disallow: /p/",
+      headers: {content_type: 'application/html; charset=utf-8'})
+
+WebMock.stub_request(:get, "https://newsdict.blog/content/images/size/w100/2019/03/facebook-3.jpg")
+    .to_return(
+      status: 200,
+      body: File.new(File.join(File.dirname(__FILE__), "fixtures", "images", "facebook-3.jpg")),
+      headers: {content_type: 'application/html; charset=utf-8'})

data/spec/web_stat/fetch_spec.rb

@@ -30,9 +30,21 @@ RSpec.describe WebStat::Fetch do
       end
     end
     
+    it "Get eyecatch image blob  by #{fetch[:class].to_s}" do
+      fetch[:fixture].each do |fixture|
+        web_stat = fetch[:class].new(fixture)
+	web_stat.url = "https://newsdict.blog"
+	unless web_stat.stat[:eyecatch_image_path].nil?
+	  image = File.read(web_stat.stat[:eyecatch_image_path])
+	  expect(image.encoding.to_s).to eq("UTF-8")
+	end
+      end
+    end
+    
     it "Get eyecatch image path by #{fetch[:class].to_s}" do
       fetch[:fixture].each do |fixture|
         web_stat = fetch[:class].new(fixture)
+	web_stat.url = "https://newsdict.blog"
         expect(web_stat.eyecatch_image_path).to be_string_or_nil
       end
     end
@@ -40,6 +52,7 @@ RSpec.describe WebStat::Fetch do
     it "Get local path of eyecatch image by #{fetch[:class].to_s}" do
       fetch[:fixture].each do |fixture|
         web_stat = fetch[:class].new(fixture)
+	web_stat.url = "https://newsdict.blog"
         expect(web_stat.stat[:eyecatch_image_path]).to be_tmp_file_or_nil
       end
     end
@@ -47,7 +60,7 @@ RSpec.describe WebStat::Fetch do
     
   it "WebStat.stat_by_html" do
     WebStatTestHelper.htmls.each do |fixture|
-      web_stat = WebStat.stat_by_html(fixture)
+      web_stat = WebStat.stat_by_html(fixture, "https://newsdict.blog")
       expect(web_stat[:title]).to eq "gem作成でついまずいたところ"
       expect(web_stat[:site_name]).to eq "newsdict.blog"
       expect(web_stat[:content]).not_to eq nil
@@ -66,4 +79,4 @@ RSpec.describe WebStat::Fetch do
       expect(web_stat[:eyecatch_image_path]).to be_tmp_file_or_nil
     end
   end
-end
+end

data/web_stat.gemspec

@@ -31,4 +31,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry", "~> 0.12.2"
+  spec.add_development_dependency "webmock", "~> 3.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-30 00:00:00.000000000 Z
+date: 2019-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -150,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.12.2
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
 description: Fetch the web pages and stat.
 email:
 - yube@newsdict.jp
@@ -180,6 +194,7 @@ files:
 - spec/fixtures/htmls/blog.html
 - spec/fixtures/htmls/h1-title.html
 - spec/fixtures/htmls/image.html
+- spec/fixtures/images/facebook-3.jpg
 - spec/spec_helper.rb
 - spec/web_stat/configure_spec.rb
 - spec/web_stat/fetch_spec.rb
@@ -213,6 +228,7 @@ test_files:
 - spec/fixtures/htmls/blog.html
 - spec/fixtures/htmls/h1-title.html
 - spec/fixtures/htmls/image.html
+- spec/fixtures/images/facebook-3.jpg
 - spec/spec_helper.rb
 - spec/web_stat/configure_spec.rb
 - spec/web_stat/fetch_spec.rb