link_thumbnailer 3.3.0 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 62e43acf3460b4ef3da648fa16fb8f3f970d88b8
-  data.tar.gz: 4768729820579a8387d692ddba91a3557f548d60
+  metadata.gz: fef68fdf205078290139fd831a9b8d46581e64eb
+  data.tar.gz: 27a3e86b0181068e06bd80e8f053fca17d3a906b
 SHA512:
-  metadata.gz: 1e4a5a30c0166b19f57a0e2dbc57fda86074ad26caa28fde4ad8c15e1afe278478c4ca27223a155b66ce777765a729e78f5a95ddd080abb535fa5b806eaeafbb
-  data.tar.gz: 93464538468ced5cb85119c2e85219f592c33a91733bf1e89741d074253e9a540f949355258a723db38698e712c7c2d079ec6d75539fb249f48cb25109ab772b
+  metadata.gz: 5dcfa7683c65f546ce8e794d51c9d7103736fb3cf9f5919ddbcea611fbe7243faf3ef76ed372d99ea316a6d193032d9b47e48411dc9c0fb1c972d77c29bc2a2a
+  data.tar.gz: e041d9174be5b066fac70b70b965bc28b053235bffa808c0a109c3c1f10512643ae58dc21b3f8691ca3ff6a3b9f5346e31c4b0cf1430d16ac829244d3252ab96

data/.ruby-version

@@ -0,0 +1 @@
+2.4.0

data/.travis.yml

@@ -1,4 +1,5 @@
 bundler_args: --without development
 language: ruby
 rvm:
-  - 2.2.0
+  - 2.2.2
+  - 2.4.0

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 3.3.1
+
+Gem upgrade (json)
+
 # 3.3.0
 
 - Allows to configure overrided http headers

data/Gemfile

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :test do
-  gem 'rspec',      '~> 2.14'
-  gem 'webmock',    '~> 1.14'
-  gem 'pry',        '~> 0.9'
+  gem 'rspec',      '~> 3.5.0'
+  gem 'webmock',    '~> 2.3.2'
+  gem 'pry',        '~> 0.10.4'
 end

data/README.md

@@ -165,6 +165,10 @@ LinkThumbnailer.configure do |config|
   # Sets number of concurrent http connections that can be opened to fetch images informations such as size and type.
   #
   # config.max_concurrency = 20
+
+  # Sets the default encoding.
+  #
+  # config.encoding = 'utf-8'
 end
 ```
 

data/lib/generators/templates/initializer.rb

@@ -80,4 +80,8 @@ LinkThumbnailer.configure do |config|
   # Defines the strategies to use to scrap the website. See the [Open Graph Protocol](http://ogp.me/) for more information.
   #
   # config.scrapers = [:opengraph, :default]
+
+  # Sets the default encoding.
+  #
+  # config.encoding = 'utf-8'
 end

data/lib/link_thumbnailer/configuration.rb

@@ -26,7 +26,7 @@ module LinkThumbnailer
                   :verify_ssl, :http_open_timeout, :http_read_timeout, :attributes,
                   :graders, :description_min_length, :positive_regex, :negative_regex,
                   :image_limit, :image_stats, :raise_on_invalid_format, :max_concurrency,
-                  :scrapers, :http_override_headers
+                  :scrapers, :http_override_headers, :encoding
 
     alias_method :http_timeout, :http_open_timeout
     alias_method :http_timeout=, :http_open_timeout=
@@ -63,6 +63,7 @@ module LinkThumbnailer
       @max_concurrency = 20
       @scrapers = [:opengraph, :default]
       @http_override_headers = { 'Accept-Encoding' => 'none' }
+      @encoding = 'utf-8'
     end
 
 	end

data/lib/link_thumbnailer/page.rb

@@ -1,3 +1,4 @@
+require 'link_thumbnailer/response'
 require 'link_thumbnailer/processor'
 require 'link_thumbnailer/scraper'
 

data/lib/link_thumbnailer/parser.rb

@@ -4,7 +4,7 @@ module LinkThumbnailer
   class Parser
 
     def call(source)
-      ::Nokogiri::HTML(source)
+      ::Nokogiri::HTML(source, nil, LinkThumbnailer.page.config.encoding)
     rescue ::Nokogiri::XML::SyntaxError => e
       raise ::LinkThumbnailer::SyntaxError.new(e.message)
     end

data/lib/link_thumbnailer/processor.rb

@@ -59,7 +59,7 @@ module LinkThumbnailer
 
       case response
       when ::Net::HTTPSuccess
-        response.body
+        Response.new(response).body
       when ::Net::HTTPRedirection
         call(
           resolve_relative_url(response['location'].to_s),

data/lib/link_thumbnailer/response.rb

@@ -0,0 +1,37 @@
+module LinkThumbnailer
+  class Response
+    def initialize(response)
+      @response = response
+    end
+
+    def charset
+      @charset ||= extract_charset
+    end
+
+    def body
+      @body ||= extract_body
+    end
+
+    private
+
+    def extract_charset
+      content_type = @response['Content-Type'] || ''
+      m = content_type.match(/charset=(\w+)/)
+      (m && m[1]) || ''
+    end
+
+    def extract_body
+      should_convert_body_to_utf8? ? convert_encoding_to_utf8(@response.body, charset) : @response.body
+    end
+
+    def should_convert_body_to_utf8?
+      charset != '' && charset != 'utf-8'
+    end
+
+    def convert_encoding_to_utf8(body, from)
+      Encoding::Converter.new(from, 'utf-8').convert(body)
+    rescue EncodingError
+      body
+    end
+  end
+end

data/lib/link_thumbnailer/version.rb

@@ -1,3 +1,3 @@
 module LinkThumbnailer
-  VERSION = '3.3.0'
+  VERSION = '3.3.1'
 end

data/link_thumbnailer.gemspec

@@ -18,10 +18,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activesupport',          '>= 3.0'
-  spec.add_dependency 'json',                   ['>= 1.7.7', '~> 1.8']
-  spec.add_dependency 'rake',                   '>= 0.9'
-  spec.add_dependency 'nokogiri',               '~> 1.6'
-  spec.add_dependency 'net-http-persistent',    '~> 2.9'
-  spec.add_dependency 'video_info',             '~> 2.6'
-  spec.add_dependency 'image_info',             '~> 1.0'
+  spec.add_dependency 'json',                   '~> 2.1'
+  spec.add_dependency 'rake',                   ['>= 0.9']
+  spec.add_dependency 'nokogiri',               '~> 1.7'
+  spec.add_dependency 'net-http-persistent',    '~> 3.0'
+  spec.add_dependency 'video_info',             '~> 2.7'
+  spec.add_dependency 'image_info',             '~> 1.1'
 end

data/spec/configuration_spec.rb

@@ -18,6 +18,7 @@ describe LinkThumbnailer::Configuration do
   it { expect(instance.image_limit).to            eq(5) }
   it { expect(instance.image_stats).to            eq(true) }
   it { expect(instance.max_concurrency).to        eq(20) }
+  it { expect(instance.encoding).to               eq('utf-8') }
 
   describe "#http_timeout" do
     it { expect(instance.method(:http_timeout)).to eq(instance.method(:http_open_timeout)) }
@@ -33,7 +34,7 @@ describe LinkThumbnailer::Configuration do
   describe '.configure' do
 
     before do
-      LinkThumbnailer.stub(:config).and_return(instance)
+      allow(LinkThumbnailer).to receive(:config).and_return(instance)
     end
 
     context 'when block given' do
@@ -56,4 +57,3 @@ describe LinkThumbnailer::Configuration do
   end
 
 end
-

data/spec/fixtures/google_shift_jis.html

@@ -0,0 +1,6 @@
+<!doctype html><html itemscope="" itemtype="http://schema.org/WebPage" lang="ja"><head><meta content="���E���̂���������������邽�߂̃c�[����񋟂��Ă��܂��B���܂��܂Ȍ����@�\�����p���āA���T���̏������‚��Ă��������B" name="description"><meta content="noodp" name="robots"><meta content="text/html; charset=UTF-8" http-equiv="Content-Type"><meta content="/images/branding/googleg/1x/googleg_standard_color_128dp.png" itemprop="image"><title>Google</title><script>(function(){window.google={kEI:'716VWJL3JYq18QWo_qzoAg',kEXPI:'750722,1351903,1352241,1352381,3700253,3700347,4028875,4029815,4032677,4038012,4043492,4045839,4048347,4062666,4065787,4068816,4069773,4069838,4069840,4070138,4072773,4073405,4073726,4073959,4076096,4076931,4076999,4077777,4078438,4079081,4079105,4079894,4081038,4081485,4082194,4082201,4082298,4082619,4083044,4083476,4084343,4084673,4085336,4085412,4085683,4086011,4089003,4089144,4089183,4089427,4089538,4089913,4090414,4090547,4090549,4090598,4090657,4090806,4090893,4091966,4092028,4092182,4092218,4092474,4092478,4092598,4092864,4092867,4092875,4092897,4092934,4093073,4093948,4094169,4094250,4094769,4094987,4094997,4095554,4095771,4095907,4095998,8300096,8300272,8507380,8507419,8507861,8507940,8508624,8510023,10200083,13500022,13500024',authuser:0,kscs:'c9c918f0_24'};google.kHL='ja';})();(function(){google.lc=[];google.li=0;google.getEI=function(a){for(var b;a&&(!a.getAttribute||!(b=a.getAttribute("eid")));)a=a.parentNode;return b||google.kEI};google.getLEI=function(a){for(var b=null;a&&(!a.getAttribute||!(b=a.getAttribute("leid")));)a=a.parentNode;return b};google.https=function(){return"https:"==window.location.protocol};google.ml=function(){return null};google.wl=function(a,b){try{google.ml(Error(a),!1,b)}catch(c){}};google.time=function(){return(new Date).getTime()};google.log=function(a,b,c,d,g){a=google.logUrl(a,b,c,d,g);if(""!=a){b=new Image;var e=google.lc,f=google.li;e[f]=b;b.onerror=b.onload=b.onabort=function(){delete e[f]};window.google&&window.google.vel&&window.google.vel.lu&&window.google.vel.lu(a);b.src=a;google.li=f+1}};google.logUrl=function(a,b,c,d,g){var e="",f=google.ls||"";c||-1!=b.search("&ei=")||(e="&ei="+google.getEI(d),-1==b.search("&lei=")&&(d=google.getLEI(d))&&(e+="&lei="+d));a=c||"/"+(g||"gen_204")+"?atyp=i&ct="+a+"&cad="+b+e+f+"&zx="+google.time();/^http:/i.test(a)&&google.https()&&(google.ml(Error("a"),!1,{src:a,glmm:1}),a="");return a};google.y={};google.x=function(a,b){google.y[a.id]=[a,b];return!1};google.lq=[];google.load=function(a,b,c){google.lq.push([[a],b,c])};google.loadAll=function(a,b){google.lq.push([a,b])};}).call(this);var a=window.location,b=a.href.indexOf("#");if(0<=b){var c=a.href.substring(b+1);/(^|&)q=/.test(c)&&-1==c.indexOf("#")&&a.replace("/search?"+c.replace(/(^|&)fp=[^&]*/g,"")+"&cad=h")};</script><style>#gbar,#guser{font-size:13px;padding-top:1px !important;}#gbar{height:22px}#guser{padding-bottom:7px !important;text-align:right}.gbh,.gbd{border-top:1px solid #c9d7f1;font-size:1px}.gbh{height:0;position:absolute;top:24px;width:100%}@media all{.gb1{height:22px;margin-right:.5em;vertical-align:top}#gbar{float:left}}a.gb1,a.gb4{text-decoration:underline !important}a.gb1,a.gb4{color:#00c !important}.gbi .gb4{color:#dd8e27 !important}.gbf .gb4{color:#900 !important}
+</style><style>body,td,a,p,.h{font-family:arial,sans-serif}body{margin:0;overflow-y:scroll}#gog{padding:3px 8px 0}td{line-height:.8em}.gac_m td{line-height:17px}form{margin-bottom:20px}.h{color:#36c}.q{color:#00c}.ts td{padding:0}.ts{border-collapse:collapse}em{font-weight:bold;font-style:normal}.lst{height:25px;width:496px}.gsfi,.lst{font:18px arial,sans-serif}.gsfs{font:17px arial,sans-serif}.ds{display:inline-box;display:inline-block;margin:3px 0 4px;margin-left:4px}input{font-family:inherit}a.gb1,a.gb2,a.gb3,a.gb4{color:#11c !important}body{background:#fff;color:black}a{color:#11c;text-decoration:none}a:hover,a:active{text-decoration:underline}.fl a{color:#36c}a:visited{color:#551a8b}a.gb1,a.gb4{text-decoration:underline}a.gb3:hover{text-decoration:none}#ghead a.gb2:hover{color:#fff !important}.sblc{padding-top:5px}.sblc a{display:block;margin:2px 0;margin-left:13px;font-size:11px}.lsbb{background:#eee;border:solid 1px;border-color:#ccc #999 #999 #ccc;height:30px}.lsbb{display:block}.ftl,#fll a{display:inline-block;margin:0 12px}.lsb{background:url(/images/nav_logo229.png) 0 -261px repeat-x;border:none;color:#000;cursor:pointer;height:30px;margin:0;outline:0;font:15px arial,sans-serif;vertical-align:top}.lsb:active{background:#ccc}.lst:focus{outline:none}</style><script></script><link href="/images/branding/product/ico/googleg_lodp.ico" rel="shortcut icon"></head><body bgcolor="#fff"><script>(function(){var src='/images/nav_logo229.png';var iesg=false;document.body.onload = function(){window.n && window.n();if (document.images){new Image().src=src;}
+if (!iesg){document.f&&document.f.q.focus();document.gbqf&&document.gbqf.q.focus();}
+}
+})();</script><div id="mngb"> <div id=gbar><nobr><b class=gb1>����</b> <a class=gb1 href="https://www.google.co.jp/imghp?hl=ja&tab=wi">�摜</a> <a class=gb1 href="https://maps.google.co.jp/maps?hl=ja&tab=wl">�}�b�v</a> <a class=gb1 href="https://play.google.com/?hl=ja&tab=w8">Play</a> <a class=gb1 href="https://www.youtube.com/?gl=JP&tab=w1">YouTube</a> <a class=gb1 href="https://news.google.co.jp/nwshp?hl=ja&tab=wn">�j���[�X</a> <a class=gb1 href="https://mail.google.com/mail/?tab=wm">Gmail</a> <a class=gb1 href="https://drive.google.com/?tab=wo">�h���C�u</a> <a class=gb1 style="text-decoration:none" href="https://www.google.co.jp/intl/ja/options/"><u>�����ƌ���</u> &raquo;</a></nobr></div><div id=guser width=100%><nobr><span id=gbn class=gbi></span><span id=gbf class=gbf></span><span id=gbe></span><a href="http://www.google.co.jp/history/optout?hl=ja" class=gb4>�E�F�u����</a> | <a  href="/preferences?hl=ja" class=gb4>�ݒ�</a> | <a target=_top id=gb_70 href="https://accounts.google.com/ServiceLogin?hl=ja&passive=true&continue=https://www.google.co.jp/" class=gb4>���O�C��</a></nobr></div><div class=gbh style=left:0></div><div class=gbh style=right:0></div> </div><center><br clear="all" id="lgpd"><div id="lga"><div style="padding:28px 0 3px"><div style="height:110px;width:276px;background:url(/images/branding/googlelogo/1x/googlelogo_white_background_color_272x92dp.png) no-repeat" title="Google" align="left" id="hplogo" onload="window.lol&&lol()"><div style="color:#777;font-size:16px;font-weight:bold;position:relative;top:70px;left:218px" nowrap="">���{</div></div></div><br></div><form action="/search" name="f"><table cellpadding="0" cellspacing="0"><tr valign="top"><td width="25%">&nbsp;</td><td align="center" nowrap=""><input name="ie" value="Shift_JIS" type="hidden"><input value="ja" name="hl" type="hidden"><input name="source" type="hidden" value="hp"><input name="biw" type="hidden"><input name="bih" type="hidden"><div class="ds" style="height:32px;margin:4px 0"><input style="color:#000;margin:0;padding:5px 8px 0 6px;vertical-align:top" autocomplete="off" class="lst" value="" title="Google ����" maxlength="2048" name="q" size="57"></div><br style="line-height:0"><span class="ds"><span class="lsbb"><input class="lsb" value="Google ����" name="btnG" type="submit"></span></span><span class="ds"><span class="lsbb"><input class="lsb" value="I'm Feeling Lucky" name="btnI" onclick="if(this.form.q.value)this.checked=1; else top.location='/doodles/'" type="submit"></span></span></td><td class="fl sblc" align="left" nowrap="" width="25%"><a href="/advanced_search?hl=ja&amp;authuser=0">�����I�v�V����</a><a href="/language_tools?hl=ja&amp;authuser=0">����c�[��</a></td></tr></table><input id="gbv" name="gbv" type="hidden" value="1"></form><div id="gac_scont"></div><div style="font-size:83%;min-height:3.5em"><br></div><span id="footer"><div style="font-size:10pt"><div style="margin:19px auto;text-align:center" id="fll"><a href="/intl/ja/ads/">�L���f��</a><a href="http://www.google.co.jp/intl/ja/services/">�r�W�l�X �\�����[�V����</a><a href="https://plus.google.com/115899767381375908215" rel="publisher">+Google</a><a href="/intl/ja/about.html">Google �ɂ‚���</a><a href="https://www.google.co.jp/setprefdomain?prefdom=US&amp;sig=__NSwOcr0nRFmauaxJWrHJmztdkTc%3D" id="fehl">Google.com</a></div></div><p style="color:#767676;font-size:8pt">&copy; 2017 - <a href="/intl/ja/policies/privacy/">�v���C�o�V�[</a> - <a href="/intl/ja/policies/terms/">�K��</a></p></span></center><script>(function(){window.google.cdo={height:0,width:0};(function(){var a=window.innerWidth,b=window.innerHeight;if(!a||!b)var c=window.document,d="CSS1Compat"==c.compatMode?c.documentElement:c.body,a=d.clientWidth,b=d.clientHeight;a&&b&&(a!=google.cdo.width||b!=google.cdo.height)&&google.log("","","/client_204?&atyp=i&biw="+a+"&bih="+b+"&ei="+google.kEI);}).call(this);})();</script><div id="xjsd"></div><div id="xjsi"><script>(function(){function c(b){window.setTimeout(function(){var a=document.createElement("script");a.src=b;document.getElementById("xjsd").appendChild(a)},0)}google.dljp=function(b,a){google.xjsu=b;c(a)};google.dlj=c;}).call(this);(function(){window.google.xjsrm=[];})();if(google.y)google.y.first=[];if(!google.xjs){window._=window._||{};window._._DumpException=function(e){throw e};if(google.timers&&google.timers.load.t){google.timers.load.t.xjsls=new Date().getTime();}google.dljp('/xjs/_/js/k\x3dxjs.hp.en_US.usfcb5N0rIw.O/m\x3dsb_he,d/rt\x3dj/d\x3d1/t\x3dzcms/rs\x3dACT90oH_6o1ZdJ1OdBerX9LppN-xEd11Eg','/xjs/_/js/k\x3dxjs.hp.en_US.usfcb5N0rIw.O/m\x3dsb_he,d/rt\x3dj/d\x3d1/t\x3dzcms/rs\x3dACT90oH_6o1ZdJ1OdBerX9LppN-xEd11Eg');google.xjs=1;}google.pmc={"sb_he":{"agen":true,"cgen":true,"client":"heirloom-hp","dh":true,"dhqt":true,"ds":"","fl":true,"host":"google.co.jp","isbh":28,"jam":0,"jsonp":true,"msgs":{"cibl":"�������N���A","dym":"����������:","lcky":"I\u0026#39;m Feeling Lucky","lml":"�ڍ�","oskt":"���̓c�[��","psrc":"���̌����L�[���[�h��\u003Ca href=\"/history\"\u003E�E�F�u����\u003C/a\u003E����폜����܂���","psrl":"�폜","sbit":"�摜�Ō���","srch":"Google ����"},"nds":true,"ovr":{},"pq":"","refpd":true,"refspre":true,"rfs":[],"scd":10,"sce":5,"stok":"WXDdTX_0YGr0J9JRNcHRvk23BlI"},"d":{}};google.y.first.push(function(){if(google.med){google.med('init');google.initHistory();google.med('history');}});if(google.j&&google.j.en&&google.j.xi){window.setTimeout(google.j.xi,0);}
+</script></div></body></html>

data/spec/fixtures/google_utf8.html

@@ -0,0 +1,6 @@
+<!doctype html><html itemscope="" itemtype="http://schema.org/WebPage" lang="ja"><head><meta content="世界中のあらゆる情報を検索するためのツールを提供しています。さまざまな検索機能を活用して、お探しの情報を見つけてください。" name="description"><meta content="noodp" name="robots"><meta content="text/html; charset=UTF-8" http-equiv="Content-Type"><meta content="/images/branding/googleg/1x/googleg_standard_color_128dp.png" itemprop="image"><title>Google</title><script>(function(){window.google={kEI:'716VWJL3JYq18QWo_qzoAg',kEXPI:'750722,1351903,1352241,1352381,3700253,3700347,4028875,4029815,4032677,4038012,4043492,4045839,4048347,4062666,4065787,4068816,4069773,4069838,4069840,4070138,4072773,4073405,4073726,4073959,4076096,4076931,4076999,4077777,4078438,4079081,4079105,4079894,4081038,4081485,4082194,4082201,4082298,4082619,4083044,4083476,4084343,4084673,4085336,4085412,4085683,4086011,4089003,4089144,4089183,4089427,4089538,4089913,4090414,4090547,4090549,4090598,4090657,4090806,4090893,4091966,4092028,4092182,4092218,4092474,4092478,4092598,4092864,4092867,4092875,4092897,4092934,4093073,4093948,4094169,4094250,4094769,4094987,4094997,4095554,4095771,4095907,4095998,8300096,8300272,8507380,8507419,8507861,8507940,8508624,8510023,10200083,13500022,13500024',authuser:0,kscs:'c9c918f0_24'};google.kHL='ja';})();(function(){google.lc=[];google.li=0;google.getEI=function(a){for(var b;a&&(!a.getAttribute||!(b=a.getAttribute("eid")));)a=a.parentNode;return b||google.kEI};google.getLEI=function(a){for(var b=null;a&&(!a.getAttribute||!(b=a.getAttribute("leid")));)a=a.parentNode;return b};google.https=function(){return"https:"==window.location.protocol};google.ml=function(){return null};google.wl=function(a,b){try{google.ml(Error(a),!1,b)}catch(c){}};google.time=function(){return(new Date).getTime()};google.log=function(a,b,c,d,g){a=google.logUrl(a,b,c,d,g);if(""!=a){b=new Image;var e=google.lc,f=google.li;e[f]=b;b.onerror=b.onload=b.onabort=function(){delete e[f]};window.google&&window.google.vel&&window.google.vel.lu&&window.google.vel.lu(a);b.src=a;google.li=f+1}};google.logUrl=function(a,b,c,d,g){var e="",f=google.ls||"";c||-1!=b.search("&ei=")||(e="&ei="+google.getEI(d),-1==b.search("&lei=")&&(d=google.getLEI(d))&&(e+="&lei="+d));a=c||"/"+(g||"gen_204")+"?atyp=i&ct="+a+"&cad="+b+e+f+"&zx="+google.time();/^http:/i.test(a)&&google.https()&&(google.ml(Error("a"),!1,{src:a,glmm:1}),a="");return a};google.y={};google.x=function(a,b){google.y[a.id]=[a,b];return!1};google.lq=[];google.load=function(a,b,c){google.lq.push([[a],b,c])};google.loadAll=function(a,b){google.lq.push([a,b])};}).call(this);var a=window.location,b=a.href.indexOf("#");if(0<=b){var c=a.href.substring(b+1);/(^|&)q=/.test(c)&&-1==c.indexOf("#")&&a.replace("/search?"+c.replace(/(^|&)fp=[^&]*/g,"")+"&cad=h")};</script><style>#gbar,#guser{font-size:13px;padding-top:1px !important;}#gbar{height:22px}#guser{padding-bottom:7px !important;text-align:right}.gbh,.gbd{border-top:1px solid #c9d7f1;font-size:1px}.gbh{height:0;position:absolute;top:24px;width:100%}@media all{.gb1{height:22px;margin-right:.5em;vertical-align:top}#gbar{float:left}}a.gb1,a.gb4{text-decoration:underline !important}a.gb1,a.gb4{color:#00c !important}.gbi .gb4{color:#dd8e27 !important}.gbf .gb4{color:#900 !important}
+</style><style>body,td,a,p,.h{font-family:arial,sans-serif}body{margin:0;overflow-y:scroll}#gog{padding:3px 8px 0}td{line-height:.8em}.gac_m td{line-height:17px}form{margin-bottom:20px}.h{color:#36c}.q{color:#00c}.ts td{padding:0}.ts{border-collapse:collapse}em{font-weight:bold;font-style:normal}.lst{height:25px;width:496px}.gsfi,.lst{font:18px arial,sans-serif}.gsfs{font:17px arial,sans-serif}.ds{display:inline-box;display:inline-block;margin:3px 0 4px;margin-left:4px}input{font-family:inherit}a.gb1,a.gb2,a.gb3,a.gb4{color:#11c !important}body{background:#fff;color:black}a{color:#11c;text-decoration:none}a:hover,a:active{text-decoration:underline}.fl a{color:#36c}a:visited{color:#551a8b}a.gb1,a.gb4{text-decoration:underline}a.gb3:hover{text-decoration:none}#ghead a.gb2:hover{color:#fff !important}.sblc{padding-top:5px}.sblc a{display:block;margin:2px 0;margin-left:13px;font-size:11px}.lsbb{background:#eee;border:solid 1px;border-color:#ccc #999 #999 #ccc;height:30px}.lsbb{display:block}.ftl,#fll a{display:inline-block;margin:0 12px}.lsb{background:url(/images/nav_logo229.png) 0 -261px repeat-x;border:none;color:#000;cursor:pointer;height:30px;margin:0;outline:0;font:15px arial,sans-serif;vertical-align:top}.lsb:active{background:#ccc}.lst:focus{outline:none}</style><script></script><link href="/images/branding/product/ico/googleg_lodp.ico" rel="shortcut icon"></head><body bgcolor="#fff"><script>(function(){var src='/images/nav_logo229.png';var iesg=false;document.body.onload = function(){window.n && window.n();if (document.images){new Image().src=src;}
+if (!iesg){document.f&&document.f.q.focus();document.gbqf&&document.gbqf.q.focus();}
+}
+})();</script><div id="mngb"> <div id=gbar><nobr><b class=gb1>検索</b> <a class=gb1 href="https://www.google.co.jp/imghp?hl=ja&tab=wi">画像</a> <a class=gb1 href="https://maps.google.co.jp/maps?hl=ja&tab=wl">マップ</a> <a class=gb1 href="https://play.google.com/?hl=ja&tab=w8">Play</a> <a class=gb1 href="https://www.youtube.com/?gl=JP&tab=w1">YouTube</a> <a class=gb1 href="https://news.google.co.jp/nwshp?hl=ja&tab=wn">ニュース</a> <a class=gb1 href="https://mail.google.com/mail/?tab=wm">Gmail</a> <a class=gb1 href="https://drive.google.com/?tab=wo">ドライブ</a> <a class=gb1 style="text-decoration:none" href="https://www.google.co.jp/intl/ja/options/"><u>もっと見る</u> &raquo;</a></nobr></div><div id=guser width=100%><nobr><span id=gbn class=gbi></span><span id=gbf class=gbf></span><span id=gbe></span><a href="http://www.google.co.jp/history/optout?hl=ja" class=gb4>ウェブ履歴</a> | <a  href="/preferences?hl=ja" class=gb4>設定</a> | <a target=_top id=gb_70 href="https://accounts.google.com/ServiceLogin?hl=ja&passive=true&continue=https://www.google.co.jp/" class=gb4>ログイン</a></nobr></div><div class=gbh style=left:0></div><div class=gbh style=right:0></div> </div><center><br clear="all" id="lgpd"><div id="lga"><div style="padding:28px 0 3px"><div style="height:110px;width:276px;background:url(/images/branding/googlelogo/1x/googlelogo_white_background_color_272x92dp.png) no-repeat" title="Google" align="left" id="hplogo" onload="window.lol&&lol()"><div style="color:#777;font-size:16px;font-weight:bold;position:relative;top:70px;left:218px" nowrap="">日本</div></div></div><br></div><form action="/search" name="f"><table cellpadding="0" cellspacing="0"><tr valign="top"><td width="25%">&nbsp;</td><td align="center" nowrap=""><input name="ie" value="Shift_JIS" type="hidden"><input value="ja" name="hl" type="hidden"><input name="source" type="hidden" value="hp"><input name="biw" type="hidden"><input name="bih" type="hidden"><div class="ds" style="height:32px;margin:4px 0"><input style="color:#000;margin:0;padding:5px 8px 0 6px;vertical-align:top" autocomplete="off" class="lst" value="" title="Google 検索" maxlength="2048" name="q" size="57"></div><br style="line-height:0"><span class="ds"><span class="lsbb"><input class="lsb" value="Google 検索" name="btnG" type="submit"></span></span><span class="ds"><span class="lsbb"><input class="lsb" value="I'm Feeling Lucky" name="btnI" onclick="if(this.form.q.value)this.checked=1; else top.location='/doodles/'" type="submit"></span></span></td><td class="fl sblc" align="left" nowrap="" width="25%"><a href="/advanced_search?hl=ja&amp;authuser=0">検索オプション</a><a href="/language_tools?hl=ja&amp;authuser=0">言語ツール</a></td></tr></table><input id="gbv" name="gbv" type="hidden" value="1"></form><div id="gac_scont"></div><div style="font-size:83%;min-height:3.5em"><br></div><span id="footer"><div style="font-size:10pt"><div style="margin:19px auto;text-align:center" id="fll"><a href="/intl/ja/ads/">広告掲載</a><a href="http://www.google.co.jp/intl/ja/services/">ビジネス ソリューション</a><a href="https://plus.google.com/115899767381375908215" rel="publisher">+Google</a><a href="/intl/ja/about.html">Google について</a><a href="https://www.google.co.jp/setprefdomain?prefdom=US&amp;sig=__NSwOcr0nRFmauaxJWrHJmztdkTc%3D" id="fehl">Google.com</a></div></div><p style="color:#767676;font-size:8pt">&copy; 2017 - <a href="/intl/ja/policies/privacy/">プライバシー</a> - <a href="/intl/ja/policies/terms/">規約</a></p></span></center><script>(function(){window.google.cdo={height:0,width:0};(function(){var a=window.innerWidth,b=window.innerHeight;if(!a||!b)var c=window.document,d="CSS1Compat"==c.compatMode?c.documentElement:c.body,a=d.clientWidth,b=d.clientHeight;a&&b&&(a!=google.cdo.width||b!=google.cdo.height)&&google.log("","","/client_204?&atyp=i&biw="+a+"&bih="+b+"&ei="+google.kEI);}).call(this);})();</script><div id="xjsd"></div><div id="xjsi"><script>(function(){function c(b){window.setTimeout(function(){var a=document.createElement("script");a.src=b;document.getElementById("xjsd").appendChild(a)},0)}google.dljp=function(b,a){google.xjsu=b;c(a)};google.dlj=c;}).call(this);(function(){window.google.xjsrm=[];})();if(google.y)google.y.first=[];if(!google.xjs){window._=window._||{};window._._DumpException=function(e){throw e};if(google.timers&&google.timers.load.t){google.timers.load.t.xjsls=new Date().getTime();}google.dljp('/xjs/_/js/k\x3dxjs.hp.en_US.usfcb5N0rIw.O/m\x3dsb_he,d/rt\x3dj/d\x3d1/t\x3dzcms/rs\x3dACT90oH_6o1ZdJ1OdBerX9LppN-xEd11Eg','/xjs/_/js/k\x3dxjs.hp.en_US.usfcb5N0rIw.O/m\x3dsb_he,d/rt\x3dj/d\x3d1/t\x3dzcms/rs\x3dACT90oH_6o1ZdJ1OdBerX9LppN-xEd11Eg');google.xjs=1;}google.pmc={"sb_he":{"agen":true,"cgen":true,"client":"heirloom-hp","dh":true,"dhqt":true,"ds":"","fl":true,"host":"google.co.jp","isbh":28,"jam":0,"jsonp":true,"msgs":{"cibl":"検索をクリア","dym":"もしかして:","lcky":"I\u0026#39;m Feeling Lucky","lml":"詳細","oskt":"入力ツール","psrc":"この検索キーワードは\u003Ca href=\"/history\"\u003Eウェブ履歴\u003C/a\u003Eから削除されました","psrl":"削除","sbit":"画像で検索","srch":"Google 検索"},"nds":true,"ovr":{},"pq":"","refpd":true,"refspre":true,"rfs":[],"scd":10,"sce":5,"stok":"WXDdTX_0YGr0J9JRNcHRvk23BlI"},"d":{}};google.y.first.push(function(){if(google.med){google.med('init');google.initHistory();google.med('history');}});if(google.j&&google.j.en&&google.j.xi){window.setTimeout(google.j.xi,0);}
+</script></div></body></html>

data/spec/grader_spec.rb

@@ -15,7 +15,7 @@ describe LinkThumbnailer::Grader do
     let(:action)  { instance.call }
 
     before do
-      instance.stub(:graders).and_return(graders)
+      allow(instance).to receive(:graders).and_return(graders)
     end
 
     it { expect(action).to eq((0.5 * 0.5) ** weight) }

data/spec/graders/html_attribute_spec.rb

@@ -12,7 +12,7 @@ describe LinkThumbnailer::Graders::HtmlAttribute do
     context 'when current node does not match attribute' do
 
       before do
-        instance.stub(:attribute?).and_return(false)
+        allow(instance).to receive(:attribute?).and_return(false)
       end
 
       it { expect(action).to eq(1.0) }
@@ -22,9 +22,9 @@ describe LinkThumbnailer::Graders::HtmlAttribute do
     context 'when current node has a negative attribute' do
 
       before do
-        instance.stub(:attribute?).and_return(true)
-        instance.stub(:negative?).and_return(true)
-        instance.stub(:positive?).and_return(false)
+        allow(instance).to receive(:attribute?).and_return(true)
+        allow(instance).to receive(:negative?).and_return(true)
+        allow(instance).to receive(:positive?).and_return(false)
       end
 
       it { expect(action).to eq(0.0) }
@@ -34,9 +34,9 @@ describe LinkThumbnailer::Graders::HtmlAttribute do
     context 'when current node has a positive attribute' do
 
       before do
-        instance.stub(:attribute?).and_return(true)
-        instance.stub(:negative?).and_return(false)
-        instance.stub(:positive?).and_return(true)
+        allow(instance).to receive(:attribute?).and_return(true)
+        allow(instance).to receive(:negative?).and_return(false)
+        allow(instance).to receive(:positive?).and_return(true)
       end
 
       it { expect(action).to eq(1.0) }

data/spec/graders/length_spec.rb

@@ -7,7 +7,7 @@ describe LinkThumbnailer::Graders::Length do
   let(:instance)    { described_class.new(description) }
 
   before do
-    instance.stub(:config).and_return(config)
+    allow(instance).to receive(:config).and_return(config)
   end
 
   describe '#call' do
@@ -17,7 +17,7 @@ describe LinkThumbnailer::Graders::Length do
     context 'when text is too short' do
 
       before do
-        instance.stub(:too_short?).and_return(true)
+        allow(instance).to receive(:too_short?).and_return(true)
       end
 
       it { expect(action).to eq(0.0) }
@@ -27,8 +27,8 @@ describe LinkThumbnailer::Graders::Length do
     context 'when text is not too short' do
 
       before do
-        instance.stub(:too_short?).and_return(false)
-        instance.stub(:text).and_return(text)
+        allow(instance).to receive(:too_short?).and_return(false)
+        allow(instance).to receive(:text).and_return(text)
       end
 
       context 'when text length is 120' do
@@ -62,10 +62,12 @@ describe LinkThumbnailer::Graders::Length do
   describe '#too_short?' do
 
     let(:action) { instance.send(:too_short?) }
+    let(:config) { double }
 
     before do
-      instance.stub_chain(:config, :description_min_length).and_return(10)
-      instance.stub(:text).and_return(text)
+      allow(instance).to receive(:config).and_return(config)
+      allow(config).to receive(:description_min_length).and_return(10)
+      allow(instance).to receive(:text).and_return(text)
     end
 
     context 'when true' do

data/spec/graders/link_density_spec.rb

@@ -10,8 +10,8 @@ describe LinkThumbnailer::Graders::LinkDensity do
     let(:action) { instance.call }
 
     before do
-      instance.stub(:text).and_return(text)
-      instance.stub(:links).and_return(links)
+      allow(instance).to receive(:text).and_return(text)
+      allow(instance).to receive(:links).and_return(links)
     end
 
     context 'when text length is 0' do

data/spec/graders/position_spec.rb

@@ -12,8 +12,8 @@ describe LinkThumbnailer::Graders::Position do
     context 'when position is 0' do
 
       before do
-        description.stub(:position).and_return(0)
-        description.stub(:candidates_number).and_return(1)
+        allow(description).to receive(:position).and_return(0)
+        allow(description).to receive(:candidates_number).and_return(1)
       end
 
       it { expect(action).to eq(1.0) }
@@ -23,8 +23,8 @@ describe LinkThumbnailer::Graders::Position do
     context 'when position is 1 over 1 candidates' do
 
       before do
-        description.stub(:position).and_return(1)
-        description.stub(:candidates_number).and_return(1)
+        allow(description).to receive(:position).and_return(1)
+        allow(description).to receive(:candidates_number).and_return(1)
       end
 
       it { expect(action).to eq(0.0) }
@@ -34,8 +34,8 @@ describe LinkThumbnailer::Graders::Position do
     context 'when position is 1 over more than 1 candidates' do
 
       before do
-        description.stub(:position).and_return(1)
-        description.stub(:candidates_number).and_return(2)
+        allow(description).to receive(:position).and_return(1)
+        allow(description).to receive(:candidates_number).and_return(2)
       end
 
       it { expect(action).to eq(0.5) }

data/spec/image_validator_spec.rb

@@ -11,7 +11,7 @@ describe LinkThumbnailer::ImageValidator do
     let(:action) { instance.call }
 
     before do
-      instance.stub(:blacklist_urls).and_return(blacklist_urls)
+      allow(instance).to receive(:blacklist_urls).and_return(blacklist_urls)
     end
 
     context 'when image url is blacklisted' do

data/spec/models/description_spec.rb

@@ -8,7 +8,7 @@ describe LinkThumbnailer::Models::Description do
   let(:instance)  { described_class.new(node, text) }
 
   before do
-    ::LinkThumbnailer::Grader.should_receive(:new).at_least(1).and_return(grader)
+    expect(LinkThumbnailer::Grader).to receive(:new).at_least(1).times.and_return(grader)
   end
 
   describe '#to_s' do

data/spec/models/image_spec.rb

@@ -23,7 +23,7 @@ describe LinkThumbnailer::Models::Image do
     let(:action)  { instance <=> other }
 
     before do
-      instance.stub(:size).and_return([20, 20])
+      allow(instance).to receive(:size).and_return([20, 20])
     end
 
     context 'when other has a smaller image' do
@@ -58,7 +58,7 @@ describe LinkThumbnailer::Models::Image do
     let(:action)    { instance.valid? }
 
     before do
-      instance.stub(:validator).and_return(validator)
+      allow(instance).to receive(:validator).and_return(validator)
     end
 
     it 'calls validator public method' do
@@ -82,8 +82,8 @@ describe LinkThumbnailer::Models::Image do
     }
 
     before do
-      instance.stub(:size).and_return(size)
-      instance.stub(:type).and_return(type)
+      allow(instance).to receive(:size).and_return(size)
+      allow(instance).to receive(:type).and_return(type)
     end
 
     it { expect(action).to eq(result) }

data/spec/models/video_spec.rb

@@ -33,11 +33,11 @@ describe LinkThumbnailer::Models::Video do
     }
 
     before do
-      instance.stub(:id).and_return(id)
-      instance.stub(:size).and_return(size)
-      instance.stub(:duration).and_return(duration)
-      instance.stub(:provider).and_return(provider)
-      instance.stub(:embed_code).and_return(embed_code)
+      allow(instance).to receive(:id).and_return(id)
+      allow(instance).to receive(:size).and_return(size)
+      allow(instance).to receive(:duration).and_return(duration)
+      allow(instance).to receive(:provider).and_return(provider)
+      allow(instance).to receive(:embed_code).and_return(embed_code)
     end
 
     it { expect(action).to eq(result) }

data/spec/models/website_spec.rb

@@ -12,7 +12,7 @@ describe LinkThumbnailer::Models::Website do
     let(:action)  { instance.image = image }
 
     before do
-      image.stub(:valid?).and_return(true)
+      allow(image).to receive(:valid?).and_return(true)
     end
 
     it { expect { action }.to change { instance.images.size }.by(1) }
@@ -27,7 +27,7 @@ describe LinkThumbnailer::Models::Website do
     context 'when image is valid' do
 
       before do
-        image.stub(:valid?).and_return(true)
+        allow(image).to receive(:valid?).and_return(true)
       end
 
       it { expect { action }.to change { instance.images.size }.by(1) }
@@ -37,7 +37,7 @@ describe LinkThumbnailer::Models::Website do
     context 'when image is not valid' do
 
       before do
-        image.stub(:valid?).and_return(false)
+        allow(image).to receive(:valid?).and_return(false)
       end
 
       it { expect { action }.to_not change { instance.images.size } }

data/spec/processor_spec.rb

@@ -7,7 +7,7 @@ describe LinkThumbnailer::Processor do
   let(:url)       { 'http://foo.com' }
 
   before do
-    LinkThumbnailer.stub(:page).and_return(page)
+    allow(LinkThumbnailer).to receive(:page).and_return(page)
   end
 
   describe '#call' do
@@ -81,7 +81,7 @@ describe LinkThumbnailer::Processor do
     context 'when valid' do
 
       before do
-        instance.stub(:valid_url_format?).and_return(true)
+        allow(instance).to receive(:valid_url_format?).and_return(true)
       end
 
       it 'yields' do
@@ -94,7 +94,7 @@ describe LinkThumbnailer::Processor do
     context 'when not valid' do
 
       before do
-        instance.stub(:valid_url_format?).and_return(false)
+        allow(instance).to receive(:valid_url_format?).and_return(false)
       end
 
       it { expect { instance.send(:with_valid_url) }.to raise_error(LinkThumbnailer::BadUriFormat) }
@@ -110,7 +110,7 @@ describe LinkThumbnailer::Processor do
     let(:action)      { instance.send(:set_http_headers) }
 
     before do
-      instance.stub(:user_agent).and_return(user_agent)
+      allow(instance).to receive(:user_agent).and_return(user_agent)
       action
     end
 
@@ -128,7 +128,7 @@ describe LinkThumbnailer::Processor do
       context 'when verify_ssl is true' do
 
         before do
-          instance.stub(:ssl_required?).and_return(true)
+          allow(instance).to receive(:ssl_required?).and_return(true)
           action
         end
 
@@ -139,7 +139,7 @@ describe LinkThumbnailer::Processor do
       context 'when verify_ssl is not true' do
 
         before do
-          instance.stub(:ssl_required?).and_return(false)
+          allow(instance).to receive(:ssl_required?).and_return(false)
           action
         end
 
@@ -154,7 +154,7 @@ describe LinkThumbnailer::Processor do
       let(:http_open_timeout) { 1 }
 
       before do
-        instance.stub(:http_open_timeout).and_return(http_open_timeout)
+        allow(instance).to receive(:http_open_timeout).and_return(http_open_timeout)
         action
       end
 
@@ -167,7 +167,7 @@ describe LinkThumbnailer::Processor do
       let(:http_read_timeout) { 1 }
 
       before do
-        instance.stub(:http_read_timeout).and_return(http_read_timeout)
+        allow(instance).to receive(:http_read_timeout).and_return(http_read_timeout)
         action
       end
 
@@ -179,9 +179,11 @@ describe LinkThumbnailer::Processor do
   describe '#perform_request' do
 
     let(:action) { instance.send(:perform_request) }
+    let(:http) { double }
 
     before do
-      instance.stub_chain(:http, :request).and_return(response)
+      allow(instance).to receive(:http).and_return(http)
+      allow(http).to receive(:request).and_return(response)
     end
 
     context 'when http success' do
@@ -191,13 +193,40 @@ describe LinkThumbnailer::Processor do
       let(:response)  { ::Net::HTTPSuccess.new('', code, body) }
 
       before do
-        response.stub(:body).and_return(body)
+        allow(response).to receive(:body).and_return(body)
       end
 
       it { expect(action).to eq(body) }
 
     end
 
+    context 'when access non-utf8-encoding website' do
+      let(:code) { 200 }
+      let(:body_shift_jis) do
+        File.read(File.expand_path('fixtures/google_shift_jis.html', File.dirname(__FILE__)))
+      end
+      let(:body_utf8) do
+        File.read(File.expand_path('fixtures/google_utf8.html', File.dirname(__FILE__)))
+      end
+      let(:response) do
+        r = ::Net::HTTPSuccess.new('', code, body_shift_jis)
+        r['Content-Type'] = 'text/html'
+        r.body = body_shift_jis
+        r.instance_variable_set(:@read, true)
+        r
+      end
+
+      context 'when http success with valid charset provided in Content-Type' do
+        before { response['Content-Type'] = 'text/html; charset=Shift_JIS' }
+        it { expect(action).to eq body_utf8 }
+      end
+
+      context 'when http success with invalid charset provided in Content-Type' do
+        before { response['Content-Type'] = 'text/html; charset=Shift-JIS' }
+        it { expect(action).to eq body_shift_jis }
+      end
+    end
+
     context 'when http redirection' do
 
       let(:code)      { 200 }
@@ -206,8 +235,8 @@ describe LinkThumbnailer::Processor do
       let(:new_url)   { 'http://foo.com/bar' }
 
       before do
-        instance.stub(:redirect_count).and_return(0)
-        instance.stub(:resolve_relative_url).and_return(new_url)
+        allow(instance).to receive(:redirect_count).and_return(0)
+        allow(instance).to receive(:resolve_relative_url).and_return(new_url)
       end
 
       it 'calls call method' do
@@ -229,7 +258,7 @@ describe LinkThumbnailer::Processor do
     let(:action)        { instance.send(:build_absolute_url_for, relative_url).to_s }
 
     before do
-      instance.stub(:url).and_return(url)
+      allow(instance).to receive(:url).and_return(url)
     end
 
     it { expect(action).to eq("#{scheme}://#{host}#{relative_url}") }
@@ -308,7 +337,7 @@ describe LinkThumbnailer::Processor do
     context 'when bad format' do
 
       before do
-        instance.stub(:url).and_return("http://foo.com")
+        allow(instance).to receive(:url).and_return("http://foo.com")
       end
 
       it { expect(action).to be_falsey }
@@ -318,7 +347,7 @@ describe LinkThumbnailer::Processor do
     context 'when valid format' do
 
       before do
-        instance.stub(:url).and_return(URI("http://foo.com"))
+        allow(instance).to receive(:url).and_return(URI("http://foo.com"))
       end
 
       it { expect(action).to be_truthy }
@@ -334,8 +363,8 @@ describe LinkThumbnailer::Processor do
     context 'when redirect count is greater than redirect limit' do
 
       before do
-        instance.stub(:redirect_count).and_return(5)
-        instance.stub(:redirect_limit).and_return(4)
+        allow(instance).to receive(:redirect_count).and_return(5)
+        allow(instance).to receive(:redirect_limit).and_return(4)
       end
 
       it { expect(action).to be_truthy }
@@ -345,8 +374,8 @@ describe LinkThumbnailer::Processor do
     context 'when redirect count is less than redirect limit' do
 
       before do
-        instance.stub(:redirect_count).and_return(4)
-        instance.stub(:redirect_limit).and_return(5)
+        allow(instance).to receive(:redirect_count).and_return(4)
+        allow(instance).to receive(:redirect_limit).and_return(5)
       end
 
       it { expect(action).to be_falsey }
@@ -356,8 +385,8 @@ describe LinkThumbnailer::Processor do
     context 'when redirect count is equal to redirect limit' do
 
       before do
-        instance.stub(:redirect_count).and_return(5)
-        instance.stub(:redirect_limit).and_return(5)
+        allow(instance).to receive(:redirect_count).and_return(5)
+        allow(instance).to receive(:redirect_limit).and_return(5)
       end
 
       it { expect(action).to be_falsey }

data/spec/response_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+describe LinkThumbnailer::Response do
+  let(:url) { 'https://www.google.co.jp' }
+  let(:page) { ::LinkThumbnailer::Page.new(url, {}) }
+
+  let(:response) do
+    r = ::Net::HTTPSuccess.new('', 200, body_shift_jis)
+    r['Content-Type'] = 'text/html'
+    r.body = body_shift_jis
+    r.instance_variable_set(:@read, true)
+    r
+  end
+
+  let(:instance) { described_class.new(response) }
+
+  let(:body_shift_jis) do
+    File.read(File.expand_path('fixtures/google_shift_jis.html', File.dirname(__FILE__)))
+  end
+
+  let(:body_utf8) do
+    File.read(File.expand_path('fixtures/google_utf8.html', File.dirname(__FILE__)))
+  end
+
+  before do
+    allow(LinkThumbnailer).to receive(:page).and_return(page)
+  end
+
+  describe '#charset' do
+    context 'when charset provided in content-type' do
+      before do
+        response['Content-Type'] = 'text/html; charset=Shift_JIS'
+      end
+
+      it { expect(instance.charset).to eq 'Shift_JIS' }
+    end
+
+    context 'when no charset available in content-type' do
+      it { expect(instance.charset).to eq '' }
+    end
+  end
+
+  describe '#body' do
+    context 'when provide valid charset' do
+      before do
+        response['Content-Type'] = 'text/html; charset=Shift_JIS'
+      end
+
+      it { expect(instance.body).to eq body_utf8 }
+    end
+
+    context 'when provide invalid charset' do
+      before do
+        response['Content-Type'] = 'text/html; charset=Shift-JIS'
+      end
+
+      it { expect(instance.body).to eq body_shift_jis }
+    end
+  end
+end

data/spec/scraper_spec.rb

@@ -9,8 +9,8 @@ describe LinkThumbnailer::Scraper do
   let(:instance)  { described_class.new(source, url) }
 
   before do
-    instance.stub(:document).and_return(document)
-    instance.stub(:website).and_return(website)
+    allow(instance).to receive(:document).and_return(document)
+    allow(instance).to receive(:website).and_return(website)
   end
 
   describe '#call' do
@@ -22,10 +22,10 @@ describe LinkThumbnailer::Scraper do
     let(:attributes)              { [:bar] }
     let(:scrapers)                { [prefix_1, prefix_2] }
     let(:action)                  { instance.call }
+    let(:config)                  { double(attributes: attributes, scrapers: scrapers) }
 
     before do
-      instance.stub_chain(:config, :attributes).and_return(attributes)
-      instance.stub_chain(:config, :scrapers).and_return(scrapers)
+      allow(instance).to receive(:config).and_return(config)
     end
 
     context 'when first one return a result' do

data/spec/scrapers/base_spec.rb

@@ -13,7 +13,7 @@ describe LinkThumbnailer::Scrapers::Base do
     let(:action)  { instance.call(attr) }
 
     before do
-      instance.stub(:value).and_return(value)
+      allow(instance).to receive(:value).and_return(value)
     end
 
     it { expect { action }.to change { website.title }.from(nil).to(value) }
@@ -25,7 +25,7 @@ describe LinkThumbnailer::Scrapers::Base do
     let(:action) { instance.send(:model_class) }
 
     before do
-      instance.stub(:attribute_name).and_return(attr)
+      allow(instance).to receive(:attribute_name).and_return(attr)
     end
 
     context 'when internal class exists' do
@@ -40,7 +40,7 @@ describe LinkThumbnailer::Scrapers::Base do
 
       let(:attr) { :foo }
 
-      it { expect { action }.to raise_exception }
+      it { expect { action }.to raise_error(NameError) }
 
     end
 
@@ -54,7 +54,7 @@ describe LinkThumbnailer::Scrapers::Base do
     let(:action)      { instance.send(:modelize, node, text) }
 
     before do
-      instance.stub(:model_class).and_return(model_class)
+      allow(instance).to receive(:model_class).and_return(model_class)
     end
 
     it 'instantiates a new model' do

data/spec/scrapers/opengraph/base_spec.rb

@@ -12,13 +12,13 @@ describe LinkThumbnailer::Scrapers::Opengraph::Base do
     let(:action)  { instance.applicable? }
 
     before do
-      instance.stub(:meta).and_return(meta)
+      allow(instance).to receive(:meta).and_return(meta)
     end
 
     context 'when all node is an opengraph' do
 
       before do
-        instance.stub(:opengraph_node?).and_return(true, true)
+        allow(instance).to receive(:opengraph_node?).and_return(true, true)
       end
 
       it { expect(action).to be_truthy }
@@ -28,7 +28,7 @@ describe LinkThumbnailer::Scrapers::Opengraph::Base do
     context 'when any node is an opengraph' do
 
       before do
-        instance.stub(:opengraph_node?).and_return(true, false)
+        allow(instance).to receive(:opengraph_node?).and_return(true, false)
       end
 
       it { expect(action).to be_truthy }
@@ -38,7 +38,7 @@ describe LinkThumbnailer::Scrapers::Opengraph::Base do
     context 'when no node is an opengraph' do
 
       before do
-        instance.stub(:opengraph_node?).and_return(false, false)
+        allow(instance).to receive(:opengraph_node?).and_return(false, false)
       end
 
       it { expect(action).to be_falsey }
@@ -52,7 +52,7 @@ describe LinkThumbnailer::Scrapers::Opengraph::Base do
     let(:action) { instance.send(:opengraph_node?, node) }
 
     before do
-      node.stub(:attribute).with('name').and_return(attribute_from_name)
+      allow(node).to receive(:attribute).with('name').and_return(attribute_from_name)
     end
 
     context 'with attribute from name valid' do
@@ -68,7 +68,7 @@ describe LinkThumbnailer::Scrapers::Opengraph::Base do
       let(:attribute_from_name) { 'foo' }
 
       before do
-        node.stub(:attribute).with('property').and_return(attribute_from_property)
+        allow(node).to receive(:attribute).with('property').and_return(attribute_from_property)
       end
 
       context 'and attribute from property valid' do

data/spec/uri_spec.rb

@@ -12,7 +12,7 @@ describe LinkThumbnailer::URI do
     context 'when bad format' do
 
       before do
-        instance.stub(:attribute).and_return("/invalid/path")
+        allow(instance).to receive(:attribute).and_return("/invalid/path")
       end
 
       it { expect(action).to be_falsey }
@@ -22,7 +22,7 @@ describe LinkThumbnailer::URI do
     context 'when valid format' do
 
       before do
-        instance.stub(:attribute).and_return("http://foo.com")
+        allow(instance).to receive(:attribute).and_return("http://foo.com")
       end
 
       it { expect(action).to be_truthy }

data/spec/video_parser_spec.rb

@@ -8,7 +8,7 @@ describe LinkThumbnailer::VideoParser do
   let(:instance)  { described_class.new(video) }
 
   before do
-    instance.stub(:parser).and_return(parser)
+    allow(instance).to receive(:parser).and_return(parser)
   end
 
   describe '#id' do
@@ -18,7 +18,7 @@ describe LinkThumbnailer::VideoParser do
     context 'when respond to video_id' do
 
       before do
-        parser.stub(:video_id).and_return(1)
+        allow(parser).to receive(:video_id).and_return(1)
       end
 
       it { expect(action).to eq(parser.video_id) }
@@ -28,7 +28,7 @@ describe LinkThumbnailer::VideoParser do
     context 'when do not respond to video_id' do
 
       before do
-        parser.stub(:video_id).and_raise(NoMethodError)
+        allow(parser).to receive(:video_id).and_raise(NoMethodError)
       end
 
       it { expect(action).to be_nil }
@@ -44,8 +44,8 @@ describe LinkThumbnailer::VideoParser do
     context 'when respond to width and height' do
 
       before do
-        parser.stub(:width).and_return(1)
-        parser.stub(:height).and_return(1)
+        allow(parser).to receive(:width).and_return(1)
+        allow(parser).to receive(:height).and_return(1)
       end
 
       it { expect(action).to eq([parser.width, parser.height]) }
@@ -55,8 +55,8 @@ describe LinkThumbnailer::VideoParser do
     context 'when do not respond to width and height' do
 
       before do
-        parser.stub(:width).and_raise(NoMethodError)
-        parser.stub(:height).and_raise(NoMethodError)
+        allow(parser).to receive(:width).and_raise(NoMethodError)
+        allow(parser).to receive(:height).and_raise(NoMethodError)
       end
 
       it { expect(action).to be_empty }
@@ -72,7 +72,7 @@ describe LinkThumbnailer::VideoParser do
     context 'when respond to duration' do
 
       before do
-        parser.stub(:duration).and_return(1)
+        allow(parser).to receive(:duration).and_return(1)
       end
 
       it { expect(action).to eq(parser.duration) }
@@ -82,7 +82,7 @@ describe LinkThumbnailer::VideoParser do
     context 'when do not respond to duration' do
 
       before do
-        parser.stub(:duration).and_raise(NoMethodError)
+        allow(parser).to receive(:duration).and_raise(NoMethodError)
       end
 
       it { expect(action).to be_nil }
@@ -98,7 +98,7 @@ describe LinkThumbnailer::VideoParser do
     context 'when respond to provider' do
 
       before do
-        parser.stub(:provider).and_return(1)
+        allow(parser).to receive(:provider).and_return(1)
       end
 
       it { expect(action).to eq(parser.provider) }
@@ -108,7 +108,7 @@ describe LinkThumbnailer::VideoParser do
     context 'when do not respond to provider' do
 
       before do
-        parser.stub(:provider).and_raise(NoMethodError)
+        allow(parser).to receive(:provider).and_raise(NoMethodError)
       end
 
       it { expect(action).to be_nil }
@@ -124,7 +124,7 @@ describe LinkThumbnailer::VideoParser do
     context 'when respond to embed_code' do
 
       before do
-        parser.stub(:embed_code).and_return('')
+        allow(parser).to receive(:embed_code).and_return('')
       end
 
       it { expect(action).to eq(parser.embed_code) }
@@ -134,7 +134,7 @@ describe LinkThumbnailer::VideoParser do
     context 'when do not respond to embed_code' do
 
       before do
-        parser.stub(:embed_code).and_raise(NoMethodError)
+        allow(parser).to receive(:embed_code).and_raise(NoMethodError)
       end
 
       it { expect(action).to be_nil }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: link_thumbnailer
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.3.1
 platform: ruby
 authors:
 - Pierre-Louis Gottfrois
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-27 00:00:00.000000000 Z
+date: 2017-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -28,22 +28,16 @@ dependencies:
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.7.7
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.7.7
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -64,56 +58,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: net-http-persistent
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: video_info
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: image_info
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
 description: Ruby gem generating thumbnail images from a given URL.
 email:
 - pierrelouis.gottfrois@gmail.com
@@ -123,6 +117,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".ruby-version"
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
@@ -156,6 +151,7 @@ files:
 - lib/link_thumbnailer/parser.rb
 - lib/link_thumbnailer/processor.rb
 - lib/link_thumbnailer/railtie.rb
+- lib/link_thumbnailer/response.rb
 - lib/link_thumbnailer/scraper.rb
 - lib/link_thumbnailer/scrapers/base.rb
 - lib/link_thumbnailer/scrapers/default/base.rb
@@ -182,6 +178,8 @@ files:
 - spec/fixtures/default_from_body.html
 - spec/fixtures/default_from_meta.html
 - spec/fixtures/foo.png
+- spec/fixtures/google_shift_jis.html
+- spec/fixtures/google_utf8.html
 - spec/fixtures/og_not_valid_example.html
 - spec/fixtures/og_valid_example.html
 - spec/fixtures/og_valid_multi_image_example.html
@@ -203,6 +201,7 @@ files:
 - spec/models/website_spec.rb
 - spec/page_spec.rb
 - spec/processor_spec.rb
+- spec/response_spec.rb
 - spec/scraper_spec.rb
 - spec/scrapers/base_spec.rb
 - spec/scrapers/opengraph/base_spec.rb
@@ -228,7 +227,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: Ruby gem ranking images from a given URL returning an object containing images
@@ -240,6 +239,8 @@ test_files:
 - spec/fixtures/default_from_body.html
 - spec/fixtures/default_from_meta.html
 - spec/fixtures/foo.png
+- spec/fixtures/google_shift_jis.html
+- spec/fixtures/google_utf8.html
 - spec/fixtures/og_not_valid_example.html
 - spec/fixtures/og_valid_example.html
 - spec/fixtures/og_valid_multi_image_example.html
@@ -261,6 +262,7 @@ test_files:
 - spec/models/website_spec.rb
 - spec/page_spec.rb
 - spec/processor_spec.rb
+- spec/response_spec.rb
 - spec/scraper_spec.rb
 - spec/scrapers/base_spec.rb
 - spec/scrapers/opengraph/base_spec.rb