ogo 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07a11957eb9e8e7a755ef7508125319f86e14b65
-  data.tar.gz: 31e44eeb7a939ea3afbcdd9a5a309bbdf86e8f8a
+  metadata.gz: 41df5206b948578ceb2d31a306c73f84f35b837e
+  data.tar.gz: f871b617057c6b54f11dd2aa255880d6a623db8c
 SHA512:
-  metadata.gz: 4cbbc4c57a3f4fab3e09fafd6853d78f2674d9da677aa73977051c0610eac0f35e0339af37f7d3031d33590197f80800d6f740f68a00e86126f41b4702acd261
-  data.tar.gz: 3e3c2516b4fdaf125c1bb7d378630f4c3faa6762f8efc22613d3fceae9ebcfa89b39831ff4c4e13f8c87315ce3dca2bee52db34eed96dc87af681573476b5a1c
+  metadata.gz: ddeef19fba2db90c1f70729e9d99ed8d7c2f908eafb83b2b791139262a88d66d096cf8b855e695ad6608112f6652f27fc5a21a579e135b067caf0fdb784117e6
+  data.tar.gz: 1b714a39fece5216852e31eadfbcf9822e50cd3233c98871015b55d0d99ceeed3587132bd6cb96129cef644a62374ff5102654730931eb938f16f12ef96b9fb3

data/.gitignore

@@ -5,3 +5,4 @@ pkg/*
 tmp/*
 .rvmrc
 *.idea
+.byebug_history

data/README.md

@@ -3,7 +3,9 @@
 ## Information
 
 Implementation of first verion of this gem based mostly on @huyha85's opengraph_parser
-https://github.com/huyha85/opengraph_parser and its forks.
+https://github.com/huyha85/opengraph_parser and its forks:
+
+Charset support taken from @codefist fork: https://github.com/codefirst/opengraph_parser
 
 I prefer to make my own repo for this gem as origin repo is seemingly-abandoned and
 I need more functionality than just opengraph (ogp.me) tags parsing.

data/lib/ogo.rb

@@ -1,4 +1,7 @@
 require 'addressable/uri'
+require 'uri'
+require 'nokogiri'
+
 require_relative 'ogo/opengraph'
 require_relative 'ogo/utils/redirect_follower'
 

data/lib/ogo/opengraph.rb

@@ -1,18 +1,17 @@
-require 'nokogiri'
-require 'utils/redirect_follower'
-require "addressable/uri"
-require 'uri'
-
 module Ogo
   class Opengraph
 
-    attr_accessor :src, :url, :type, :title, :description, :images, :metadata, :response, :original_images
+    attr_accessor :src, :url, :type, :title, :description,
+      :images, :metadata, :response, :body, :charset, :original_images,
+      :error
 
     def initialize(src, options = {})
       @src = src
       @body = nil
       @images = []
       @metadata = {}
+      @charset = 'utf-8'
+      @error = nil
 
       @_fallback = options[:fallback] || true
       @_options = options
@@ -22,6 +21,22 @@ module Ogo
       parse_opengraph(@_options)
       load_fallback if @_fallback
       check_images_path
+      self
+    end
+
+    def parse!
+      parse
+      error ? raise(error) : self
+    end
+
+    def inspect
+      str = "<Ogo::Opengraph:0x00#{'%x' % (self.object_id << 1)}\nurl=\"#{url}\",\nmetadata=#{metadata},\n"
+      str << "images=#{images},\ntype=\"#{type}\",\ntitle=\"#{title}\">"
+      str
+    end
+
+    def to_s
+      inspect
     end
 
     private
@@ -31,16 +46,19 @@ module Ogo
         if src.include? '</html>'
           self.body = src
         else
-          self.body = Ogo::Utils::RedirectFollower.new(src, options).resolve.body
+          resolved = Ogo::Utils::RedirectFollower.new(src, options).resolve
+          self.body = resolved.body
+          self.charset = resolved.charset if resolved.charset
         end
-      rescue
+      rescue => e
         self.title = self.url = src
+        self.error = e
         return
       end
 
       if body
         attrs_list = %w(title url type description)
-        doc = Nokogiri.parse(body)
+        doc = parse_html
         doc.css('meta').each do |m|
           if m.attribute('property') && m.attribute('property').to_s.match(/^og:(.+)$/i)
             m_content = m.attribute('content').to_s.strip
@@ -59,7 +77,7 @@ module Ogo
 
     def load_fallback
       if body
-        doc = Nokogiri.parse(body)
+        doc = parse_html
 
         if title.to_s.empty? && doc.xpath("//head//title").size > 0
           self.title = doc.xpath("//head//title").first.text.to_s.strip
@@ -82,7 +100,7 @@ module Ogo
 
     def check_images_path
       self.original_images = images.dup
-      uri = Addressable::URI.parse(src)
+      uri = Addressable::URI.parse(url)
       imgs = images.dup
       self.images = []
       imgs.each do |img|
@@ -135,5 +153,27 @@ module Ogo
         metadata_container
       end
     end
+
+    def parse_html
+      unless charset
+        doc = Nokogiri.parse(body.scrub)
+        self.charset = guess_encoding(doc)
+      end
+      Nokogiri::HTML(body, nil, charset)
+    end
+
+    def guess_encoding(doc)
+      _charset = doc.xpath('//meta/@charset').first
+      return _charset.value.to_s if charset
+
+      _charset = doc.xpath('//meta').each do |m|
+        if m.attribute('http-equiv') && m.attribute('content') && m.attribute('http-equiv').value.casecmp('Content-Type')
+          return m.attribute('content').value.split('charset=').last.strip
+        end
+      end
+
+      'UTF-8'
+    end
+
   end
 end

data/lib/ogo/utils/redirect_follower.rb

@@ -6,22 +6,23 @@ module Ogo
       REDIRECT_DEFAULT_LIMIT = 5
 
       class TooManyRedirects < StandardError; end
+      class EmptyURLError < ArgumentError; end
 
-      attr_accessor :url, :body, :redirect_limit, :response, :headers
+      attr_accessor :url, :body, :charset, :redirect_limit, :response, :headers
 
       def initialize(url, options = {})
-        @limit = options[:limit] || REDIRECT_DEFAULT_LIMIT
+        raise EmptyURLError if url.to_s.empty?
+        @redirect_limit = options[:limit] || REDIRECT_DEFAULT_LIMIT
         @headers = options[:headers] || {}
-        @url = url
+        @url = url.start_with?('http') ? url : "http://#{url}"
       end
 
       def resolve
         raise TooManyRedirects if redirect_limit < 0
 
-        url = "http://#{url}" unless url.starts_with?('http')
-        uri = Addressable::URI.parse(URI.escape(url))
+        uri = Addressable::URI.parse(url).normalize
 
-        http = Net::HTTP.new(uri.host, uri.port)
+        http = Net::HTTP.new(uri.host, uri.inferred_port)
         if uri.scheme == 'https'
           http.use_ssl = true
           http.verify_mode = OpenSSL::SSL::VERIFY_PEER
@@ -35,6 +36,14 @@ module Ogo
           resolve
         end
 
+        charset = nil
+        if content_type = response['content-type']
+          if content_type =~ /charset=(.+)/i
+            charset = $1
+          end
+        end
+        self.charset = charset
+
         self.body = response.body
         self
       end

data/lib/ogo/version.rb

@@ -1,3 +1,3 @@
 module Ogo
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

data/ogo.gemspec

@@ -19,5 +19,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'nokogiri', '>= 1.6'
   s.add_dependency 'addressable', '>= 2.4.0'
   s.add_development_dependency 'rspec', '>= 3.0'
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'byebug'
   s.add_development_dependency 'rake'
 end

data/spec/lib/ogo/opengraph_spec.rb

@@ -0,0 +1,133 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+describe Ogo::Opengraph do
+  describe "#initialize" do
+    context "with invalid src" do
+      it "set title and url the same as src" do
+        og = Ogo::Opengraph.new("invalid")
+        og.parse
+        expect(og.src).to eq "invalid"
+        expect(og.title).to eq "invalid"
+        expect(og.url).to eq "invalid"
+      end
+    end
+
+    context "with no fallback" do
+      it "should get values from opengraph metadata" do
+        response = double(charset: 'UTF-8', body: File.open("#{File.dirname(__FILE__)}/../../view/opengraph.html", 'r') { |f| f.read })
+        Ogo::Utils::RedirectFollower.stub(:new) { double(resolve: response) }
+
+        og = Ogo::Opengraph.new("http://test.host", fallback: false)
+        og.parse
+        og.src.should == "http://test.host"
+        og.title.should == "OpenGraph Title"
+        og.type.should == "article"
+        og.url.should == "http://test.host"
+        og.description.should == "My OpenGraph sample site for Rspec"
+        og.images.should == ["http://test.host/images/rock1.jpg", "http://test.host/images/rock2.jpg"]
+        og.original_images.should == ["http://test.host/images/rock1.jpg", "/images/rock2.jpg"]
+        og.metadata.should == {
+          title: [{_value: "OpenGraph Title"}],
+          type: [{_value: "article"}],
+          url: [{_value: "http://test.host"}],
+          description: [{_value: "My OpenGraph sample site for Rspec"}],
+          image: [
+            {
+              _value: "http://test.host/images/rock1.jpg",
+              width: [{ _value: "300" }],
+              height: [{ _value: "300" }]
+            },
+            {
+              _value: "/images/rock2.jpg",
+              height: [{ _value: "1000" }]
+            }
+          ],
+          locale: [
+            {
+              _value: "en_GB",
+              alternate: [
+                { _value: "fr_FR" },
+                { _value: "es_ES" }
+              ]
+            }
+          ]
+        }
+      end
+    end
+
+    context "with fallback" do
+      context "when website has opengraph metadata" do
+        it "should get values from opengraph metadata" do
+          response = double(charset: 'UTF-8', body: File.open("#{File.dirname(__FILE__)}/../../view/opengraph.html", 'r') { |f| f.read })
+          Ogo::Utils::RedirectFollower.stub(:new) { double(resolve: response) }
+
+          og = Ogo::Opengraph.new("http://test.host")
+          og.parse
+          og.src.should == "http://test.host"
+          og.title.should == "OpenGraph Title"
+          og.type.should == "article"
+          og.url.should == "http://test.host"
+          og.description.should == "My OpenGraph sample site for Rspec"
+          og.images.should == ["http://test.host/images/rock1.jpg", "http://test.host/images/rock2.jpg"]
+        end
+      end
+
+      context "when website has no opengraph metadata" do
+        it "should lookup for other data from website" do
+          response = double(charset: 'UTF-8', body: File.open("#{File.dirname(__FILE__)}/../../view/opengraph_no_metadata.html", 'r') { |f| f.read })
+          Ogo::Utils::RedirectFollower.stub(:new) { double(resolve: response) }
+
+          og = Ogo::Opengraph.new("http://test.host/child_page")
+          og.parse
+          og.src.should == "http://test.host/child_page"
+          og.title.should == "OpenGraph Title Fallback"
+          og.type.should be_nil
+          og.url.should == "http://test.host/child_page"
+          og.description.should == "Short Description Fallback"
+          og.images.should == ["http://test.host/images/wall1.jpg", "http://test.host/images/wall2.jpg"]
+        end
+      end
+    end
+
+    context "with body" do
+      it "should parse body instead of downloading it" do
+        content = File.read("#{File.dirname(__FILE__)}/../../view/opengraph.html")
+        Ogo::Utils::RedirectFollower.should_not_receive(:new)
+
+        og = Ogo::Opengraph.new(content)
+        og.parse
+        og.src.should == content
+        og.title.should == "OpenGraph Title"
+        og.type.should == "article"
+        og.url.should == "http://test.host"
+        og.description.should == "My OpenGraph sample site for Rspec"
+        og.images.should == ["http://test.host/images/rock1.jpg", "http://test.host/images/rock2.jpg"]
+      end
+    end
+
+    context "with Shift_JIS" do
+      it "should get values from opengraph metadata" do
+        pending 'Not working'
+        response = double(charset: 'euc-jp', body: File.open("#{File.dirname(__FILE__)}/../../view/shift_jis.html", 'r') { |f| f.read })
+        Ogo::Utils::RedirectFollower.stub(:new) { double(resolve: response) }
+
+        og = Ogo::Opengraph.new("http://test.host", fallback: false)
+        og.parse
+        og.src.should == "http://test.host"
+        og.title.should == "日本語"
+      end
+    end
+    context "with EUC-JP" do
+      it "should get values from opengraph metadata" do
+        pending 'Not working'
+        response = double(charset: 'euc-jp', body: File.open("#{File.dirname(__FILE__)}/../../view/euc-jp.html", 'r') { |f| f.read })
+        Ogo::Utils::RedirectFollower.stub(:new) { double(resolve: response) }
+
+        og = Ogo::Opengraph.new("http://test.host", fallback: false)
+        og.parse
+        og.src.should == "http://test.host"
+        og.title.should == "日本語"
+      end
+    end
+  end
+end

data/spec/lib/ogo/utils/redirect_follower_spec.rb

@@ -0,0 +1,83 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../../spec_helper')
+
+describe Ogo::Utils::RedirectFollower do
+  describe "#resolve" do
+    let(:url) { "http://test.host" }
+    let(:https_url) { "https://test.host" }
+    let(:mock_res) { double(body: "Body is here.") }
+    let(:mock_redirect) {
+      m = double(body: %Q{<body><a href="http://new.test.host"></a></body>}, kind_of?: Net::HTTPRedirection)
+      m.stub(:[]).and_return(nil)
+      m
+    }
+
+    context "with no redirection" do
+      it "should return the response" do
+        uri = URI.parse(URI.escape(url))
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        Net::HTTP.should_receive(:new).with(uri.host, uri.port).and_return(http)
+        http.should_receive(:request_get).and_return(mock_res)
+
+        res = Ogo::Utils::RedirectFollower.new(url).resolve
+        res.body.should == "Body is here."
+        res.redirect_limit.should == Ogo::Utils::RedirectFollower::REDIRECT_DEFAULT_LIMIT
+      end
+
+      describe "and uri scheme is HTTPS" do
+        it "should use https method to retrieve the uri" do
+          uri = URI.parse(URI.escape(https_url))
+
+          https = Net::HTTP.new(uri.host, uri.port)
+          Net::HTTP.should_receive(:new).with(uri.host, uri.port).and_return(https)
+          https.should_receive(:request_get).and_return(mock_res)
+
+          res = Ogo::Utils::RedirectFollower.new(https_url).resolve
+          res.body.should == "Body is here."
+          res.redirect_limit.should == Ogo::Utils::RedirectFollower::REDIRECT_DEFAULT_LIMIT
+        end
+      end
+
+      describe "and has headers option" do
+        it "should add headers when retrieve the uri" do
+          uri = URI.parse(URI.escape(url))
+
+          http = Net::HTTP.new(uri.host, uri.port)
+          Net::HTTP.should_receive(:new).with(uri.host, uri.port).and_return(http)
+          http.should_receive(:request_get).and_return(mock_res)
+          res = Ogo::Utils::RedirectFollower.new(url, {:headers => {'User-Agent' => 'My Custom User-Agent'}}).resolve
+          res.body.should == "Body is here."
+          res.redirect_limit.should == Ogo::Utils::RedirectFollower::REDIRECT_DEFAULT_LIMIT
+        end
+      end
+    end
+
+    context "with redirection" do
+      it "should follow the link in redirection" do
+        uri = URI.parse(URI.escape(url))
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        Net::HTTP.should_receive(:new).twice.and_return(http)
+        http.should_receive(:request_get).twice.and_return(mock_redirect, mock_res)
+
+        res = Ogo::Utils::RedirectFollower.new(url).resolve
+        res.body.should == "Body is here."
+        res.redirect_limit.should == Ogo::Utils::RedirectFollower::REDIRECT_DEFAULT_LIMIT - 1
+      end
+    end
+
+    context "with unlimited redirection" do
+      it "should raise TooManyRedirects error" do
+        uri = URI.parse(URI.escape(url))
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        Net::HTTP.stub(:new).and_return(http)
+        http.stub(:request_get).and_return(mock_redirect)
+
+        lambda {
+          Ogo::Utils::RedirectFollower.new(url).resolve
+        }.should raise_error(Ogo::Utils::RedirectFollower::TooManyRedirects)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'ogo'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+end

data/spec/view/euc-jp.html

@@ -0,0 +1,23 @@
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=euc-jp">
+    <title>OpenGraph Title Fallback</title>
+    <meta property="og:title" content="ÆüËܸì" />
+    <meta property="og:type" content="article" />
+    <meta property="og:url" content="http://test.host" />
+    <meta property="og:description" content="My OpenGraph sample site for Rspec" />
+    <meta property="og:image" content="http://test.host/images/rock1.jpg" />
+    <meta property="og:image:width" content="300" />
+    <meta property="og:image:height" content="300" />
+    <meta property="og:image" content="/images/rock2.jpg" />
+    <meta property="og:image:height" content="1000" />
+    <meta property="og:locale" content="en_GB" />
+    <meta property="og:locale:alternate" content="fr_FR" />
+    <meta property="og:locale:alternate" content="es_ES" />
+    <meta name="description" content="Short Description Fallback" />
+  </head>
+  <body>
+    <img src="http://test.host/images/wall1.jpg" />
+    <img src="http://test.host/images/wall2.jpg" />
+  </body>
+</html>

data/spec/view/opengraph.html

@@ -0,0 +1,22 @@
+<html>
+  <head>
+    <title>OpenGraph Title Fallback</title>
+    <meta property="og:title" content="OpenGraph Title" />
+    <meta property="og:type" content="article" />
+    <meta property="og:url" content="http://test.host" />
+    <meta property="og:description" content="My OpenGraph sample site for Rspec" />
+    <meta property="og:image" content="http://test.host/images/rock1.jpg" />
+    <meta property="og:image:width" content="300" />
+    <meta property="og:image:height" content="300" />
+    <meta property="og:image" content="/images/rock2.jpg" />
+    <meta property="og:image:height" content="1000" />
+    <meta property="og:locale" content="en_GB" />
+    <meta property="og:locale:alternate" content="fr_FR" />
+    <meta property="og:locale:alternate" content="es_ES" />
+    <meta name="description" content="Short Description Fallback" />
+  </head>
+  <body>
+    <img src="http://test.host/images/wall1.jpg" />
+    <img src="http://test.host/images/wall2.jpg" />
+  </body>
+</html>

data/spec/view/opengraph_no_meta_nor_description.html

@@ -0,0 +1,12 @@
+<html>
+  <head>
+    <title>OpenGraph Title Fallback</title>
+    <meta property="og:type" content="" />
+    <meta property="og:url" content="" />
+  </head>
+  <body>
+    <img src="/images/wall1.jpg" />
+    <img src="../images/wall2.jpg" />
+    <p>No description meta here.</p>
+  </body>
+</html>

data/spec/view/opengraph_no_metadata.html

@@ -0,0 +1,12 @@
+<html>
+  <head>
+    <title>OpenGraph Title Fallback</title>
+    <meta name="description" content="Short Description Fallback" />
+    <meta property="og:type" content="" />
+    <meta property="og:url" content="" />
+  </head>
+  <body>
+    <img src="/images/wall1.jpg" />
+    <img src="../images/wall2.jpg" />
+  </body>
+</html>

data/spec/view/shift_jis.html

@@ -0,0 +1,23 @@
+<html>
+  <head>
+    <meta charset="Shift_JIS" />
+    <title>OpenGraph Title Fallback</title>
+    <meta property="og:title" content="“ú–{Œê" />
+    <meta property="og:type" content="article" />
+    <meta property="og:url" content="http://test.host" />
+    <meta property="og:description" content="My OpenGraph sample site for Rspec" />
+    <meta property="og:image" content="http://test.host/images/rock1.jpg" />
+    <meta property="og:image:width" content="300" />
+    <meta property="og:image:height" content="300" />
+    <meta property="og:image" content="/images/rock2.jpg" />
+    <meta property="og:image:height" content="1000" />
+    <meta property="og:locale" content="en_GB" />
+    <meta property="og:locale:alternate" content="fr_FR" />
+    <meta property="og:locale:alternate" content="es_ES" />
+    <meta name="description" content="Short Description Fallback" />
+  </head>
+  <body>
+    <img src="http://test.host/images/wall1.jpg" />
+    <img src="http://test.host/images/wall2.jpg" />
+  </body>
+</html>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ogo
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - gazay
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-23 00:00:00.000000000 Z
+date: 2016-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -52,6 +52,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -83,6 +111,14 @@ files:
 - lib/ogo/utils/redirect_follower.rb
 - lib/ogo/version.rb
 - ogo.gemspec
+- spec/lib/ogo/opengraph_spec.rb
+- spec/lib/ogo/utils/redirect_follower_spec.rb
+- spec/spec_helper.rb
+- spec/view/euc-jp.html
+- spec/view/opengraph.html
+- spec/view/opengraph_no_meta_nor_description.html
+- spec/view/opengraph_no_metadata.html
+- spec/view/shift_jis.html
 homepage: https://github.com/gazay/ogo
 licenses:
 - MIT