ogo 0.0.2 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41df5206b948578ceb2d31a306c73f84f35b837e
-  data.tar.gz: f871b617057c6b54f11dd2aa255880d6a623db8c
+  metadata.gz: 3916d06ba08cf0c304c487d5754590b0bfdb64aa
+  data.tar.gz: 06cc4a8c9e9c38f84c86fe56acab9756901ea058
 SHA512:
-  metadata.gz: ddeef19fba2db90c1f70729e9d99ed8d7c2f908eafb83b2b791139262a88d66d096cf8b855e695ad6608112f6652f27fc5a21a579e135b067caf0fdb784117e6
-  data.tar.gz: 1b714a39fece5216852e31eadfbcf9822e50cd3233c98871015b55d0d99ceeed3587132bd6cb96129cef644a62374ff5102654730931eb938f16f12ef96b9fb3
+  metadata.gz: c13484e58cf8a94f6cf675f346116277e315bc42ba21b9e031d7abf2be99d48bfffa7a3ab5134bb71e87b134a17c855b62bb7a2cd1af048f29a09fa66eb61d5b
+  data.tar.gz: 866bf793b4475f784ca534119063e6ebe18452993ff1fdef32f08f48231c90d9c4bf4f622561512a37b3a83c5024226b92ca6015a184a9d0c1e7a0bb599976af

data/lib/ogo/image_info.rb

@@ -0,0 +1,76 @@
+module Ogo
+  class ImageInfo
+
+    attr_accessor :url
+
+    def initialize(opts={})
+      @url    = opts[:url]
+      @width  = opts[:width]
+      @height = opts[:height]
+      @type   = opts[:type]
+    end
+
+    def width
+      fetch_size[0]
+    end
+
+    def height
+      fetch_size[1]
+    end
+
+    def type
+      fetch_type
+    end
+
+    def content_type
+      "image/#{fetch_type}"
+    end
+
+    def fetch_size
+      return [@width, @height] if @width && @height
+      if defined?(FastImage)
+        @width, @height = fi_check(:size, url)
+      else
+        []
+      end
+    end
+
+    def fetch_size!
+      @width, @height = nil
+      fetch_size
+    end
+
+    def fetch_type
+      @type ||= \
+        if defined?(FastImage)
+          fi_check(:type, url).to_s
+        else
+          uri = Addressable::URI.parse(url).normalize
+          uri.path.split('.').last.to_s
+        end
+    end
+
+    def fetch_type!
+      @type = nil
+      fetch_type
+    end
+
+    private
+
+    def fi_check(method, url, options=nil)
+      options ||= {raise_on_failure: true, timeout: 2.0}
+      FastImage.send(method, url, options)
+    rescue
+      begin
+        url = Addressable::URI.parse(url).normalize
+        val = FastImage.send(method, url, options)
+        @url = url
+        val
+      rescue => e
+        puts "Image url error: url=\"#{url}\", error=\"#{e}\""
+        nil
+      end
+    end
+
+  end
+end

data/lib/ogo/page_source.rb

@@ -0,0 +1,60 @@
+module Ogo
+  class PageSource
+
+    attr_reader :url, :src, :charset, :doc
+
+    def initialize(src, options={})
+      @src = src
+      @url = options[:url]
+      @charset = options[:charset]
+    end
+
+    def parse
+      unless charset
+        _doc = Nokogiri.parse(src.scrub)
+        @charset = guess_encoding(_doc)
+      end
+      Nokogiri::HTML(src, nil, charset)
+    end
+
+    def parse!
+      @doc = parse
+      self
+    end
+
+    def inspect
+      str = "<Ogo::PageSource:0x00#{'%x' % (self.object_id << 1)}\n"
+      str << "url=\"#{url}\",\n"
+      str << "charset=\"#{charset}\",\n"
+      str << "src=\"#{src.to_s.truncate(100, omission: '...')}\",\n"
+      str << "doc=#{doc.to_s.truncate(100, omission: '...')}\" >"
+      str
+    end
+
+    def to_s
+      inspect
+    end
+
+    private
+
+    def guess_encoding(_doc)
+      _charset = _doc.xpath('//meta/@charset').first
+      return _charset.value.to_s if charset
+
+      _charset = _doc.xpath('//meta').each do |m|
+        if content_tag?(m)
+          return m.attribute('content').value.split('charset=').last.strip
+        end
+      end
+
+      'UTF-8'
+    end
+
+    def content_tag?(m)
+      m.attribute('http-equiv') &&
+        m.attribute('content') &&
+        m.attribute('http-equiv').value.casecmp('Content-Type')
+    end
+
+  end
+end

data/lib/ogo/parsers/base.rb

@@ -0,0 +1,106 @@
+module Ogo
+  module Parsers
+    class Base
+
+      attr_reader :page, :url
+
+      def initialize(parseable)
+        @page = \
+          if parseable.include?('</html>')
+            @url = ''
+            Ogo::PageSource.new(parseable).parse!
+          else
+            _rf = Ogo::Utils::RedirectFollower.new(parseable).resolve
+            page = Ogo::PageSource.new(_rf.body, charset: _rf.charset, url: _rf.url)
+            @url = _rf.url
+            page.parse!
+          end
+        @type = 'website'
+      end
+
+      def title(fallback=false)
+        title_tag = page.doc.xpath('//head//title').first
+        title_tag && title_tag.text.to_s.strip
+      end
+
+      def description(fallback=false)
+        description_meta = page.doc.xpath("//head//meta[@name='description']").first
+        _desc = description_meta && description_meta.attribute("content").to_s.strip
+        if !_desc || _desc.empty?
+          _desc = fetch_first_text
+        end
+        _desc
+      end
+
+      def image(fallback=false)
+        all_images.first
+      end
+
+      def type(fallback=false)
+        @type
+      end
+
+      def all_images
+        @all_images ||= \
+          begin
+            imgs = (
+              fetch_images("//head//meta[@itemprop='image']", "content") +
+              fetch_images("//head//meta[@itemprop='logo']", "content") +
+              fetch_images("//head//meta[@property='og:image']", "content") +
+              fetch_images("//head//meta[@property='twitter:image:src']", "content") +
+              fetch_images("//head//link[@rel='image_src']", "href") +
+              fetch_images("//img", "src")
+            ).flatten.compact.uniq
+            host_uri = Addressable::URI.parse(url)
+            imgs.map { |img|
+              Ogo::ImageInfo.new(url: fix_image_path(img, host_uri))
+            }
+          end
+      end
+
+      def metadata(fallback=false)
+        _meta = {
+          title: title,
+          description: description,
+          type: type,
+          image: nil
+        }
+        if image
+          _meta[:image] = {
+            url:    image.url,
+            width:  image.width,
+            height: image.height,
+            type:   image.type
+          }
+        end
+        _meta
+      end
+
+      private
+
+      def fix_image_path(img, host_uri)
+        return "http:#{img}" if img.start_with?('//')
+        return img if host_uri.host.nil?
+        if Addressable::URI.parse(img).host.nil?
+          host_uri.join(img).to_s
+        else
+          img
+        end
+      end
+
+      def fetch_first_text
+        page.doc.xpath('//p').each do |p|
+          s = p.text.to_s.strip
+          return s if s.length > 20
+        end
+      end
+
+      def fetch_images(xpath_str, attr)
+        page.doc.xpath(xpath_str).map do |tag|
+          tag.attribute(attr).to_s.strip
+        end.reject { |it| it.empty? }.uniq
+      end
+
+    end
+  end
+end

data/lib/ogo/parsers/opengraph.rb

@@ -0,0 +1,90 @@
+module Ogo
+  module Parsers
+    class Opengraph < Ogo::Parsers::Base
+
+      def initialize(parseable, fallback=false)
+        @global_fallback = fallback
+        super parseable
+      end
+
+      def title(fallback=false)
+        if fallback
+          super
+        else
+          _val = find_meta('title')
+          (!_val.empty? && _val) ||
+            (@global_fallback && super) ||
+            ''
+        end
+      end
+
+      def description(fallback=false)
+        if fallback
+          super
+        else
+          _val = find_meta('description')
+          (!_val.empty? && _val) ||
+            (@global_fallback && super) ||
+            ''
+        end
+      end
+
+      def image(fallback=false)
+        if fallback
+          super
+        else
+          _val = find_meta('image')
+          if _val.empty?
+            (@global_fallback && super) || nil
+          else
+            host_uri = Addressable::URI.parse(url)
+            Ogo::ImageInfo.new(url: fix_image_path(_val, host_uri))
+          end
+        end
+      end
+
+      def type(fallback=false)
+        if fallback
+          super
+        else
+          _val = find_meta('type')
+          (!_val.empty? && _val) ||
+            (@global_fallback && super) ||
+            ''
+        end
+      end
+
+      def metadata(fallback=false)
+        _meta = super
+        if fallback
+          _meta[:fallback] = {
+            title: title(true),
+            description: description(true),
+            type: type(true),
+            image: nil
+          }
+          img = image(true)
+          if img
+            _meta[:fallback][:image] = {
+              url:    img.url,
+              width:  img.width,
+              height: img.height,
+              type:   img.type
+            }
+          end
+        end
+        _meta
+      end
+
+      private
+
+      def find_meta(meta_type)
+        tag = page.doc.xpath('//head//meta').find { |it|
+          it.attribute('property').to_s == "og:#{meta_type}"
+        }
+        (tag && tag.attribute('content')).to_s.strip
+      end
+
+    end
+  end
+end

data/lib/ogo/parsers.rb

@@ -0,0 +1,6 @@
+module Ogo
+  module Parsers
+    require_relative 'parsers/base'
+    require_relative 'parsers/opengraph'
+  end
+end

data/lib/ogo/version.rb

@@ -1,3 +1,3 @@
 module Ogo
-  VERSION = '0.0.2'
+  VERSION = '0.1.1'
 end

data/lib/ogo.rb

@@ -2,8 +2,11 @@ require 'addressable/uri'
 require 'uri'
 require 'nokogiri'
 
-require_relative 'ogo/opengraph'
+require_relative 'ogo/page_source'
+require_relative 'ogo/image_info'
+require_relative 'ogo/parsers'
 require_relative 'ogo/utils/redirect_follower'
+require_relative 'ogo/version'
 
 module Ogo
 

data/ogo.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'nokogiri', '>= 1.6'
   s.add_dependency 'addressable', '>= 2.4.0'
   s.add_development_dependency 'rspec', '>= 3.0'
+  s.add_development_dependency 'fastimage'
   s.add_development_dependency 'pry'
   s.add_development_dependency 'byebug'
   s.add_development_dependency 'rake'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ogo
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.1
 platform: ruby
 authors:
 - gazay
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-24 00:00:00.000000000 Z
+date: 2016-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fastimage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -107,7 +121,11 @@ files:
 - README.md
 - Rakefile
 - lib/ogo.rb
-- lib/ogo/opengraph.rb
+- lib/ogo/image_info.rb
+- lib/ogo/page_source.rb
+- lib/ogo/parsers.rb
+- lib/ogo/parsers/base.rb
+- lib/ogo/parsers/opengraph.rb
 - lib/ogo/utils/redirect_follower.rb
 - lib/ogo/version.rb
 - ogo.gemspec

data/lib/ogo/opengraph.rb

@@ -1,179 +0,0 @@
-module Ogo
-  class Opengraph
-
-    attr_accessor :src, :url, :type, :title, :description,
-      :images, :metadata, :response, :body, :charset, :original_images,
-      :error
-
-    def initialize(src, options = {})
-      @src = src
-      @body = nil
-      @images = []
-      @metadata = {}
-      @charset = 'utf-8'
-      @error = nil
-
-      @_fallback = options[:fallback] || true
-      @_options = options
-    end
-
-    def parse
-      parse_opengraph(@_options)
-      load_fallback if @_fallback
-      check_images_path
-      self
-    end
-
-    def parse!
-      parse
-      error ? raise(error) : self
-    end
-
-    def inspect
-      str = "<Ogo::Opengraph:0x00#{'%x' % (self.object_id << 1)}\nurl=\"#{url}\",\nmetadata=#{metadata},\n"
-      str << "images=#{images},\ntype=\"#{type}\",\ntitle=\"#{title}\">"
-      str
-    end
-
-    def to_s
-      inspect
-    end
-
-    private
-
-    def parse_opengraph(options = {})
-      begin
-        if src.include? '</html>'
-          self.body = src
-        else
-          resolved = Ogo::Utils::RedirectFollower.new(src, options).resolve
-          self.body = resolved.body
-          self.charset = resolved.charset if resolved.charset
-        end
-      rescue => e
-        self.title = self.url = src
-        self.error = e
-        return
-      end
-
-      if body
-        attrs_list = %w(title url type description)
-        doc = parse_html
-        doc.css('meta').each do |m|
-          if m.attribute('property') && m.attribute('property').to_s.match(/^og:(.+)$/i)
-            m_content = m.attribute('content').to_s.strip
-            metadata_name = m.attribute('property').to_s.gsub("og:", "")
-            self.metadata = add_metadata(metadata, metadata_name, m_content)
-            case metadata_name
-              when *attrs_list
-                self.instance_variable_set("@#{metadata_name}", m_content) unless m_content.empty?
-              when "image"
-                add_image(m_content)
-            end
-          end
-        end
-      end
-    end
-
-    def load_fallback
-      if body
-        doc = parse_html
-
-        if title.to_s.empty? && doc.xpath("//head//title").size > 0
-          self.title = doc.xpath("//head//title").first.text.to_s.strip
-        end
-
-        self.url = src if url.to_s.empty?
-
-        if description.to_s.empty? && description_meta = doc.xpath("//head//meta[@name='description']").first
-          self.description = description_meta.attribute("content").to_s.strip
-        end
-
-        if description.to_s.empty?
-          self.description = fetch_first_text(doc)
-        end
-
-        fetch_images(doc, "//head//link[@rel='image_src']", "href") if images.empty?
-        fetch_images(doc, "//img", "src") if images.empty?
-      end
-    end
-
-    def check_images_path
-      self.original_images = images.dup
-      uri = Addressable::URI.parse(url)
-      imgs = images.dup
-      self.images = []
-      imgs.each do |img|
-        if Addressable::URI.parse(img).host.nil?
-          full_path = uri.join(img).to_s
-          add_image(full_path)
-        else
-          add_image(img)
-        end
-      end
-    end
-
-    def add_image(image_url)
-      unless images.include?(image_url) || image_url.to_s.empty?
-        self.images << image_url
-      end
-    end
-
-    def fetch_images(doc, xpath_str, attr)
-      doc.xpath(xpath_str).each do |link|
-        add_image(link.attribute(attr).to_s.strip)
-      end
-    end
-
-    def fetch_first_text(doc)
-      doc.xpath('//p').each do |p|
-        s = p.text.to_s.strip
-        return s if s.length > 20
-      end
-    end
-
-    def add_metadata(metadata_container, path, content)
-      path_elements = path.split(':')
-      if path_elements.size > 1
-        current_element = path_elements.delete_at(0)
-        path = path_elements.join(':')
-        if metadata_container[current_element.to_sym]
-          path_pointer = metadata_container[current_element.to_sym].last
-          index_count = metadata_container[current_element.to_sym].size
-          metadata_container[current_element.to_sym][index_count - 1] = add_metadata(path_pointer, path, content)
-          metadata_container
-        else
-          metadata_container[current_element.to_sym] = []
-          metadata_container[current_element.to_sym] << add_metadata({}, path, content)
-          metadata_container
-        end
-      else
-        metadata_container[path.to_sym] ||= []
-        metadata_container[path.to_sym] << {'_value'.to_sym => content}
-        metadata_container
-      end
-    end
-
-    def parse_html
-      unless charset
-        doc = Nokogiri.parse(body.scrub)
-        self.charset = guess_encoding(doc)
-      end
-      Nokogiri::HTML(body, nil, charset)
-    end
-
-    def guess_encoding(doc)
-      _charset = doc.xpath('//meta/@charset').first
-      return _charset.value.to_s if charset
-
-      _charset = doc.xpath('//meta').each do |m|
-        if m.attribute('http-equiv') && m.attribute('content') && m.attribute('http-equiv').value.casecmp('Content-Type')
-          return m.attribute('content').value.split('charset=').last.strip
-        end
-      end
-
-      'UTF-8'
-    end
-
-  end
-end