richurls 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e1fcf6a779cb196b1b72fe63156af98b340fc8be86b2b78a4e233a479c47b9d
-  data.tar.gz: 4dd73e4c5c52b1bdcbba13d612158f2827d8d770a67b0ac7229d3a331032b1a5
+  metadata.gz: 42cff335ffa60073310cdec6c1b16f2a1f1a4cfce0bde60835a76538a0f138aa
+  data.tar.gz: cdded0800498b21f24b5c89910da73fd5150492c4f9c7ccedcb5499d538b1423
 SHA512:
-  metadata.gz: bf8186d4e4dc41447c9bc3a0adad8575f8893eebd64702703371f0ad612e9eab2ffd5eb06161d7c8248a0c21e25cddc2b48af3d13527f5a381bf08c5ba427f29
-  data.tar.gz: 8cf6e808d2df2f97b9912d67b79396c9f11e541d94cb6f48e015028c630e2991003757c1aacf9036656befd2a1d31509fdc55fbce494d2991a3bef8211dc18cf
+  metadata.gz: a5633b6852996d4938f002e2e1e4a90e1b25037ace20ba5da9303abc4dc073efa9ba5819134080d6c467dbe905733226cd66554e99edcd9ab57fbfd49bdc4a57
+  data.tar.gz: '08bce91b2a1c2a301b3eb288883621f7f7d607e56e36765ee00d5d09a84348f281cff0e48f61e074c9c394d5478b9c7140ca9a4f02eb37778e129c3ec8ed6277'

data/.rubocop.yml

@@ -27,6 +27,12 @@ Style/FrozenStringLiteralComment:
   Enabled: false
 Style/GuardClause:
   Enabled: false
+Style/HashEachMethods:
+  Enabled: true
+Style/HashTransformKeys:
+  Enabled: true
+Style/HashTransformValues:
+  Enabled: true
 Naming/MemoizedInstanceVariableName:
   Enabled: false
 Style/RegexpLiteral:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    richurls (0.3.0)
+    richurls (0.4.0)
       oj (~> 3)
       ox (~> 2)
       patron (~> 0.13)

data/lib/body_decorator.rb

@@ -2,26 +2,30 @@ require 'ox'
 
 require_relative 'xml_handler'
 require_relative 'url_helper'
-require_relative 'parsers/title_parser'
-require_relative 'parsers/description_parser'
-require_relative 'parsers/image_parser'
+require_relative 'parsers/property'
+require_relative 'parsers/url'
 require_relative 'parsers/embed_parser'
 require_relative 'parsers/provider_display_parser'
-require_relative 'parsers/favicon_parser'
 
 module RichUrls
   class BodyDecorator
     NoXMLError = Class.new(StandardError)
 
     PARSERS = {
-      'title' => Parsers::TitleParser,
-      'description' => Parsers::DescriptionParser,
-      'image' => Parsers::ImageParser,
+      'title' => Parsers::Property,
+      'description' => Parsers::Property,
+      'image' => Parsers::Url,
+      'favicon' => Parsers::Url,
       'provider_display' => Parsers::ProviderDisplayParser,
-      'favicon' => Parsers::FaviconParser,
       'embed' => Parsers::EmbedParser
     }.freeze
 
+    def self.decorate(url, body)
+      new(url, body).decorate
+    end
+
+    private_class_method :new
+
     def initialize(url, body)
       @url = url
       @xml = XMLHandler.new
@@ -37,7 +41,7 @@ module RichUrls
 
     def decorate
       PARSERS.each_with_object({}) do |(key, parser), object|
-        object[key] = parser.call(@xml, @url)&.force_encoding('UTF-8')
+        object[key] = parser.call(@xml.properties[key], @url)
       end
     end
   end

data/lib/el.rb

@@ -0,0 +1,41 @@
+module RichUrls
+  class El
+    MAX_TEXT_LENGTH = 1000
+
+    attr_reader :tag, :open, :attributes
+
+    def initialize(tag)
+      @tag = tag
+      @open = true
+      @attributes = {}
+    end
+
+    def add(key, value)
+      return if @attributes[key]
+
+      @attributes[key] = value
+    end
+
+    def append_text(str)
+      @attributes[:text] ||= ''
+
+      str = str.strip
+      length = @attributes[:text].length
+
+      if length <= MAX_TEXT_LENGTH
+        end_slice = MAX_TEXT_LENGTH - length
+        sliced = str[0...end_slice]
+
+        @attributes[:text] << sliced + ' '
+      end
+    end
+
+    def text
+      @attributes[:text].strip
+    end
+
+    def close!
+      @open = false
+    end
+  end
+end

data/lib/finders/favicon.rb

@@ -0,0 +1,14 @@
+module RichUrls
+  module Finders
+    module Favicon
+      KEYWORDS = ['shortcut icon', 'icon shortcut', 'icon'].freeze
+
+      def self.call(elem)
+        return unless elem.tag == :link &&
+                      KEYWORDS.include?(elem.attributes[:rel])
+
+        elem.attributes[:href]
+      end
+    end
+  end
+end

data/lib/finders/meta_description.rb

@@ -0,0 +1,12 @@
+module RichUrls
+  module Finders
+    module MetaDescription
+      def self.call(elem)
+        return unless elem.tag == :meta &&
+                      elem.attributes[:property] == 'og:description'
+
+        elem.attributes[:content]
+      end
+    end
+  end
+end

data/lib/finders/meta_image.rb

@@ -0,0 +1,12 @@
+module RichUrls
+  module Finders
+    module MetaImage
+      def self.call(elem)
+        return unless elem.tag == :meta &&
+                      elem.attributes[:property] == 'og:image'
+
+        elem.attributes[:content]
+      end
+    end
+  end
+end

data/lib/finders/meta_title.rb

@@ -0,0 +1,12 @@
+module RichUrls
+  module Finders
+    module MetaTitle
+      def self.call(elem)
+        return unless elem.tag == :meta &&
+                      elem.attributes[:property] == 'og:title'
+
+        elem.attributes[:content]
+      end
+    end
+  end
+end

data/lib/finders.rb

@@ -0,0 +1,7 @@
+module RichUrls
+  module Finders
+    Description = ->(elem) { elem.tag == :p && elem.text }
+    Image = ->(elem) { elem.tag == :img && elem.attributes[:src] }
+    Title = ->(elem) { elem.tag == :title && elem.text }
+  end
+end

data/lib/parsers/embed_parser.rb

@@ -1,28 +1,30 @@
-module Parsers
-  class EmbedParser
-    require_relative 'embed_parsers/base'
-    require_relative 'embed_parsers/youtube'
-    require_relative 'embed_parsers/youtube_short'
-    require_relative 'embed_parsers/paste'
-    require_relative 'embed_parsers/spotify'
+module RichUrls
+  module Parsers
+    class EmbedParser
+      require_relative 'embed_parsers/base'
+      require_relative 'embed_parsers/youtube'
+      require_relative 'embed_parsers/youtube_short'
+      require_relative 'embed_parsers/paste'
+      require_relative 'embed_parsers/spotify'
 
-    PARSERS = [
-      Youtube,
-      YoutubeShort,
-      Paste,
-      Spotify
-    ].freeze
+      PARSERS = [
+        Youtube,
+        YoutubeShort,
+        Paste,
+        Spotify
+      ].freeze
 
-    def self.call(_, url)
-      uri = URI(url)
+      def self.call(_, url)
+        uri = URI(url)
 
-      PARSERS.each do |parser|
-        embed_parser = parser.new(uri)
+        PARSERS.each do |parser|
+          embed_parser = parser.new(uri)
 
-        return embed_parser.parse if embed_parser.match?
-      end
+          return embed_parser.parse if embed_parser.match?
+        end
 
-      nil
+        nil
+      end
     end
   end
 end

data/lib/parsers/embed_parsers/base.rb

@@ -1,14 +1,16 @@
-module Parsers
-  class EmbedParser
-    class Base
-      def initialize(uri)
-        @uri = uri
-      end
+module RichUrls
+  module Parsers
+    class EmbedParser
+      class Base
+        def initialize(uri)
+          @uri = uri
+        end
 
-      private
+        private
 
-      def query
-        @query ||= Hash[URI.decode_www_form(@uri.query)]
+        def query
+          @query ||= Hash[URI.decode_www_form(@uri.query)]
+        end
       end
     end
   end

data/lib/parsers/embed_parsers/paste.rb

@@ -1,18 +1,20 @@
-module Parsers
-  class EmbedParser
-    class Paste < Base
-      IFRAME = '<iframe src="https://pasteapp.com/p/%s/embed?view=%s" '\
-               'width="480" height="480" scrolling="no" '\
-               'frameborder="0" allowfullscreen></iframe>'.freeze
+module RichUrls
+  module Parsers
+    class EmbedParser
+      class Paste < Base
+        IFRAME = '<iframe src="https://pasteapp.com/p/%s/embed?view=%s" '\
+                 'width="480" height="480" scrolling="no" '\
+                 'frameborder="0" allowfullscreen></iframe>'.freeze
 
-      def match?
-        @uri.host == 'pasteapp.com' && @uri.path =~ /\/p\/[a-zA-Z0-9]+/
-      end
+        def match?
+          @uri.host == 'pasteapp.com' && @uri.path =~ /\/p\/[a-zA-Z0-9]+/
+        end
 
-      def parse
-        path_id = @uri.path.sub(/\/p\//, '')
+        def parse
+          path_id = @uri.path.sub(/\/p\//, '')
 
-        IFRAME % [path_id, query.fetch('view')]
+          IFRAME % [path_id, query.fetch('view')]
+        end
       end
     end
   end

data/lib/parsers/embed_parsers/spotify.rb

@@ -1,30 +1,32 @@
-module Parsers
-  class EmbedParser
-    class Spotify < Base
-      IFRAME = '<iframe src="https://open.spotify.com/embed/%s/%s" '\
-               'width="300" height="380" frameborder="0" '\
-               'allowtransparency="true" allow="encrypted-media">'\
-               '</iframe>'.freeze
+module RichUrls
+  module Parsers
+    class EmbedParser
+      class Spotify < Base
+        IFRAME = '<iframe src="https://open.spotify.com/embed/%s/%s" '\
+                 'width="300" height="380" frameborder="0" '\
+                 'allowtransparency="true" allow="encrypted-media">'\
+                 '</iframe>'.freeze
 
-      SCOPES = %w[
-        album
-        track
-        playlist
-      ].freeze
+        SCOPES = %w[
+          album
+          track
+          playlist
+        ].freeze
 
-      def match?
-        valid_path = SCOPES.any? do |path|
-          @uri.path.start_with?("/#{path}")
-        end
+        def match?
+          valid_path = SCOPES.any? do |path|
+            @uri.path.start_with?("/#{path}")
+          end
 
-        @uri.host == 'open.spotify.com' && valid_path
-      end
+          @uri.host == 'open.spotify.com' && valid_path
+        end
 
-      def parse
-        path = @uri.path
-        path[0] = ''
+        def parse
+          path = @uri.path
+          path[0] = ''
 
-        IFRAME % path.split('/')
+          IFRAME % path.split('/')
+        end
       end
     end
   end

data/lib/parsers/embed_parsers/youtube.rb

@@ -1,20 +1,22 @@
-module Parsers
-  class EmbedParser
-    class Youtube < Base
-      IFRAME = '<iframe width="560" height="315" '\
-               'src="https://www.youtube.com/embed/%s" frameborder="0" '\
-               'allow="accelerometer; autoplay; encrypted-media; '\
-               'gyroscope; picture-in-picture" allowfullscreen>'\
-               '</iframe>'.freeze
+module RichUrls
+  module Parsers
+    class EmbedParser
+      class Youtube < Base
+        IFRAME = '<iframe width="560" height="315" '\
+                 'src="https://www.youtube.com/embed/%s" frameborder="0" '\
+                 'allow="accelerometer; autoplay; encrypted-media; '\
+                 'gyroscope; picture-in-picture" allowfullscreen>'\
+                 '</iframe>'.freeze
 
-      def match?
-        @uri.host == 'www.youtube.com' &&
-          @uri.path == '/watch' &&
-          query.key?('v')
-      end
+        def match?
+          @uri.host == 'www.youtube.com' &&
+            @uri.path == '/watch' &&
+            query.key?('v')
+        end
 
-      def parse
-        IFRAME % query.fetch('v')
+        def parse
+          IFRAME % query.fetch('v')
+        end
       end
     end
   end

data/lib/parsers/embed_parsers/youtube_short.rb

@@ -1,15 +1,17 @@
-module Parsers
-  class EmbedParser
-    class YoutubeShort < Base
-      def match?
-        @uri.host == 'youtu.be'
-      end
+module RichUrls
+  module Parsers
+    class EmbedParser
+      class YoutubeShort < Base
+        def match?
+          @uri.host == 'youtu.be'
+        end
 
-      def parse
-        path = @uri.path
-        path[0] = ''
+        def parse
+          path = @uri.path
+          path[0] = ''
 
-        Youtube::IFRAME % path
+          Youtube::IFRAME % path
+        end
       end
     end
   end

data/lib/parsers/property.rb

@@ -0,0 +1,5 @@
+module RichUrls
+  module Parsers
+    Property = ->(property, _) { property&.force_encoding('UTF-8') }
+  end
+end

data/lib/parsers/provider_display_parser.rb

@@ -1,3 +1,5 @@
-module Parsers
-  ProviderDisplayParser = ->(_, url) { URI(url).host }
+module RichUrls
+  module Parsers
+    ProviderDisplayParser = ->(_, url) { URI(url).host }
+  end
 end

data/lib/parsers/url.rb

@@ -0,0 +1,5 @@
+module RichUrls
+  module Parsers
+    Url = ->(content, url) { UrlHelper.url_for(url, content) }
+  end
+end

data/lib/url_fetcher.rb

@@ -38,7 +38,7 @@ module RichUrls
       response = session.get(@url)
 
       if response.status < 400
-        decorated = BodyDecorator.new(response.url, response.body).decorate
+        decorated = BodyDecorator.decorate(response.url, response.body)
         RichUrls.cache.set(digest, Oj.dump(decorated))
         decorated
       else

data/lib/xml_handler.rb

@@ -1,12 +1,18 @@
+require_relative 'el'
+require_relative 'finders'
+require_relative 'finders/meta_title'
+require_relative 'finders/meta_description'
+require_relative 'finders/meta_image'
+require_relative 'finders/favicon'
+
 module RichUrls
   class XMLHandler < ::Ox::Sax
     WHITELISTED_EL_NAMES = %i[
-      html
-      head
       title
       meta
       link
       img
+      p
     ].freeze
 
     WHITELISTED_ATTRS = %i[
@@ -17,48 +23,97 @@ module RichUrls
       src
     ].freeze
 
+    FALLBACK_ELEMENTS = {
+      img: 'og:image',
+      p: 'og:description',
+      title: 'og:title'
+    }.freeze
+
+    FINDERS = {
+      Finders::MetaTitle => 'title',
+      Finders::MetaDescription => 'description',
+      Finders::MetaImage => 'image',
+      Finders::Favicon => 'favicon',
+      Finders::Title => 'title',
+      Finders::Description => 'description',
+      Finders::Image => 'image'
+    }.freeze
+
     StopParsingError = Class.new(StandardError)
-    El = Struct.new(:name, :attributes)
 
-    attr_accessor :elements
+    attr_reader :elements, :properties
 
     def initialize
       @elements = []
+      @counts = Set.new
+      @properties = {
+        'title' => nil,
+        'description' => nil,
+        'image' => nil,
+        'favicon' => nil
+      }
     end
 
-    def find(name, attrs = {})
+    def find(tag, attrs = {})
       @elements.detect do |el|
         matching_attributes = attrs.all? { |k, v| el.attributes[k] == v }
 
-        el.name == name && matching_attributes
+        el.tag == tag && matching_attributes
       end
     end
 
-    def start_element(element_name)
-      return unless WHITELISTED_EL_NAMES.include?(element_name)
+    def start_element(tag)
+      return unless WHITELISTED_EL_NAMES.include?(tag)
 
-      @elements << El.new(element_name, {})
+      @elements << El.new(tag) if add_element?(tag)
     end
 
-    def attr(name, str)
-      el = @elements.last
+    def end_element(tag)
+      return unless WHITELISTED_EL_NAMES.include?(tag)
 
-      return unless el && WHITELISTED_ATTRS.include?(name)
+      el = @elements.reverse_each.detect { |e| e.open && e.tag == tag }
+      return unless el
 
-      el.attributes[name] = str
+      el.close!
+      find_element(el)
 
-      raise StopParsingError if stop?(name, el)
+      raise StopParsingError if @properties.values.all?
     end
 
-    def text(str)
+    def attr(key, value)
+      return unless WHITELISTED_ATTRS.include?(key)
+
       el = @elements.last
-      el && el.attributes[:text].nil? && el.attributes[:text] = str
+      el&.add(key, value)
+    end
+
+    def text(str)
+      el = @elements.detect(&:open)
+      el&.append_text(str)
     end
 
     private
 
-    def stop?(name, elem)
-      name == WHITELISTED_ATTRS.last && elem.name == :img
+    def find_element(elem)
+      FINDERS.each_pair do |finder, attribute|
+        next if @properties[attribute]
+
+        content = finder.call(elem)
+
+        if content
+          @properties[attribute] = content
+          break
+        end
+      end
+    end
+
+    def add_element?(tag)
+      return true unless FALLBACK_ELEMENTS.keys.include?(tag)
+      return false if @counts.include?(tag)
+
+      @counts.add(tag)
+
+      !find(:meta, property: FALLBACK_ELEMENTS.fetch(tag))
     end
   end
 end

data/richurls.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name = 'richurls'
-  spec.version = '0.3.0'
+  spec.version = '0.4.0'
   spec.authors = ['grdw']
   spec.email = ['gerard@wetransfer.com']
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: richurls
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - grdw
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-17 00:00:00.000000000 Z
+date: 2020-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oj
@@ -124,17 +124,21 @@ files:
 - README.md
 - lib/body_decorator.rb
 - lib/cache.rb
-- lib/parsers/description_parser.rb
+- lib/el.rb
+- lib/finders.rb
+- lib/finders/favicon.rb
+- lib/finders/meta_description.rb
+- lib/finders/meta_image.rb
+- lib/finders/meta_title.rb
 - lib/parsers/embed_parser.rb
 - lib/parsers/embed_parsers/base.rb
 - lib/parsers/embed_parsers/paste.rb
 - lib/parsers/embed_parsers/spotify.rb
 - lib/parsers/embed_parsers/youtube.rb
 - lib/parsers/embed_parsers/youtube_short.rb
-- lib/parsers/favicon_parser.rb
-- lib/parsers/image_parser.rb
+- lib/parsers/property.rb
 - lib/parsers/provider_display_parser.rb
-- lib/parsers/title_parser.rb
+- lib/parsers/url.rb
 - lib/richurls.rb
 - lib/url_fetcher.rb
 - lib/url_helper.rb

data/lib/parsers/description_parser.rb

@@ -1,7 +0,0 @@
-module Parsers
-  DescriptionParser = lambda do |document, _|
-    meta_el = document.find(:meta, property: 'og:description')
-
-    meta_el && meta_el.attributes[:content]
-  end
-end

data/lib/parsers/favicon_parser.rb

@@ -1,18 +0,0 @@
-module Parsers
-  module FaviconParser
-    KEYWORDS = ['shortcut icon', 'icon shortcut', 'icon'].freeze
-
-    def self.call(document, url)
-      KEYWORDS.each do |rel|
-        found_document = document.find(:link, rel: rel)
-
-        if found_document
-          @el = found_document
-          break
-        end
-      end
-
-      @el && UrlHelper.url_for(url, @el.attributes[:href])
-    end
-  end
-end

data/lib/parsers/image_parser.rb

@@ -1,14 +0,0 @@
-module Parsers
-  class ImageParser
-    def self.call(document, url)
-      meta_image = document.find(:meta, property: 'og:image')
-      image_tag = document.find(:img)
-
-      image_source =
-        (meta_image && meta_image.attributes[:content]) ||
-        (image_tag && image_tag.attributes[:src])
-
-      image_source && UrlHelper.url_for(url, image_source)
-    end
-  end
-end

data/lib/parsers/title_parser.rb

@@ -1,9 +0,0 @@
-module Parsers
-  TitleParser = lambda do |document, _|
-    meta_el = document.find(:meta, property: 'og:title')
-    title_el = document.find(:title)
-
-    meta_el && meta_el.attributes[:content] ||
-      title_el && title_el.attributes[:text]
-  end
-end