true_url 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 496c04ece239c5302c9548b0a4b7d93630125172
+  data.tar.gz: 74c68ce403f401579bbc9b354b283595b5516302
+SHA512:
+  metadata.gz: cfbf6e4099c911e66ff6061ecd98b0e98c60b8411659904d57d76c9bf1b4a2da207f0a19b9f11597ff8bf22a8ae716a7d49706b0a46730b6e824c99e970542cd
+  data.tar.gz: c186b1d04c781de99d61d64b9b3052e2f667ba140cba10c05db5e51ab0b43beddc0ee20d2318a534b84ed7ac8019294949a4fd18bcfe50a4492230431e13db6a

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+build.bat

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format doc
+--require spec_helper

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in true_url.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Jonathan Wong
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1 @@
+# true_url

data/Rakefile

@@ -0,0 +1,5 @@
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new :spec
+
+task default: :spec

data/lib/true_url/context.rb

@@ -0,0 +1,39 @@
+class TrueURL
+  class Context
+    attr_reader :original_url, :options, :attributes, :working_url
+
+    def initialize(original_url, options)
+      @original_url = parse(original_url)
+      @options = options
+      @finalized = false
+      @attributes = {}
+
+      set_working_url(original_url)
+    end
+
+    def set_working_url(url, base_url = nil)
+      @working_url = base_url.nil? ? parse(url) : parse(base_url).join(parse(url))
+
+      # If the URL has no scheme, then we assume HTTP
+      if @working_url.scheme.nil?
+        @working_url = url.to_s.start_with?('//') ? parse("http:#{url}") : parse("http://#{url}")
+      end
+
+      @working_url.normalize
+    end
+
+    def finalize
+      @finalized = true
+    end
+
+    def finalized?
+      @finalized
+    end
+
+    private
+
+    def parse(url)
+      (url.is_a? Addressable::URI) ? url : Addressable::URI.parse(url)
+    end
+  end
+end

data/lib/true_url/strategy/dailymotion.rb

@@ -0,0 +1,56 @@
+class TrueURL
+  module Strategy
+    class DailyMotion
+      def execute(context)
+        path = context.working_url.path
+
+        if context.working_url.host == 'dai.ly'
+          video_id = path[1..-1]
+
+        elsif path[0..6] == '/video/'
+          video_id = clean_video_id(path)
+
+        elsif path[0..6] == '/embed/'
+          video_id = path[13..-1]
+
+        elsif path[0..9] == '/playlist/'
+          playlist_id = clean_playlist_id(path)
+        end
+
+        unless video_id.nil?
+          context.set_working_url("https://www.dailymotion.com/video/#{video_id}")
+          context.finalize
+          context.attributes[:embed_url] = "https://www.dailymotion.com/embed/video/#{video_id}"
+        end
+
+        unless playlist_id.nil?
+          context.set_working_url("https://www.dailymotion.com/playlist/#{playlist_id}")
+          context.finalize
+        end
+
+        # DailyMotion supports both HTTP and HTTPS and doesn't redirect between them, so we prefer HTTPS
+        context.working_url.scheme = 'https'
+      end
+
+      def clean_video_id(path)
+        if path.index('_')
+          path[7..path.index('_') - 1]
+        else
+          path[7..-1]
+        end
+      end
+
+      def clean_playlist_id(path)
+        cpath = path[10..-1]
+
+        if cpath.index('_')
+          cpath[0..cpath.index('_') - 1]
+        elsif cpath.index('/')
+          cpath[0..cpath.index('/') - 1]
+        else
+          cpath[0..-1]
+        end
+      end
+    end
+  end
+end

data/lib/true_url/strategy/nicovideo.rb

@@ -0,0 +1,25 @@
+class TrueURL
+  module Strategy
+    class NicoVideo
+      def execute(context)
+        path = context.working_url.path
+
+        if path[0..6] == '/watch/'
+          video_id = path.split('/')[2]
+
+        elsif path[0..12] == '/thumb_watch/'
+          video_id = path.split('/')[2]
+        end
+
+        if video_id
+          context.set_working_url("http://www.nicovideo.jp/watch/#{video_id}")
+          context.finalize
+          context.attributes[:embed_url] = "http://embed.nicovideo.jp/watch/#{video_id}"
+        end
+
+        # Nico Video only supports HTTP
+        context.working_url.scheme = 'http'
+      end
+    end
+  end
+end

data/lib/true_url/strategy/twitter.rb

@@ -0,0 +1,25 @@
+class TrueURL
+  module Strategy
+    class Twitter
+      def execute(context)
+        fragment = context.working_url.fragment
+
+        # Special handling to collapse Twitter hashbang (#!) URLs
+        unless fragment.nil?
+          if fragment.start_with?('!/')
+            context.working_url.path = fragment[1..-1]
+            context.working_url.fragment = nil
+          end
+        end
+
+        path = context.working_url.path
+
+        if path =~ /^\/\w+\/status\/\d+/
+          parts = path.split('/')
+          context.set_working_url("https://twitter.com/#{parts[1].downcase}/status/#{parts[3]}")
+          context.finalize
+        end
+      end
+    end
+  end
+end

data/lib/true_url/strategy/vimeo.rb

@@ -0,0 +1,28 @@
+class TrueURL
+  module Strategy
+    class Vimeo
+      def execute(context)
+        path = context.working_url.path
+
+        if context.working_url.host == 'player.vimeo.com'
+          video_id = path[7..-1]
+
+        elsif path =~ /^\/channels\/\w+\/\d+$/
+          video_id = path.split('/').last
+
+        elsif path =~ /^\/\d+$/
+          video_id = path[1..-1]
+        end
+
+        if video_id
+          context.set_working_url("https://vimeo.com/#{video_id}")
+          context.finalize
+          context.attributes[:embed_url] = "https://player.vimeo.com/video/#{video_id}"
+        end
+
+        # Vimeo supports both HTTP and HTTPS and doesn't redirect between them, so we prefer HTTPS
+        context.working_url.scheme = 'https'
+      end
+    end
+  end
+end

data/lib/true_url/strategy/youtube.rb

@@ -0,0 +1,39 @@
+class TrueURL
+  module Strategy
+    class YouTube
+      def execute(context)
+        path = context.working_url.path
+
+        if context.working_url.host == 'youtu.be'
+          video_id = path[1..-1]
+
+        elsif path == '/watch'
+          video_id = context.working_url.query_values['v']
+
+        elsif path == '/playlist'
+          playlist_id = context.working_url.query_values['list']
+
+        elsif path[0..17] == '/embed/videoseries'
+          playlist_id = context.working_url.query_values['list']
+
+        elsif path[0..6] == '/embed/'
+          video_id = path[7..-1]
+        end
+
+        unless video_id.nil?
+          context.set_working_url("https://www.youtube.com/watch?v=#{video_id}")
+          context.finalize
+          context.attributes[:embed_url] = "https://www.youtube.com/embed/#{video_id}"
+          context.attributes[:embed_url_private] = "https://www.youtube-nocookie.com/embed/#{video_id}"
+        end
+
+        unless playlist_id.nil?
+          context.set_working_url("https://www.youtube.com/playlist?list=#{playlist_id}")
+          context.finalize
+          context.attributes[:embed_url] = "https://www.youtube.com/embed/videoseries?list=#{playlist_id}"
+          context.attributes[:embed_url_private] = "https://www.youtube-nocookie.com/embed/videoseries?list=#{playlist_id}"
+        end
+      end
+    end
+  end
+end

data/lib/true_url/strategy.rb

@@ -0,0 +1,23 @@
+require 'true_url/strategy/dailymotion'
+require 'true_url/strategy/nicovideo'
+require 'true_url/strategy/twitter'
+require 'true_url/strategy/vimeo'
+require 'true_url/strategy/youtube'
+
+class TrueURL
+  module Strategy
+    def self.default_list
+      [
+        [/youtube.com$/, TrueURL::Strategy::YouTube.new],
+        [/youtube-nocookie.com$/, TrueURL::Strategy::YouTube.new],
+        [/^youtu.be$/, TrueURL::Strategy::YouTube.new],
+        [/dailymotion.com$/, TrueURL::Strategy::DailyMotion.new],
+        [/^dai.ly$/, TrueURL::Strategy::DailyMotion.new],
+        [/vimeo.com$/, TrueURL::Strategy::Vimeo.new],
+        [/nicovideo.jp$/, TrueURL::Strategy::NicoVideo.new],
+        [/^nico.ms$/, TrueURL::Strategy::NicoVideo.new],
+        [/twitter.com$/, TrueURL::Strategy::Twitter.new]
+      ]
+    end
+  end
+end

data/lib/true_url/version.rb

@@ -0,0 +1,3 @@
+class TrueURL
+  VERSION = '0.0.1'.freeze
+end

data/lib/true_url.rb

@@ -0,0 +1,140 @@
+require 'addressable/uri'
+
+class TrueURL
+  autoload :Version, 'true_url/version'
+  autoload :Context, 'true_url/context'
+  autoload :Strategy, 'true_url/strategy'
+
+  attr_accessor :context, :strategies
+
+  OPTIONS = {
+    scheme_override: nil, # Possible choices: "https", "http", nil (preserve scheme)
+    fetch: true # Whether to fetch the URL
+  }.freeze
+
+  QUERY_VALUES_TO_REMOVE = %w(
+    utm_source
+    utm_medium
+    utm_term
+    utm_content
+    utm_campaign
+    sms_ss
+    awesm
+    xtor
+    PHPSESSID
+  ).freeze
+
+  def initialize(url, options = {})
+    @context = TrueURL::Context.new(url, OPTIONS.merge(options))
+    @strategies = TrueURL::Strategy.default_list
+    @executed = false
+  end
+
+  def canonical
+    execute
+    @context.working_url.to_s
+  end
+
+  def attributes
+    execute
+    @context.attributes
+  end
+
+  private
+
+  def execute
+    return if @executed
+
+    execute_strategies
+
+    unless @context.finalized?
+      if attempt_fetch?
+        fetch
+        execute_strategies
+      end
+    end
+
+    scheme_override
+    remove_fragments
+    clean_query_values
+
+    @executed = true
+  end
+
+  def execute_strategies
+    @strategies.each do |s|
+      match_criteria = s[0]
+      strategy = s[1]
+
+      strategy.execute(@context) unless @context.finalized? || !strategy_match?(match_criteria)
+    end
+  end
+
+  def strategy_match?(match_criteria)
+    return true if match_criteria.nil?
+
+    host = @context.working_url.host
+    host.nil? ? false : host.match(match_criteria)
+  end
+
+  def attempt_fetch?
+    return false unless @context.options[:fetch]
+
+    # Must at least have a host, otherwise we can't find the site to crawl
+    return false if @context.working_url.host.nil?
+
+    # We only support HTTP or HTTPS
+    %w(http https).include?(@context.working_url.scheme)
+  end
+
+  def fetch
+    require 'http' unless defined? HTTP
+
+    starting_url = @context.working_url
+
+    response = HTTP.follow
+                   .get(starting_url)
+
+    canonical_url = find_canonical_header(response.headers) || find_canonical_url(response.to_s) || response.uri
+    @context.set_working_url(canonical_url, starting_url)
+  end
+
+  def find_canonical_header(headers)
+    return if headers['Link'].nil?
+
+    links = (headers['Link'].is_a? String) ? [headers['Link']] : headers['Link']
+    links.each { |link| return link.split(/[<>;]/)[1] if link.end_with?('rel="canonical"') }
+    nil
+  end
+
+  def find_canonical_url(html)
+    require 'nokogiri' unless defined? Nokogiri::HTML
+
+    doc = Nokogiri::HTML(html)
+
+    elem = doc.at('link[rel="canonical"]')
+    canonical_url = elem['href'] unless elem.nil?
+
+    elem = doc.at('meta[property="og:url"]')
+    og_url = elem['content'] unless elem.nil?
+
+    canonical_url || og_url
+  end
+
+  def scheme_override
+    @context.working_url.scheme = @context.options[:scheme_override] unless @context.options[:scheme_override].nil?
+  end
+
+  def remove_fragments
+    @context.working_url.fragment = nil
+  end
+
+  def clean_query_values
+    query_values = @context.working_url.query_values
+
+    unless query_values.nil?
+      QUERY_VALUES_TO_REMOVE.each { |p| query_values.delete(p) }
+      @context.working_url.query_values = query_values.empty? ? nil : query_values
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,96 @@
+require 'true_url'
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # The settings below are suggested to provide a good initial experience
+  # with RSpec, but feel free to customize to your heart's content.
+  #   # These two settings work together to allow you to limit a spec run
+  #   # to individual examples or groups you care about by tagging them with
+  #   # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  #   # get run.
+  #   config.filter_run :focus
+  #   config.run_all_when_everything_filtered = true
+  #
+  #   # Allows RSpec to persist some state between runs in order to support
+  #   # the `--only-failures` and `--next-failure` CLI options. We recommend
+  #   # you configure your source control system to ignore this file.
+  #   config.example_status_persistence_file_path = "spec/examples.txt"
+  #
+  #   # Limits the available syntax to the non-monkey patched syntax that is
+  #   # recommended. For more details, see:
+  #   #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  #   config.disable_monkey_patching!
+  #
+  #   # This setting enables warnings. It's recommended, but in some cases may
+  #   # be too noisy due to issues in dependencies.
+  #   config.warnings = true
+  #
+  #   # Many RSpec users commonly either run the entire suite or an individual
+  #   # file, and it's useful to allow more verbose output when running an
+  #   # individual spec file.
+  #   if config.files_to_run.one?
+  #     # Use the documentation formatter for detailed output,
+  #     # unless a formatter has already been configured
+  #     # (e.g. via a command-line flag).
+  #     config.default_formatter = 'doc'
+  #   end
+  #
+  #   # Print the 10 slowest examples and example groups at the
+  #   # end of the spec run, to help surface which specs are running
+  #   # particularly slow.
+  #   config.profile_examples = 10
+  #
+  #   # Run specs in random order to surface order dependencies. If you find an
+  #   # order dependency and want to debug it, you can fix the order by providing
+  #   # the seed, which is printed after each run.
+  #   #     --seed 1234
+  #   config.order = :random
+  #
+  #   # Seed global randomization in this process using the `--seed` CLI option.
+  #   # Setting this allows you to use `--seed` to deterministically reproduce
+  #   # test failures related to randomization by passing the same `--seed` value
+  #   # as the one that triggered the failure.
+  #   Kernel.srand config.seed
+end

data/spec/true_url_spec.rb

@@ -0,0 +1,204 @@
+require 'spec_helper'
+
+def gc(unclean_url, options = {})
+  TrueURL.new(unclean_url, options).canonical
+end
+
+describe TrueURL do
+  describe 'YouTube' do
+    it 'supports direct video links' do
+      t = 'https://www.youtube.com/watch?v=RDocnbkHjhI'
+      expect(gc('https://www.youtube.com/watch?v=RDocnbkHjhI')).to eq t
+      expect(gc('https://www.youtube.com/watch?v=RDocnbkHjhI&feature=youtu.be&list=PLs4hTtftqnlAkiQNdWn6bbKUr-P1wuSm0')).to eq t
+      expect(gc('https://youtu.be/RDocnbkHjhI?list=PLs4hTtftqnlAkiQNdWn6bbKUr-P1wuSm0')).to eq t
+    end
+
+    it 'supports embedded video links' do
+      t = 'https://www.youtube.com/watch?v=RDocnbkHjhI'
+      expect(gc('https://www.youtube.com/embed/RDocnbkHjhI?list=PLs4hTtftqnlAkiQNdWn6bbKUr-P1wuSm0')).to eq t
+      expect(gc('https://www.youtube-nocookie.com/embed/RDocnbkHjhI?list=PLs4hTtftqnlAkiQNdWn6bbKUr-P1wuSm0&controls=0&showinfo=0')).to eq t
+    end
+
+    it 'supports direct playlist links' do
+      t = 'https://www.youtube.com/playlist?list=PLVL8S3lUHf0RqD7TZ6hohWk8Sd3asaqnY'
+      expect(gc('https://www.youtube.com/playlist?list=PLVL8S3lUHf0RqD7TZ6hohWk8Sd3asaqnY')).to eq t
+      expect(gc('https://www.youtube.com/playlist?list=PLVL8S3lUHf0RqD7TZ6hohWk8Sd3asaqnY')).to eq t
+    end
+
+    it 'supports embedded playlist links' do
+      t = 'https://www.youtube.com/playlist?list=PLVL8S3lUHf0RqD7TZ6hohWk8Sd3asaqnY'
+      expect(gc('https://www.youtube.com/embed/videoseries?list=PLVL8S3lUHf0RqD7TZ6hohWk8Sd3asaqnY')).to eq t
+      expect(gc('https://www.youtube-nocookie.com/embed/videoseries?list=PLVL8S3lUHf0RqD7TZ6hohWk8Sd3asaqnY')).to eq t
+    end
+
+    it 'supports direct channel links' do
+      t = 'https://www.youtube.com/user/WatchMojo'
+      expect(gc('https://www.youtube.com/channel/UCaWd5_7JhbQBe4dknZhsHJg')).to eq t
+    end
+
+    it 'supports retrieving embed links as attributes' do
+      x = TrueURL.new('https://www.youtube-nocookie.com/embed/videoseries?list=PLVL8S3lUHf0RqD7TZ6hohWk8Sd3asaqnY')
+      expect(x.attributes[:embed_url]).to eq 'https://www.youtube.com/embed/videoseries?list=PLVL8S3lUHf0RqD7TZ6hohWk8Sd3asaqnY'
+
+      x = TrueURL.new('https://www.youtube.com/embed/RDocnbkHjhI?list=PLs4hTtftqnlAkiQNdWn6bbKUr-P1wuSm0&controls=0&showinfo=0')
+      expect(x.attributes[:embed_url_private]).to eq 'https://www.youtube-nocookie.com/embed/RDocnbkHjhI'
+    end
+  end
+
+  describe 'DailyMotion' do
+    it 'supports direct video links' do
+      t = 'https://www.dailymotion.com/video/x2k01a9'
+      expect(gc('http://dai.ly/x2k01a9')).to eq t
+      expect(gc('http://www.dailymotion.com/video/x2k01a9_battlefield-what-s-it-like-to-be-in-a-real-life-video-game_fun')).to eq t
+    end
+
+    it 'supports embedded video links' do
+      t = 'https://www.dailymotion.com/video/x2k01a9'
+      expect(gc('http://www.dailymotion.com/embed/video/x2k01a9?autoPlay=1&start=40')).to eq t
+    end
+
+    it 'supports direct playlist links' do
+      t = 'https://www.dailymotion.com/playlist/x1ybux'
+      expect(gc('https://www.dailymotion.com/playlist/x1ybux/1#video=xlbw3e')).to eq t
+      expect(gc('https://www.dailymotion.com/playlist/x1ybux')).to eq t
+      expect(gc('http://www.dailymotion.com/playlist/x1ybux_ODNandfinally_amazing-world-records/1#video=xlbw3e')).to eq t
+    end
+
+    it 'supports retrieving embed links as attributes' do
+      x = TrueURL.new('http://www.dailymotion.com/video/x2k01a9_battlefield-what-s-it-like-to-be-in-a-real-life-video-game_fun')
+      expect(x.attributes[:embed_url]).to eq 'https://www.dailymotion.com/embed/video/x2k01a9'
+    end
+
+    it 'supports force HTTPS' do
+      t = 'https://www.dailymotion.com/ODNandfinally'
+      expect(gc('http://www.dailymotion.com/ODNandfinally')).to eq t
+    end
+  end
+
+  describe 'Vimeo' do
+    it 'supports direct video links' do
+      t = 'https://vimeo.com/122258599'
+      expect(gc('https://vimeo.com/channels/staffpicks/122258599')).to eq t
+      expect(gc('http://vimeo.com/122258599')).to eq t
+    end
+
+    it 'supports embedded video links' do
+      t = 'https://vimeo.com/122258599'
+      expect(gc('https://player.vimeo.com/video/122258599?loop=1&color=c9ff23&title=0')).to eq t
+    end
+
+    it "supports Vimeo's relative canonical links" do
+      t = 'https://vimeo.com/channels/staffpicks'
+      expect(gc('http://vimeo.com/channels/staffpicks?some=silly&params=here')).to eq t
+    end
+
+    it 'supports retrieving embed links as attributes' do
+      x = TrueURL.new('https://vimeo.com/channels/staffpicks/122258599')
+      expect(x.attributes[:embed_url]).to eq 'https://player.vimeo.com/video/122258599'
+    end
+
+    it 'supports force HTTPS' do
+      t = 'https://vimeo.com/user3190002'
+      expect(gc('http://vimeo.com/user3190002')).to eq t
+    end
+  end
+
+  describe 'Nico Nico Douga' do
+    it 'should work with direct and embedded video links' do
+      t = 'http://www.nicovideo.jp/watch/sm25956031'
+      expect(gc('http://ext.nicovideo.jp/thumb_watch/sm25956031?w=490&h=307')).to eq t
+      expect(gc('http://embed.nicovideo.jp/watch/sm25956031/script?w=490&h=307&redirect=1')).to eq t
+      expect(gc('http://embed.nicovideo.jp/watch/sm25956031?oldScript=1')).to eq t
+    end
+
+    it 'supports retrieving embed links as attributes' do
+      x = TrueURL.new('http://ext.nicovideo.jp/thumb_watch/sm25956031?w=490&h=307')
+      expect(x.attributes[:embed_url]).to eq 'http://embed.nicovideo.jp/watch/sm25956031'
+    end
+  end
+
+  describe 'Twitter' do
+    it 'supports direct links to tweets' do
+      t = 'https://twitter.com/gangsta_project/status/578483098284748801'
+      expect(gc('https://twitter.com/GANGSTA_Project/status/578483098284748801/photo/1')).to eq t
+      expect(gc('https://twitter.com/GANGSTA_Project/status/578483098284748801/')).to eq t
+    end
+
+    it 'should handle hashbangs' do
+      t = 'https://twitter.com/gangsta_project/status/578483098284748801'
+      expect(gc('https://twitter.com/#!/GANGSTA_Project/status/578483098284748801/')).to eq t
+    end
+  end
+
+  describe 'URL Shorteners' do
+    it 'should work with t.co' do
+      t = 'http://www.prdaily.com/Main/Articles/3_essential_skills_for_todays_PR_pro__18404.aspx'
+      expect(gc('http://t.co/fvaGuRa5Za')).to eq t
+      expect(gc('https://t.co/fvaGuRa5Za')).to eq t
+    end
+
+    it 'should work with fb.me' do
+      t = 'https://www.facebook.com/aksuperdance/posts/1388968827814771'
+      expect(gc('http://fb.me/8qm5kW89k')).to eq t
+    end
+
+    it 'should work with ift.tt' do
+      t = 'http://tedxtaipei.com/articles/the_best_kindergarten_you_have_ever_seen/'
+      expect(gc('http://ift.tt/2iCbPy8')).to eq t
+    end
+
+    it 'should work with compounded URL shorteners' do
+      t = 'https://www.youtube.com/watch?v=jLhjsPjR-xk'
+      expect(gc('https://t.co/g4NYtZE3lW')).to eq t # http://bit.ly/2iCKic3 --> http://youtu.be/jLhjsPjR-xk?list=PLVL8S3lUHf0RqD7TZ6hohWk8Sd3asaqnY
+    end
+  end
+
+  describe 'WordPress' do
+    it 'supports missing trailing slashes' do
+      t = 'http://wowjapan.asia/2015/04/anime-gargantia-on-the-verdurous-planet-2nd-season-cancelled/'
+      expect(gc('http://wowjapan.asia/2015/04/anime-gargantia-on-the-verdurous-planet-2nd-season-cancelled')).to eq t
+    end
+  end
+
+  describe 'Blogger' do
+    it 'supports missing localized Blogger domains' do
+      t = 'http://thevikiblog.blogspot.com/2015/12/soompi-ios-android.html'
+      expect(gc('http://thevikiblog.blogspot.sg/2015/12/soompi-ios-android.html')).to eq t
+    end
+  end
+
+  describe 'Other Scenarios' do
+    it 'supports missing schemes' do
+      t = 'http://wowjapan.asia/2015/04/anime-gargantia-on-the-verdurous-planet-2nd-season-cancelled/'
+      expect(gc('//wowjapan.asia/2015/04/anime-gargantia-on-the-verdurous-planet-2nd-season-cancelled/')).to eq t
+    end
+
+    it 'supports scheme override' do
+      t = 'https://wowjapan.asia/2015/04/anime-gargantia-on-the-verdurous-planet-2nd-season-cancelled/'
+      expect(gc('//wowjapan.asia/2015/04/anime-gargantia-on-the-verdurous-planet-2nd-season-cancelled/', scheme_override: 'https')).to eq t
+    end
+
+    it 'supports CDJapan' do
+      t = 'http://www.cdjapan.co.jp/product/MDR-1012'
+      expect(gc('http://www.cdjapan.co.jp/aff/click.cgi/e86NDzbdSLQ/4323/A323439/detailview.html?KEY=MDR-1012')).to eq t
+    end
+
+    it 'supports MyAnimeList' do
+      t = 'https://myanimelist.net/forum/?topicid=1371295'
+      expect(gc('https://myanimelist.net/forum?topicid=1371295&goto=newpost')).to eq t
+    end
+
+    it 'supports URLs with escapable characters' do
+      t = 'http://goboiano.com/news/2568-attack-on-titan%2527s-first-live-action-trailer-finally-launches'
+      expect(gc("http://media.goboiano.com/news/2568-attack-on-titan's-first-live-action-trailer-finally-launches")).to eq t
+
+      t = 'http://randomc.net/image/Kekkai%20Sensen/Kekkai%20Sensen%20-%2001%20-%20Large%2001.jpg'
+      expect(gc('http://randomc.net/image/Kekkai Sensen/Kekkai Sensen - 01 - Large 01.jpg')).to eq t
+    end
+
+    it 'supports canonical HTTP headers' do
+      t = 'http://www.seoreviewtools.com/canonical-url-location-checker/'
+      expect(gc('http://www.seoreviewtools.com/tests/canonical-header.php')).to eq t
+    end
+  end
+end

data/true_url.gemspec

@@ -0,0 +1,22 @@
+require File.expand_path('../lib/true_url/version', __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'true_url'
+  spec.version       = TrueURL::VERSION
+  spec.authors       = ['Jonathan Wong']
+  spec.email         = ['jonathan@armchairtheorist.com']
+  spec.summary       = 'A multi-strategy approach to find the absolutely cleanest and most likely canonical URL of any given URL.'
+  spec.homepage      = 'http://github.com/armchairtheorist/true_url'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split("\n")
+  spec.test_files    = `git ls-files -- {spec}/*`.split("\n")
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'rspec', '~> 0'
+  spec.add_development_dependency 'rake', '~> 0'
+  spec.add_development_dependency 'http', '~> 2.1', ">= 2.1.0"
+  spec.add_development_dependency 'nokogiri', '~> 1.6', ">= 1.6.8"
+
+  spec.add_runtime_dependency 'addressable', '~> 2.4', ">= 2.4.0"
+end

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification
+name: true_url
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jonathan Wong
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-12-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.4.0
+description: 
+email:
+- jonathan@armchairtheorist.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/true_url.rb
+- lib/true_url/context.rb
+- lib/true_url/strategy.rb
+- lib/true_url/strategy/dailymotion.rb
+- lib/true_url/strategy/nicovideo.rb
+- lib/true_url/strategy/twitter.rb
+- lib/true_url/strategy/vimeo.rb
+- lib/true_url/strategy/youtube.rb
+- lib/true_url/version.rb
+- spec/spec_helper.rb
+- spec/true_url_spec.rb
+- true_url.gemspec
+homepage: http://github.com/armchairtheorist/true_url
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A multi-strategy approach to find the absolutely cleanest and most likely
+  canonical URL of any given URL.
+test_files: []