embiggen 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f799848928c0c97fed7139de5465eaf62b564c35
-  data.tar.gz: 6da8a48878868a89f5837b15c0cceca531eece18
+  metadata.gz: 0b3a9d107ca9bb152b4430ce5d756fbead2d3815
+  data.tar.gz: 17319b76182cb9a89851657088ee611b74d2c429
 SHA512:
-  metadata.gz: 185ef0720cd5c8a438aad74c4f15f68c68a5d37934ae829d777377330a57c2cfb070ac880945f5e85277a1359bad9070c45be273e567529377ce033a1e316acf
-  data.tar.gz: 84d6bc662878057e2dfe1f8a0e212947d41f53b4d0cd664810a47a2f96d7617de1957d79005f1cc3e700b74cfb270770f6da19cb492f279f7451a92303b1672d
+  metadata.gz: bd6f7205ef18c9b9154d26086b5ee83ec24088e03171c4b9c814a336d9385b9bcac4b3b992775db4b3189e7d6bab9b7991d1510f7f3d9f384ee5c4a7164eb3d4
+  data.tar.gz: c34a4493bfb2eabb949d8f62fcaf86c3742fc6c379a09423545a8feee51a245fd83b787c18ac2eaa16bd2e235630db9cce99c9f556f8ba195202b01f572b8603

data/LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015 Altmetric LLP
+Copyright (c) 2015-2016 Altmetric LLP
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -2,19 +2,19 @@
 
 A Ruby library to expand shortened URLs.
 
-**Current version:** 1.0.0  
+**Current version:** 1.1.0  
 **Supported Ruby versions:** 1.8.7, 1.9.2, 1.9.3, 2.0, 2.1, 2.2
 
 ## Installation
 
 ```
-gem install embiggen -v '~> 1.0'
+gem install embiggen -v '~> 1.1'
 ```
 
 Or, in your `Gemfile`:
 
 ```ruby
-gem 'embiggen', '~> 1.0'
+gem 'embiggen', '~> 1.1'
 ```
 
 ## Usage
@@ -64,9 +64,10 @@ end
 ## Shorteners
 
 Embiggen ships with a default list of URL shortening service domains (c.f.
-[Acknowledgements](#acknowledgements)) but as it is likely to be outdated and
-incomplete, you are strongly encouraged to supply your own via
-`Embiggen.configure`.
+[Acknowledgements](#acknowledgements)) in
+[`shorteners.txt`](https://github.com/altmetric/embiggen/blob/master/shorteners.txt)
+but as it is likely to be outdated and incomplete, you are strongly encouraged
+to supply your own via `Embiggen.configure`.
 
 The list of shorteners is an object that responds to `include?` and will be
 passed a URI. By default, Embiggen ships with a `ShortenerList` class which
@@ -205,12 +206,15 @@ Override the following settings:
 
 ## Acknowledgements
 
+* The extraction of `shorteners.txt` and performance improvements to the
+  shortener list were contributed by [Avner
+  Cohen](https://github.com/AvnerCohen);
 * The default list of shorteners comes from [LongURL.org's curated
   list](http://longurl.org/services).
 
 ## License
 
-Copyright © 2015 Altmetric LLP
+Copyright © 2015-2016 Altmetric LLP
 
 Distributed under the MIT License.
 

data/lib/embiggen/configuration.rb

@@ -18,52 +18,12 @@ module Embiggen
 
     # From http://longurl.org/services
     def self.shorteners
-      @shorteners ||= ShortenerList.new(%w(
-        0rz.tw 1link.in 1url.com 2.gp 2big.at 2tu.us 3.ly 307.to 4ms.me
-        4sq.com 4url.cc 6url.com 7.ly a.gg a.nf aa.cx abcurl.net ad.vu adf.ly
-        adjix.com afx.cc all.fuseurl.com alturl.com amzn.to ar.gy arst.ch
-        atu.ca azc.cc b23.ru b2l.me bacn.me bcool.bz binged.it bit.ly bizj.us
-        bloat.me bravo.ly bsa.ly budurl.com canurl.com chilp.it chzb.gr cl.lk
-        cl.ly clck.ru cli.gs cliccami.info clickthru.ca clop.in conta.cc
-        cort.as cot.ag crks.me ctvr.us cutt.us dai.ly decenturl.com dfl8.me
-        digbig.com digg.com disq.us dld.bz dlvr.it do.my doiop.com dopen.us
-        easyuri.com easyurl.net eepurl.com eweri.com fa.by fav.me fb.me
-        fbshare.me ff.im fff.to fire.to firsturl.de firsturl.net flic.kr
-        flq.us fly2.ws fon.gs freak.to fuseurl.com fuzzy.to fwd4.me fwib.net
-        g.ro.lt gizmo.do gl.am go.9nl.com go.ign.com go.usa.gov goo.gl
-        goshrink.com gurl.es hex.io hiderefer.com hmm.ph href.in hsblinks.com
-        htxt.it huff.to hulu.com hurl.me hurl.ws icanhaz.com idek.net ilix.in
-        is.gd its.my ix.lt j.mp jijr.com kl.am klck.me korta.nu krunchd.com
-        l9k.net lat.ms liip.to liltext.com linkbee.com linkbun.ch liurl.cn
-        ln-s.net ln-s.ru lnk.gd lnk.ms lnkd.in lnkurl.com lru.jp lt.tl lurl.no
-        macte.ch mash.to merky.de migre.me miniurl.com minurl.fr mke.me
-        moby.to moourl.com mrte.ch myloc.me myurl.in n.pr nbc.co nblo.gs nn.nf
-        not.my notlong.com nsfw.in nutshellurl.com nxy.in nyti.ms o-x.fr
-        oc1.us om.ly omf.gd omoikane.net on.cnn.com on.mktw.net onforb.es
-        orz.se ow.ly ping.fm pli.gs pnt.me politi.co post.ly pp.gg profile.to
-        ptiturl.com pub.vitrue.com qlnk.net qte.me qu.tc qy.fi r.im rb6.me
-        read.bi readthis.ca reallytinyurl.com redir.ec stub_redirects.ca
-        redirx.com retwt.me ri.ms rickroll.it riz.gd rt.nu ru.ly rubyurl.com
-        rurl.org rww.tw s4c.in s7y.us safe.mn sameurl.com sdut.us shar.es
-        shink.de shorl.com short.ie short.to shortlinks.co.uk shorturl.com
-        shout.to show.my shrinkify.com shrinkr.com shrt.fr shrt.st shrten.com
-        shrunkin.com simurl.com slate.me smallr.com smsh.me smurl.name sn.im
-        snipr.com snipurl.com snurl.com sp2.ro spedr.com srnk.net srs.li
-        starturl.com su.pr surl.co.uk surl.hu t.cn t.co t.lh.com ta.gd tbd.ly
-        tcrn.ch tgr.me tgr.ph tighturl.com tiniuri.com tiny.cc tiny.ly tiny.pl
-        tinylink.in tinyuri.ca tinyurl.com tk tl.gd tmi.me tnij.org tnw.to
-        tny.com to to.ly togoto.us totc.us toysr.us tpm.ly tr.im tra.kz
-        trunc.it twhub.com twirl.at twitclicks.com twitterurl.net
-        twitterurl.org twiturl.de twurl.cc twurl.nl u.mavrev.com u.nu u76.org
-        ub0.cc ulu.lu updating.me ur1.ca url.az url.co.uk url.ie url360.me
-        url4.eu urlborg.com urlbrief.com urlcover.com urlcut.com urlenco.de
-        urli.nl urls.im urlshorteningservicefortwitter.com urlx.ie urlzen.com
-        usat.ly use.my vb.ly vgn.am vl.am vm.lc w55.de wapo.st wapurl.co.uk
-        wipi.es wp.me x.vu xr.com xrl.in xrl.us xurl.es xurl.jp y.ahoo.it
-        yatuc.com ye.pe yep.it yfrog.com yhoo.it yiyd.com youtu.be yuarel.com
-        z0p.de zi.ma zi.mu zipmyurl.com zud.me zurl.ws zz.gd zzang.kr ›.ws
-        ✩.ws ✿.ws ❥.ws ➔.ws ➞.ws ➡.ws ➨.ws ➯.ws ➹.ws ➽.ws
-      ))
+      @shorteners ||= ShortenerList.new(shorteners_from_file)
+    end
+
+    def self.shorteners_from_file
+      file_path = File.expand_path('../../../shorteners.txt', __FILE__)
+      File.readlines(file_path).map(&:chomp)
     end
   end
 end

data/lib/embiggen/shortener_list.rb

@@ -9,17 +9,31 @@ module Embiggen
     attr_reader :domains
 
     def initialize(domains)
-      @domains = Set.new(domains)
+      @domains = Set.new(domains.map { |domain| host_pattern(domain) })
     end
 
     def include?(uri)
-      domains.any? { |domain| uri.host =~ /\b#{domain}\z/i }
+      domains.any? { |domain| uri.host =~ domain }
     end
 
     def +(other)
       self.class.new(domains + other)
     end
 
-    def_delegators :domains, :<<, :size, :delete, :empty?, :each
+    def <<(domain)
+      domains << host_pattern(domain)
+
+      self
+    end
+
+    def delete(domain)
+      domains.delete(host_pattern(domain))
+    end
+
+    def_delegators :domains, :size, :empty?, :each
+
+    def host_pattern(domain)
+      /\b#{domain}\z/i
+    end
   end
 end

data/shorteners.txt

@@ -0,0 +1,344 @@
+0rz.tw
+1link.in
+1url.com
+2.gp
+2big.at
+2tu.us
+3.ly
+307.to
+4ms.me
+4sq.com
+4url.cc
+6url.com
+7.ly
+a.gg
+a.nf
+aa.cx
+abcurl.net
+ad.vu
+adf.ly
+adfoc.us
+adjix.com
+afx.cc
+all.fuseurl.com
+alturl.com
+amzn.to
+ar.gy
+arst.ch
+atu.ca
+azc.cc
+b23.ru
+b2l.me
+bacn.me
+bc.vc
+bcool.bz
+binged.it
+bit.do
+bit.ly
+bizj.us
+bloat.me
+bravo.ly
+bsa.ly
+budurl.com
+canurl.com
+chilp.it
+chzb.gr
+cl.lk
+cl.ly
+clck.ru
+cli.gs
+cliccami.info
+clickthru.ca
+clop.in
+conta.cc
+cort.as
+cot.ag
+crks.me
+ctvr.us
+cutt.us
+dai.ly
+decenturl.com
+dfl8.me
+digbig.com
+digg.com
+disq.us
+dld.bz
+dlvr.it
+do.my
+doiop.com
+dopen.us
+easyuri.com
+easyurl.net
+eepurl.com
+eweri.com
+fa.by
+fav.me
+fb.me
+fbshare.me
+ff.im
+fff.to
+fire.to
+firsturl.de
+firsturl.net
+flic.kr
+flq.us
+fly2.ws
+fon.gs
+freak.to
+fur.ly
+fuseurl.com
+fuzzy.to
+fwd4.me
+fwib.net
+g.ro.lt
+gizmo.do
+gl.am
+go.9nl.com
+go.ign.com
+go.usa.gov
+goo.gl
+goshrink.com
+gurl.es
+hex.io
+hiderefer.com
+hmm.ph
+href.in
+hsblinks.com
+htxt.it
+huff.to
+hulu.com
+hurl.me
+hurl.ws
+icanhaz.com
+idek.net
+ilix.in
+is.gd
+its.my
+ix.lt
+j.mp
+jijr.com
+kl.am
+klck.me
+korta.nu
+krunchd.com
+l9k.net
+lat.ms
+liip.to
+liltext.com
+linkbee.com
+linkbun.ch
+liurl.cn
+ln-s.net
+ln-s.ru
+lnk.gd
+lnk.ms
+lnkd.in
+lnkurl.com
+lru.jp
+lt.tl
+lurl.no
+macte.ch
+mash.to
+merky.de
+migre.me
+miniurl.com
+minurl.fr
+mke.me
+moby.to
+moourl.com
+mrte.ch
+myloc.me
+myurl.in
+n.pr
+nbc.co
+nblo.gs
+nn.nf
+not.my
+notlong.com
+nsfw.in
+nutshellurl.com
+nxy.in
+nyti.ms
+o-x.fr
+oc1.us
+om.ly
+omf.gd
+omoikane.net
+on.cnn.com
+on.mktw.net
+onforb.es
+orz.se
+ow.ly
+ping.fm
+pli.gs
+pnt.me
+politi.co
+post.ly
+pp.gg
+profile.to
+ptiturl.com
+pub.vitrue.com
+qlnk.net
+qte.me
+qu.tc
+qy.fi
+r.im
+rb6.me
+read.bi
+readthis.ca
+reallytinyurl.com
+redir.ec
+redirx.com
+retwt.me
+ri.ms
+rickroll.it
+riz.gd
+rt.nu
+ru.ly
+rubyurl.com
+rurl.org
+rww.tw
+s4c.in
+s7y.us
+safe.mn
+sameurl.com
+sdut.us
+seomafia.net
+shar.es
+shink.de
+shorl.com
+short.ie
+short.to
+shortlinks.co.uk
+shorturl.com
+shout.to
+show.my
+shrinkify.com
+shrinkr.com
+shrt.fr
+shrt.st
+shrten.com
+shrunkin.com
+simurl.com
+slate.me
+smallr.com
+smsh.me
+smurl.name
+sn.im
+snipr.com
+snipurl.com
+snurl.com
+sp2.ro
+spedr.com
+srnk.net
+srs.li
+starturl.com
+su.pr
+surl.co.uk
+surl.hu
+t.cn
+t.co
+t.lh.com
+ta.gd
+tbd.ly
+tcrn.ch
+tgr.me
+tgr.ph
+tighturl.com
+tiniuri.com
+tiny.cc
+tiny.ly
+tiny.pl
+tinylink.in
+tinyuri.ca
+tinyurl.com
+tk
+tl.gd
+tmi.me
+tnij.org
+tnw.to
+tny.com
+to
+to.ly
+togoto.us
+totc.us
+toysr.us
+tpm.ly
+tr.im
+tra.kz
+trunc.it
+twhub.com
+twirl.at
+twitclicks.com
+twitterurl.net
+twitterurl.org
+twiturl.de
+twurl.cc
+twurl.nl
+u.mavrev.com
+u.nu
+u76.org
+ub0.cc
+ulu.lu
+updating.me
+ur1.ca
+url.az
+url.co.uk
+url.ie
+url360.me
+url4.eu
+urlborg.com
+urlbrief.com
+urlcover.com
+urlcut.com
+urlenco.de
+urli.nl
+urls.im
+urlshorteningservicefortwitter.com
+urlx.ie
+urlzen.com
+usat.ly
+use.my
+vb.ly
+vgn.am
+vl.am
+vm.lc
+w55.de
+wapo.st
+wapurl.co.uk
+wipi.es
+wp.me
+x.vu
+xr.com
+xrl.in
+xrl.us
+xurl.es
+xurl.jp
+y.ahoo.it
+yatuc.com
+ye.pe
+yep.it
+yfrog.com
+yhoo.it
+yiyd.com
+youtu.be
+yuarel.com
+z0p.de
+zi.ma
+zi.mu
+zipmyurl.com
+zud.me
+zurl.ws
+zz.gd
+zzang.kr
+›.ws
+✩.ws
+✿.ws
+❥.ws
+➔.ws
+➞.ws
+➡.ws
+➨.ws
+➯.ws
+➹.ws
+➽.ws

data/spec/embiggen/shortener_list_spec.rb

@@ -27,6 +27,12 @@ RSpec.describe Embiggen::ShortenerList do
 
       expect(list).to_not include(URI('http://www.altmetric.com'))
     end
+
+    it 'returns true if a URL host without a subdomain is on the whitelist' do
+      list = described_class.new(%w(bit.ly))
+
+      expect(list).to include(URI('http://www.bit.ly/foo'))
+    end
   end
 
   describe '#<<' do

data/spec/embiggen/uri_spec.rb

@@ -248,10 +248,27 @@ module Embiggen
         expect(uri).to be_shortened
       end
 
-      it 'returns false if the link is not shortened but uses a similar ' \
+      it 'returns false if link is not shortened but uses a similar ' \
          'domain' do
         uri = described_class.new('http://notbit.ly/1ciyUPh')
+        expect(uri).to_not be_shortened
+      end
+
+      it 'returns false if link is not shortened but uses a similar ' \
+         'domain without protocol' do
+        uri = described_class.new('notbit.ly/1ciyUPh')
+        expect(uri).to_not be_shortened
+      end
+
+      it 'returns false if link is not shortened but uses the same domain ' \
+         'with a different TLD' do
+        uri = described_class.new('http://nbc.com/1ciyUPh')
+        expect(uri).to_not be_shortened
+      end
 
+      it 'returns false if link is not shortened but uses the same domain ' \
+         'with a different TLD without protocol' do
+        uri = described_class.new('nbc.com/1ciyUPh')
         expect(uri).to_not be_shortened
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embiggen
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Paul Mucur
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-21 00:00:00.000000000 Z
+date: 2016-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -69,6 +69,7 @@ files:
 - lib/embiggen/http_client.rb
 - lib/embiggen/shortener_list.rb
 - lib/embiggen/uri.rb
+- shorteners.txt
 - spec/embiggen/configuration_spec.rb
 - spec/embiggen/shortener_list_spec.rb
 - spec/embiggen/uri_spec.rb
@@ -94,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: A library to expand shortened URLs