twitter-text 1.9.2 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50d926db2fdf1b07462b2fac4af8fbad3251808e
-  data.tar.gz: 816af9771e5940a210e72b6f922d0e8fad26b757
+  metadata.gz: 9bf8f0458f83224b2f818d0167deece43aa85875
+  data.tar.gz: dadc8f874379ed383cf7b66172b5eb65ec130906
 SHA512:
-  metadata.gz: 53c99aaa4071d936d041bc8931c55fe76dec979e972d0affec724541f6ab86b4344c320f9f397d234f605a9fdad5646522cb6671361da2a3ed0fc00a183ca5ac
-  data.tar.gz: 131eee6c27027fb12f105418b52a1f6aaac00acbe4b0b63a3cf97338ea626bd0f6157d7218743b859956b45fb55b06dae3f269fad97a610cff33a26ced9ba38c
+  metadata.gz: efadacd1f014d0b422dc513c548e3c7cf22df17a21c42c382160a54883045530355b4766d21bfd038d9dd20ef4fcf9c33767874e2e708a357d9c63ab8dfbd46b
+  data.tar.gz: 43d0537e6b2e0411a3929b89ce5977c91ec786ed7e409fa7b0b0ed16beb8fff425b7c012e1f2073bc279af3c1b9814b0d51b6bf10c5f05b59dbefeec2696e2ec

data/lib/twitter-text.rb

@@ -1,4 +1,4 @@
-major, minor, patch = RUBY_VERSION.split('.')
+major, minor, _patch = RUBY_VERSION.split('.')
 
 $RUBY_1_9 = if major.to_i == 1 && minor.to_i < 9
   # Ruby 1.8 KCODE check. Not needed on 1.9 and later.

data/lib/twitter-text/extractor.rb

@@ -188,7 +188,6 @@ module Twitter
     def extract_urls_with_indices(text, options = {:extract_url_without_protocol => true}) # :yields: url, start, end
       return [] unless text && (options[:extract_url_without_protocol] ? text.index(".") : text.index(":"))
       urls = []
-      position = 0
 
       text.to_s.scan(Twitter::Regex[:valid_url]) do |all, before, url, protocol, domain, port, path, query|
         valid_url_match_data = $~

data/lib/twitter-text/hit_highlighter.rb

@@ -75,8 +75,8 @@ module Twitter
         if chunk_cursor < chunk_chars.length
           result << chunk_chars[chunk_cursor..-1]
         end
-        (chunk_index+1).upto(chunks.length-1).each do |index|
-          result << (index.even? ? chunks[index] : "<#{chunks[index]}>")
+        (chunk_index+1).upto(chunks.length-1).each do |i|
+          result << (i.even? ? chunks[i] : "<#{chunks[i]}>")
         end
       end
 

data/lib/twitter-text/regex.rb

@@ -6,6 +6,8 @@ module Twitter
   # used throughout the <tt>Twitter</tt> classes. Special care has been taken to make
   # sure these reular expressions work with Tweets in all languages.
   class Regex
+    require 'yaml'
+
     REGEXEN = {} # :nodoc:
 
     def self.regex_range(from, to = nil) # :nodoc:
@@ -22,7 +24,14 @@ module Twitter
           [from].pack('U')
         end
       end
-   end
+    end
+
+    TLDS = YAML.load_file(
+      File.join(
+        File.expand_path('../../..', __FILE__), # project root
+        'test', 'twitter-text-conformance', 'tld_lib.yml'
+      )
+    )
 
     # Space is more than %20, U+3000 for example is the full-width space used with Kanji. Provide a short-hand
     # to access both the list of characters and a pattern suitible for use with String#split
@@ -51,7 +60,7 @@ module Twitter
     ].map{|cp| [cp].pack('U') }.freeze
     REGEXEN[:invalid_control_characters] = /[#{INVALID_CHARACTERS.join('')}]/o
 
-    major, minor, patch = RUBY_VERSION.split('.')
+    major, minor, _patch = RUBY_VERSION.split('.')
     if major.to_i >= 2 || major.to_i == 1 && minor.to_i >= 9 || (defined?(RUBY_ENGINE) && ["jruby", "rbx"].include?(RUBY_ENGINE))
       REGEXEN[:list_name] = /[a-zA-Z][a-zA-Z0-9_\-\u0080-\u00ff]{0,24}/
     else
@@ -189,46 +198,15 @@ module Twitter
 
     REGEXEN[:valid_gTLD] = %r{
       (?:
-        (?:
-        academy|accountants|active|actor|aero|agency|airforce|archi|army|arpa|asia|associates|attorney|audio|autos|axa|
-        bar|bargains|bayern|beer|berlin|best|bid|bike|bio|biz|black|blackfriday|blue|bmw|boutique|brussels|build|
-        builders|buzz|bzh|cab|camera|camp|cancerresearch|capetown|capital|cards|care|career|careers|cash|cat|catering|
-        center|ceo|cheap|christmas|church|citic|claims|cleaning|clinic|clothing|club|codes|coffee|college|cologne|com|
-        community|company|computer|condos|construction|consulting|contractors|cooking|cool|coop|country|credit|
-        creditcard|cruises|cuisinella|dance|dating|degree|democrat|dental|dentist|desi|diamonds|digital|direct|
-        directory|discount|dnp|domains|durban|edu|education|email|engineer|engineering|enterprises|equipment|estate|eus|
-        events|exchange|expert|exposed|fail|farm|feedback|finance|financial|fish|fishing|fitness|flights|florist|foo|
-        foundation|frogans|fund|furniture|futbol|gal|gallery|gift|gives|glass|global|globo|gmo|gop|gov|graphics|gratis|
-        green|gripe|guide|guitars|guru|hamburg|haus|hiphop|hiv|holdings|holiday|homes|horse|host|house|immobilien|
-        industries|info|ink|institute|insure|int|international|investments|jetzt|jobs|joburg|juegos|kaufen|kim|kitchen|
-        kiwi|koeln|kred|land|lawyer|lease|lgbt|life|lighting|limited|limo|link|loans|london|lotto|luxe|luxury|maison|
-        management|mango|market|marketing|media|meet|menu|miami|mil|mini|mobi|moda|moe|monash|mortgage|moscow|
-        motorcycles|museum|nagoya|name|navy|net|neustar|nhk|ninja|nyc|okinawa|onl|org|organic|ovh|paris|partners|parts|
-        photo|photography|photos|physio|pics|pictures|pink|place|plumbing|post|press|pro|productions|properties|pub|
-        qpon|quebec|recipes|red|rehab|reise|reisen|ren|rentals|repair|report|republican|rest|reviews|rich|rio|rocks|
-        rodeo|ruhr|ryukyu|saarland|schmidt|schule|scot|services|sexy|shiksha|shoes|singles|social|software|sohu|solar|
-        solutions|soy|space|spiegel|supplies|supply|support|surf|surgery|suzuki|systems|tattoo|tax|technology|tel|
-        tienda|tips|tirol|today|tokyo|tools|town|toys|trade|training|travel|university|uno|vacations|vegas|ventures|
-        versicherung|vet|viajes|villas|vision|vlaanderen|vodka|vote|voting|voto|voyage|wang|watch|webcam|website|wed|
-        wien|wiki|works|wtc|wtf|xxx|xyz|yachts|yokohama|zone|дети|москва|онлайн|орг|сайт|بازار|شبكة|موقع|संगठन|みんな|世界|
-        中信|中文网|公司|公益|商城|商标|在线|我爱你|政务|机构|游戏|移动|组织机构|网址|网络|集团|삼성
-        )(?=[^0-9a-z@]|$)
+        (?:#{TLDS['generic'].join('|')})
+        (?=[^0-9a-z@]|$)
       )
     }ix
 
     REGEXEN[:valid_ccTLD] = %r{
       (?:
-        (?:
-        ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bl|bm|bn|bo|bq|br|bs|bt|bv|bw|
-        by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cw|cx|cy|cz|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|
-        fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|
-        is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mf|mg|mh|mk|ml|
-        mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|
-        ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|
-        tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|um|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw|мкд|мон|рф|
-        срб|укр|қаз|الاردن|الجزائر|السعودية|المغرب|امارات|ایران|بھارت|تونس|سودان|سورية|عمان|فلسطين|قطر|مصر|مليسيا|
-        پاکستان|भारत|বাংলা|ভারত|ਭਾਰਤ|ભારત|இந்தியா|இலங்கை|சிங்கப்பூர்|భారత్|ලංකා|ไทย|გე|中国|中國|台湾|台灣|新加坡|香港|한국
-        )(?=[^0-9a-z@]|$)
+        (?:#{TLDS['country'].join('|')})
+        (?=[^0-9a-z@]|$)
       )
     }ix
     REGEXEN[:valid_punycode] = /(?:xn--[0-9a-z]+)/i

data/lib/twitter-text/rewriter.rb

@@ -8,8 +8,8 @@ module Twitter
       entities = entities.sort_by{|entity| entity[:indices].first}
 
       result = []
-      last_index = entities.inject(0) do |last_index, entity|
-        result << chars[last_index...entity[:indices].first]
+      last_index = entities.inject(0) do |index, entity|
+        result << chars[index...entity[:indices].first]
         result << yield(entity, chars)
         entity[:indices].last
       end

data/lib/twitter-text/validation.rb

@@ -49,7 +49,7 @@ module Twitter
       begin
         return :too_long if tweet_length(text) > MAX_LENGTH
         return :invalid_characters if Twitter::Regex::INVALID_CHARACTERS.any?{|invalid_char| text.include?(invalid_char) }
-      rescue ArgumentError => e
+      rescue ArgumentError
         # non-Unicode value.
         return :invalid_characters
       end

data/test/conformance_test.rb

@@ -107,6 +107,14 @@ class ConformanceTest < Test::Unit::TestCase
     end
   end
 
+  def_conformance_test("tlds.yml", :generic) do
+    assert_equal expected, extract_urls(text), description
+  end
+
+  def_conformance_test("tlds.yml", :country) do
+    assert_equal expected, extract_urls(text), description
+  end
+
   def_conformance_test("extract.yml", :urls_with_indices) do
     e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
     assert_equal e, extract_urls_with_indices(text), description

data/test/twitter-text-conformance/tld_lib.yml

@@ -0,0 +1,752 @@
+---
+country:
+- ac
+- ad
+- ae
+- af
+- ag
+- ai
+- al
+- am
+- an
+- ao
+- aq
+- ar
+- as
+- at
+- au
+- aw
+- ax
+- az
+- ba
+- bb
+- bd
+- be
+- bf
+- bg
+- bh
+- bi
+- bj
+- bl
+- bm
+- bn
+- bo
+- bq
+- br
+- bs
+- bt
+- bv
+- bw
+- by
+- bz
+- ca
+- cc
+- cd
+- cf
+- cg
+- ch
+- ci
+- ck
+- cl
+- cm
+- cn
+- co
+- cr
+- cu
+- cv
+- cw
+- cx
+- cy
+- cz
+- de
+- dj
+- dk
+- dm
+- do
+- dz
+- ec
+- ee
+- eg
+- eh
+- er
+- es
+- et
+- eu
+- fi
+- fj
+- fk
+- fm
+- fo
+- fr
+- ga
+- gb
+- gd
+- ge
+- gf
+- gg
+- gh
+- gi
+- gl
+- gm
+- gn
+- gp
+- gq
+- gr
+- gs
+- gt
+- gu
+- gw
+- gy
+- hk
+- hm
+- hn
+- hr
+- ht
+- hu
+- id
+- ie
+- il
+- im
+- in
+- io
+- iq
+- ir
+- is
+- it
+- je
+- jm
+- jo
+- jp
+- ke
+- kg
+- kh
+- ki
+- km
+- kn
+- kp
+- kr
+- kw
+- ky
+- kz
+- la
+- lb
+- lc
+- li
+- lk
+- lr
+- ls
+- lt
+- lu
+- lv
+- ly
+- ma
+- mc
+- md
+- me
+- mf
+- mg
+- mh
+- mk
+- ml
+- mm
+- mn
+- mo
+- mp
+- mq
+- mr
+- ms
+- mt
+- mu
+- mv
+- mw
+- mx
+- my
+- mz
+- na
+- nc
+- ne
+- nf
+- ng
+- ni
+- nl
+- 'no'
+- np
+- nr
+- nu
+- nz
+- om
+- pa
+- pe
+- pf
+- pg
+- ph
+- pk
+- pl
+- pm
+- pn
+- pr
+- ps
+- pt
+- pw
+- py
+- qa
+- re
+- ro
+- rs
+- ru
+- rw
+- sa
+- sb
+- sc
+- sd
+- se
+- sg
+- sh
+- si
+- sj
+- sk
+- sl
+- sm
+- sn
+- so
+- sr
+- ss
+- st
+- su
+- sv
+- sx
+- sy
+- sz
+- tc
+- td
+- tf
+- tg
+- th
+- tj
+- tk
+- tl
+- tm
+- tn
+- to
+- tp
+- tr
+- tt
+- tv
+- tw
+- tz
+- ua
+- ug
+- uk
+- um
+- us
+- uy
+- uz
+- va
+- vc
+- ve
+- vg
+- vi
+- vn
+- vu
+- wf
+- ws
+- ye
+- yt
+- za
+- zm
+- zw
+- "бел"
+- "мкд"
+- "мон"
+- "рф"
+- "срб"
+- "укр"
+- "қаз"
+- "الاردن"
+- "الجزائر"
+- "السعودية"
+- "المغرب"
+- "امارات"
+- "ایران"
+- "بھارت"
+- "تونس"
+- "سودان"
+- "سورية"
+- "عراق"
+- "عمان"
+- "فلسطين"
+- "قطر"
+- "مصر"
+- "مليسيا"
+- "پاکستان"
+- "भारत"
+- "বাংলা"
+- "ভারত"
+- "ਭਾਰਤ"
+- "ભારત"
+- "இந்தியா"
+- "இலங்கை"
+- "சிங்கப்பூர்"
+- "భారత్"
+- "ලංකා"
+- "ไทย"
+- "გე"
+- "中国"
+- "中國"
+- "台湾"
+- "台灣"
+- "新加坡"
+- "香港"
+- "한국"
+generic:
+- abogado
+- academy
+- accountants
+- active
+- actor
+- aero
+- agency
+- airforce
+- allfinanz
+- alsace
+- archi
+- army
+- arpa
+- asia
+- associates
+- attorney
+- auction
+- audio
+- autos
+- axa
+- band
+- bar
+- bargains
+- bayern
+- beer
+- berlin
+- best
+- bid
+- bike
+- bio
+- biz
+- black
+- blackfriday
+- blue
+- bmw
+- bnpparibas
+- boo
+- boutique
+- brussels
+- budapest
+- build
+- builders
+- business
+- buzz
+- bzh
+- cab
+- cal
+- camera
+- camp
+- cancerresearch
+- capetown
+- capital
+- caravan
+- cards
+- care
+- career
+- careers
+- casa
+- cash
+- cat
+- catering
+- center
+- ceo
+- cern
+- channel
+- cheap
+- christmas
+- chrome
+- church
+- citic
+- city
+- claims
+- cleaning
+- click
+- clinic
+- clothing
+- club
+- codes
+- coffee
+- college
+- cologne
+- com
+- community
+- company
+- computer
+- condos
+- construction
+- consulting
+- contractors
+- cooking
+- cool
+- coop
+- country
+- credit
+- creditcard
+- crs
+- cruises
+- cuisinella
+- cymru
+- dad
+- dance
+- dating
+- day
+- deals
+- degree
+- democrat
+- dental
+- dentist
+- desi
+- diamonds
+- diet
+- digital
+- direct
+- directory
+- discount
+- dnp
+- domains
+- durban
+- dvag
+- eat
+- edu
+- education
+- email
+- emerck
+- engineer
+- engineering
+- enterprises
+- equipment
+- esq
+- estate
+- eus
+- events
+- exchange
+- expert
+- exposed
+- fail
+- farm
+- feedback
+- finance
+- financial
+- fish
+- fishing
+- fitness
+- flights
+- florist
+- flsmidth
+- fly
+- foo
+- forsale
+- foundation
+- frl
+- frogans
+- fund
+- furniture
+- futbol
+- gal
+- gallery
+- gbiz
+- gent
+- gift
+- gifts
+- gives
+- glass
+- gle
+- global
+- globo
+- gmail
+- gmo
+- gmx
+- google
+- gop
+- gov
+- graphics
+- gratis
+- green
+- gripe
+- guide
+- guitars
+- guru
+- hamburg
+- haus
+- healthcare
+- help
+- here
+- hiphop
+- hiv
+- holdings
+- holiday
+- homes
+- horse
+- host
+- hosting
+- house
+- how
+- ibm
+- immo
+- immobilien
+- industries
+- info
+- ing
+- ink
+- institute
+- insure
+- int
+- international
+- investments
+- jetzt
+- jobs
+- joburg
+- juegos
+- kaufen
+- kim
+- kitchen
+- kiwi
+- koeln
+- krd
+- kred
+- lacaixa
+- land
+- lawyer
+- lease
+- lgbt
+- life
+- lighting
+- limited
+- limo
+- link
+- loans
+- london
+- lotto
+- ltda
+- luxe
+- luxury
+- maison
+- management
+- mango
+- market
+- marketing
+- media
+- meet
+- melbourne
+- meme
+- menu
+- miami
+- mil
+- mini
+- mobi
+- moda
+- moe
+- monash
+- mortgage
+- moscow
+- motorcycles
+- mov
+- museum
+- nagoya
+- name
+- navy
+- net
+- network
+- neustar
+- new
+- nexus
+- ngo
+- nhk
+- ninja
+- nra
+- nrw
+- nyc
+- okinawa
+- ong
+- onl
+- ooo
+- org
+- organic
+- otsuka
+- ovh
+- paris
+- partners
+- parts
+- pharmacy
+- photo
+- photography
+- photos
+- physio
+- pics
+- pictures
+- pink
+- pizza
+- place
+- plumbing
+- pohl
+- poker
+- post
+- praxi
+- press
+- pro
+- prod
+- productions
+- prof
+- properties
+- property
+- pub
+- qpon
+- quebec
+- realtor
+- recipes
+- red
+- rehab
+- reise
+- reisen
+- ren
+- rentals
+- repair
+- report
+- republican
+- rest
+- restaurant
+- reviews
+- rich
+- rio
+- rip
+- rocks
+- rodeo
+- rsvp
+- ruhr
+- ryukyu
+- saarland
+- sarl
+- sca
+- scb
+- schmidt
+- schule
+- scot
+- services
+- sexy
+- shiksha
+- shoes
+- singles
+- social
+- software
+- sohu
+- solar
+- solutions
+- soy
+- space
+- spiegel
+- supplies
+- supply
+- support
+- surf
+- surgery
+- suzuki
+- systems
+- taipei
+- tatar
+- tattoo
+- tax
+- technology
+- tel
+- tienda
+- tips
+- tirol
+- today
+- tokyo
+- tools
+- top
+- town
+- toys
+- trade
+- training
+- travel
+- tui
+- university
+- uno
+- uol
+- vacations
+- vegas
+- ventures
+- vermögensberater
+- vermögensberatung
+- versicherung
+- vet
+- viajes
+- villas
+- vision
+- vlaanderen
+- vodka
+- vote
+- voting
+- voto
+- voyage
+- wales
+- wang
+- watch
+- webcam
+- website
+- wed
+- wedding
+- whoswho
+- wien
+- wiki
+- williamhill
+- wme
+- work
+- works
+- world
+- wtc
+- wtf
+- xxx
+- xyz
+- yachts
+- yandex
+- yoga
+- yokohama
+- youtube
+- zip
+- zone
+- "дети"
+- "москва"
+- "онлайн"
+- "орг"
+- "рус"
+- "сайт"
+- "بازار"
+- "شبكة"
+- "موقع"
+- "संगठन"
+- "みんな"
+- "世界"
+- "中信"
+- "中文网"
+- "企业"
+- "佛山"
+- "公司"
+- "公益"
+- "商城"
+- "商标"
+- "在线"
+- "广东"
+- "我爱你"
+- "手机"
+- "政务"
+- "机构"
+- "游戏"
+- "移动"
+- "组织机构"
+- "网址"
+- "网络"
+- "集团"
+- "삼성"

data/twitter-text.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = "twitter-text"
-  s.version = "1.9.2"
+  s.version = "1.10.0"
   s.authors = ["Matt Sanford", "Patrick Ewing", "Ben Cherry", "Britt Selvitelle",
                "Raffi Krikorian", "J.P. Cummins", "Yoshimasa Niwa", "Keita Fujii", "James Koval"]
   s.email = ["matt@twitter.com", "patrick.henry.ewing@gmail.com", "bcherry@gmail.com", "bs@brittspace.com",
@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "simplecov", "~> 0.8.0"
   s.add_runtime_dependency     "unf", "~> 0.1.0"
 
-  s.files         = `git ls-files`.split("\n")
+  s.files         = `git ls-files`.split("\n") + ['test/twitter-text-conformance/tld_lib.yml']
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twitter-text
 version: !ruby/object:Gem::Version
-  version: 1.9.2
+  version: 1.10.0
 platform: ruby
 authors:
 - Matt Sanford
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-24 00:00:00.000000000 Z
+date: 2014-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -163,6 +163,7 @@ files:
 - spec/unicode_spec.rb
 - spec/validation_spec.rb
 - test/conformance_test.rb
+- test/twitter-text-conformance/tld_lib.yml
 - twitter-text.gemspec
 homepage: http://twitter.com
 licenses: