tweetparser 0.2.0 → 0.2.1

data/lib/tweetparser/grammar.treetop

@@ -11,8 +11,12 @@ grammar TweetContent
     ([a-zA-Z0-9\-] / [^\x20-\x7F])+
   end
 
+  rule tld
+    [a-zA-Z] [a-zA-Z]+
+  end
+
   rule url
-    (("http" / "HTTP") [sS]? "://" / "www." / "WWW.") subdomain ("." subdomain)+ ("/" [\.a-zA-Z0-9\?#=\-_&%]*)* {
+    (("http" / "HTTP") [sS]? "://" / "www." / "WWW.") (subdomain ".")+ tld (":" [0-9]+)? ("/" [a-zA-Z0-9\?#=\-_&%();:\.,~+]*)* {
       def content
         [:url, text_value]
       end
@@ -28,7 +32,7 @@ grammar TweetContent
   end
 
   rule username
-    ("@" / "＠") name {
+    ("@" / "＠") name !("@" / "＠" / "_") {
       def content
         [:username, text_value]
       end

data/test/{conformance_test.rb → autolink_conformance_test.rb}

@@ -4,7 +4,6 @@ require "test/unit"
 require "shoulda"
 require "tweetparser"
 require "yaml"
-require "cgi"
 
 class AutolinkConformanceTest < Test::Unit::TestCase
   DATA_PATH = File.expand_path("../twitter-text-conformance/autolink.yml", __FILE__)

data/test/extraction_conformance_test.rb

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+$KCODE = "u"
+$:.unshift(File.expand_path("../../lib", __FILE__))
+require "test/unit"
+require "shoulda"
+require "tweetparser"
+require "yaml"
+
+class ExtractionConformanceTest < Test::Unit::TestCase
+  DATA_PATH = File.expand_path("../twitter-text-conformance/extract.yml", __FILE__)
+
+  test_data = YAML.load(File.read(DATA_PATH))["tests"]
+
+  context "when extracting mentions" do
+    test_data["mentions"].each do |hash|
+      should hash["description"] do
+        parts = TweetParser.parse(hash["text"])
+        assert_equal hash["expected"], parts.select{ |a| a[0] == :username }.map{ |a| a[1][/^.(.*)/, 1] }
+      end
+    end
+  end
+
+  context "when extracting urls" do
+    test_data["urls"].each do |hash|
+      should hash["description"] do
+        parts = TweetParser.parse(hash["text"])
+        assert_equal hash["expected"], parts.select{ |a| a[0] == :url }.map{ |a| a[1] }
+      end
+    end
+  end
+
+  context "when extracting hashtags" do
+    test_data["hashtags"].each do |hash|
+      should hash["description"] do
+        parts = TweetParser.parse(hash["text"])
+        assert_equal hash["expected"], parts.select{ |a| a[0] == :hashtag }.map{ |a| a[1][/^.(.*)/, 1] }
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: tweetparser
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors: 
 - Paul Battley
@@ -51,7 +51,8 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
-- test/conformance_test.rb
+- test/autolink_conformance_test.rb
+- test/extraction_conformance_test.rb
 - test/parser_test.rb
 - test/twitter-text-conformance/autolink.yml
 - test/twitter-text-conformance/extract.yml