tweetparser 0.2.1 → 0.2.2

data/lib/tweetparser/grammar.treetop

@@ -7,32 +7,72 @@ grammar TweetContent
     }
   end
 
+  # Partial components
+
+  rule ascii_letter
+    [a-zA-Z]
+  end
+
+  rule digit
+    [0-9]
+  end
+
+  rule non_ascii
+    [^\x20-\x7F]
+  end
+
+  rule hyphen
+    "-"
+  end
+
+  rule underscore
+    "_"
+  end
+
+  rule at_sign
+    "@" / "＠"
+  end
+
+  rule hash_sign
+    "#" / "＃"
+  end
+
   rule subdomain
-    ([a-zA-Z0-9\-] / [^\x20-\x7F])+
+    (ascii_letter / digit / hyphen / non_ascii)+
   end
 
   rule tld
-    [a-zA-Z] [a-zA-Z]+
+    ascii_letter ascii_letter+
+  end
+
+  rule ascii_name
+    (ascii_letter / digit / underscore)+
+  end
+
+  rule ascii_name_with_letters
+    ascii_letter (ascii_letter / digit / underscore)* / (digit / underscore)+ ascii_letter (ascii_letter / digit / underscore)*
+  end
+
+  rule protocol
+    [hH] [tT] [tT] [pP] [sS]? "://"
   end
 
+  rule path_component
+    [a-zA-Z0-9?=\-_&%();:\.,~+/]
+  end
+
+  # Outputs
+
   rule url
-    (("http" / "HTTP") [sS]? "://" / "www." / "WWW.") (subdomain ".")+ tld (":" [0-9]+)? ("/" [a-zA-Z0-9\?#=\-_&%();:\.,~+]*)* {
+    (protocol / [Ww] [Ww] [Ww] ".") (subdomain ".")+ tld (":" [0-9]+)? ("/" path_component*)? ("#" path_component*)? {
       def content
         [:url, text_value]
       end
     }
   end
 
-  rule name
-    [a-zA-Z0-9_]+
-  end
-
-  rule name_with_letters
-    [a-zA-Z] [a-zA-Z0-9_]* / [0-9_]+ [a-zA-Z] [a-zA-Z0-9_]*
-  end
-
   rule username
-    ("@" / "＠") name !("@" / "＠" / "_") {
+    at_sign ascii_name !(at_sign / underscore) {
       def content
         [:username, text_value]
       end
@@ -40,7 +80,7 @@ grammar TweetContent
   end
 
   rule list
-    username "/" name {
+    username "/" ascii_name {
       def content
         [:list, text_value]
       end
@@ -48,7 +88,7 @@ grammar TweetContent
   end
 
   rule hashtag
-    ("#" / "＃") name_with_letters {
+    hash_sign ascii_name_with_letters {
       def content
         [:hashtag, text_value]
       end
@@ -56,7 +96,7 @@ grammar TweetContent
   end
 
   rule slash
-    "/" name {
+    "/" ascii_name {
       def content
         [:slash, text_value]
       end

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: tweetparser
 version: !ruby/object:Gem::Version 
-  version: 0.2.1
+  hash: 19
+  prerelease: false
+  segments: 
+  - 0
+  - 2
+  - 2
+  version: 0.2.2
 platform: ruby
 authors: 
 - Paul Battley
@@ -9,39 +15,39 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-24 00:00:00 +00:00
+date: 2010-09-21 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: treetop
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 1
+        - 4
+        - 2
         version: 1.4.2
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: polyglot
   type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        version: 0.2.9
-    version: 
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: shoulda
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :development
+  version_requirements: *id002
 description: 
 email: pbattley@gmail.com
 executables: []
@@ -51,12 +57,12 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
-- test/autolink_conformance_test.rb
-- test/extraction_conformance_test.rb
-- test/parser_test.rb
-- test/twitter-text-conformance/autolink.yml
 - test/twitter-text-conformance/extract.yml
+- test/twitter-text-conformance/autolink.yml
 - test/twitter-text-conformance/README
+- test/parser_test.rb
+- test/autolink_conformance_test.rb
+- test/extraction_conformance_test.rb
 - lib/tweetparser/grammar.treetop
 - lib/tweetparser.rb
 has_rdoc: true
@@ -69,21 +75,27 @@ rdoc_options: []
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Extract content from tweets