ark_tweet_nlp 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e1c7368edd30beda62c7081156d55cb42d459877
+  data.tar.gz: e422714ed6d24d3f74a7b12f0b13cc02556db649
+SHA512:
+  metadata.gz: 355ae4a472b1bba1cc62fdba4c9017a2be8303e71fafceb4839c0a247bcf039e29603e1d1dbb762987349749c6b82c45dbb541c010e2e51b8277aa418c2525ec
+  data.tar.gz: 5c1ef397bc74910781a0c465dda5c0fe9baf1f6831aa9f21ef1affe9e2e8183841c719204cb1ff017b1cadd838410f59db54b078df526005465ecaf26a8c0828

data/.gitignore

@@ -0,0 +1,47 @@
+### OSX ###
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear on external disk
+.Spotlight-V100
+.Trashes
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+*.gem
+
+
+### vim ###
+[._]*.s[a-w][a-z]
+[._]s[a-w][a-z]
+*.un~
+Session.vim
+.netrwhist
+*~

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ark_tweet_nlp.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,5 @@
+guard :rspec, cmd: 'rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Bernardo
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,53 @@
+# ArkTweetNlp
+
+Ruby wrapper for the [Carnegie Mellon Twitter NLP and Part-of-Speech Tagging](http://www.ark.cs.cmu.edu/TweetNLP/)
+Not all features are implemented yet, check the examples to see how to use it.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ark_tweet_nlp'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ark_tweet_nlp
+
+## Usage
+
+See the list of supported tags:
+```ruby
+ArkTweetNlp::Parser::TAGSET
+```
+
+Tag a tweet text:
+```ruby
+ArkTweetNlp::Parser.find_tags('faceboooooooook is awesome')
+#=> [ {'faceboooooooook' => :^,'is' => :V,'awesome' => :A }]
+```
+
+Or multiple tweets separated by \n:
+```ruby
+ ArkTweetNlp::Parser.find_tags("faceboooooooook is awesome\nfaceboooooooook is awesome")
+#=> [{'faceboooooooook' => :^,'is' => :V,'awesome' => :A},{'faceboooooooook' => :^,'is' => :V,'awesome' => :A} ]
+```
+
+Get all words tagged as a specific tag:
+```ruby
+tagged_result = [{'faceboooooooook' => :^,'is' => :V,'awesome' => :A}]
+ArkTweetNlp::Parser.get_words_tagged_as(tagged_result, :A,:V,:^)
+#=>  {:^ => ["faceboooooooook"], :V => ["is"], :A => ["awesome"]}
+```
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/ark_tweet_nlp/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require "rspec/core/rake_task"
+require "bundler/gem_tasks"
+
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.rspec_opts = [ '--color', '--format', 'nested' ]
+end
+
+task default: :spec

data/ark_tweet_nlp.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ark_tweet_nlp/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ark_tweet_nlp"
+  spec.version       = ArkTweetNlp::VERSION
+  spec.authors       = ["Bernardo"]
+  spec.email         = ["bersimoes@gmail.com"]
+  spec.summary       = %q{Ruby wrapper for the Carnegie Mellon Twitter NLP}
+  spec.description   = %q{Tags tweets word into multiple cathegories using NLP}
+  spec.homepage      = "https://github.com/golfadas/ark_tweet_nlp_ruby"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec-nc"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency 'pry-nav'
+  spec.add_development_dependency 'pry-rescue'
+  spec.add_development_dependency 'pry-stack_explorer'
+  spec.add_development_dependency 'pry-doc'
+  spec.add_development_dependency 'guard'
+  spec.add_development_dependency 'guard-rspec'
+end

data/bin/runTagger.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -eu
+
+# Run the tagger (and tokenizer).
+java -XX:ParallelGCThreads=2 -Xmx500m -jar $(dirname $0)/ark-tweet-nlp-0.3.2.jar "$@"

data/lib/ark_tweet_nlp/parser.rb

@@ -0,0 +1,72 @@
+require 'set'
+
+module ArkTweetNlp
+  module Parser
+    TAGSET = {
+      :N => 'common noun',
+      :O => 'pronoun, non possessive',
+      :^ => 'proper noun',
+      :S => 'nominal + possessive',
+      :Z => 'proper noun + possessive',
+      :V => 'verb including copula, auxiliaries',
+      :L => 'nominal + verbal (e.g. i’m), verbal + nominal (let’s)',
+      :M => 'proper noun + verbal',
+      :A => 'adjective',
+      :R => 'adverb',
+      :! => 'interjection',
+      :D => 'determiner',
+      :P => 'pre- or postposition, or subordinating conjunction',
+      :& => 'coordinating conjunction',
+      :T => 'verb particle',
+      :X => 'existential there, predeterminers',
+      :Y => 'X + verbal',
+      :'#' => 'hashtag (indicates topic/category for tweet)',
+      :'@' => 'at-mention (indicates a user as a recipient of a tweet)',
+      :~ => 'discourse marker, indications of continuation across multiple tweets',
+      :U => 'URL or email address',
+      :E => 'emoticon',
+      :'$' => 'numeral',
+      :',' => 'punctuation',
+      :G => 'other abbreviations, foreign words, possessive endings, symbols, garbage'
+    }
+    TAGGER_PATH = File.join(Dir.pwd , '/bin/runTagger.sh')
+
+    def Parser.ola
+      "ola"
+    end
+
+    def Parser.find_tags text
+      result = Parser.run_tagger(text)
+      result.split("\n").map{ |line| Parser.convert_line( line ) }
+    end
+
+    def Parser.get_words_tagged_as tagged_result, *tags
+      Parser.merge_array( tagged_result.map{ |e| Parser.safe_invert( e ).select{ |key| tags.include? key } })
+    end
+
+    private
+    def Parser.merge hash1, hash2
+      hash2.each{ |key, value| hash1[key] ||= Set.new; hash1[key] << value }
+    end
+
+    # merges all hashs inside array
+    def Parser.merge_array arr
+      arr.each.inject({}){ |res,hash| Parser.merge(res,hash) }
+    end
+
+    def Parser.run_tagger text
+      `echo '#{text}' | #{TAGGER_PATH}`
+    end
+
+    def Parser.convert_line line
+      text = line.split("\t")[0].split
+      tags = line.split("\t")[1].split
+      text.each.with_index.inject({}){ |result,(value,index)| result[value] = tags[index].to_sym; result }
+    end
+
+    def Parser.safe_invert hash
+       hash.each.inject({}){|sum,val| sum[val.last] ||= Set.new; sum[val.last] << val.first; sum}
+    end
+
+  end
+end

data/lib/ark_tweet_nlp/version.rb

@@ -0,0 +1,3 @@
+module ArkTweetNlp
+  VERSION = "0.1.0"
+end

data/lib/ark_tweet_nlp.rb

@@ -0,0 +1,5 @@
+require "ark_tweet_nlp/version"
+require "ark_tweet_nlp/parser"
+
+module ArkTweetNlp
+end

data/spec/parser_spec.rb

@@ -0,0 +1,35 @@
+require_relative "../lib/ark_tweet_nlp/parser.rb"
+
+describe ArkTweetNlp::Parser do
+  describe 'TAGSET' do
+    it "It has 25 different types of tags" do
+      expect(ArkTweetNlp::Parser::TAGSET.size).to eq 25
+    end
+  end
+  describe '#find_tags' do
+    it "cathegorizes words in tweets" do
+      expect(ArkTweetNlp::Parser.find_tags('faceboooooooook is awesome')).to eq([ {'faceboooooooook' => :^,
+                                                                                 'is' => :V,
+                                                                                 'awesome' => :A }])
+    end
+    it "tags multiple tweets per line" do
+      expect(ArkTweetNlp::Parser.find_tags("faceboooooooook is awesome\nfaceboooooooook is awesome")).to eq([{'faceboooooooook' => :^,'is' => :V,'awesome' => :A},{'faceboooooooook' => :^,'is' => :V,'awesome' => :A} ])
+
+    end
+  end
+
+  describe '#get_words_tagged_as' do
+    it "returns only the words that where tagged with the specified tags" do
+      tagged_result =[{'faceboooooooook' => :^,'is' => :V,'awesome' => :A}]
+      expect(ArkTweetNlp::Parser.get_words_tagged_as(tagged_result, :A)).to eq(:A => (Set.new(['awesome']) ))
+    end
+    it "supports multiple tags" do
+      tagged_result = [{'faceboooooooook' => :^,'is' => :V,'awesome' => :A}]
+      expect(ArkTweetNlp::Parser.get_words_tagged_as(tagged_result, :A,:V,:^)).to eq( {:^ => Set.new(["faceboooooooook"]), :V => Set.new(["is"]), :A => Set.new(["awesome"]) })
+    end
+    it "supports muliple hashes" do
+      tagged_result = [{'faceboooooooook' => :^,'is' => :V,'awesome' => :A},{'faceboooooooook' => :^,'is' => :V,'awesome' => :A, 'blossom' => :A}]
+      expect(ArkTweetNlp::Parser.get_words_tagged_as(tagged_result, :A,:V,:^)).to eq( {:^ => Set.new(["faceboooooooook"]), :V => Set.new(["is"]), :A => Set.new(["awesome","blossom"]) })
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+ require 'pry'
+ require 'ArkTweetNlp'

metadata

@@ -0,0 +1,216 @@
+--- !ruby/object:Gem::Specification
+name: ark_tweet_nlp
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Bernardo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-nc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-rescue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-stack_explorer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-doc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Tags tweets word into multiple cathegories using NLP
+email:
+- bersimoes@gmail.com
+executables:
+- ark-tweet-nlp-0.3.2.jar
+- runTagger.sh
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- ark_tweet_nlp.gemspec
+- bin/ark-tweet-nlp-0.3.2.jar
+- bin/runTagger.sh
+- lib/ark_tweet_nlp.rb
+- lib/ark_tweet_nlp/parser.rb
+- lib/ark_tweet_nlp/version.rb
+- spec/parser_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/golfadas/ark_tweet_nlp_ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Ruby wrapper for the Carnegie Mellon Twitter NLP
+test_files:
+- spec/parser_spec.rb
+- spec/spec_helper.rb