chipper 0.4.2

data/test/helper.rb

@@ -0,0 +1,5 @@
+$:.unshift File.dirname(__FILE__) + '/../ext'
+require 'chipper'
+require 'minitest/spec'
+
+MiniTest::Unit.autorun

data/test/test_entities.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+require 'helper'
+
+describe 'Chipper entities' do
+  before do
+    @tweet = "@youtube, why is that stories abt @apple on #cnn always get #removed from http://t.co/IsSh1t12"
+  end
+
+  after do
+    Chipper.skip_users    nil
+    Chipper.skip_hashtags nil
+  end
+
+  it 'should extract users' do
+    Chipper.users(@tweet).must_equal %w(@youtube @apple)
+  end
+
+  it 'should extract hashtags' do
+    Chipper.hashtags(@tweet).must_equal %w(#cnn #removed)
+  end
+
+  it 'should extract urls' do
+    Chipper.urls(@tweet).must_equal %w(http://t.co/IsSh1t12)
+  end
+
+  # NB, these should be only [a-zA-Z0-9]+ in the path in fact, not /w as that will include unicode chars
+  it 'should extract t.co urls cleanly w/quote' do
+    text = "Hello http://t.co/97CLxVkD\" http://t.co/12345678, http://t.co/xxxxxxxx. http://t.co/xxxx1234' damn!"
+    urls = %w(http://t.co/97CLxVkD http://t.co/12345678 http://t.co/xxxxxxxx http://t.co/xxxx1234)
+    Chipper.urls(text).must_equal(urls)
+  end
+
+  it 'should skip users' do
+    Chipper.skip_users(%w(youtube))
+    Chipper.users(@tweet).must_equal %w(@apple)
+  end
+
+  it 'should skip hashtags' do
+    Chipper.skip_hashtags(%w(cnn))
+    Chipper.hashtags(@tweet).must_equal %w(#removed)
+  end
+
+  it 'should return all entities using #entities method' do
+    expected = {}
+    expected.merge! users:    %w(@youtube @apple)
+    expected.merge! hashtags: %w(#cnn #removed)
+    expected.merge! urls:     %w(http://t.co/IsSh1t12)
+    expected.merge! tokens:   [["why"], ["that", "stories", "abt"], ["always", "get"], ["from"]]
+
+    Chipper.entities(@tweet).must_equal expected
+  end
+
+  it 'should work around shitty urls' do
+    crap = "foo bar https://t.co/KCZSuVx½"
+    Chipper.urls(crap).must_equal ['https://t.co/KCZSuVx']
+  end
+end

data/test/test_tokens.rb

@@ -0,0 +1,118 @@
+require 'helper'
+
+describe 'Chipper tokens' do
+  before do
+    @tweet = "@youtube, why is that stories abt @apple on #cnn always get #removed from http://www1.youtube.com/videos/?"
+  end
+
+  after do
+    Chipper.skip_tokens    nil
+  end
+
+  it 'should extract tokens' do
+    expected = [%w(why), %w(that stories abt), %w(always get), %w(from)]
+    Chipper.tokens(@tweet).must_equal expected
+  end
+
+  it 'should skip tokens' do
+    expected = [%w(why), %w(that), %w(abt), %w(get)]
+    Chipper.skip_tokens(%w(story always from video))
+    Chipper.tokens(@tweet).must_equal expected
+  end
+
+  it 'should skip numbers dates and times' do
+    expected = [["flubble","bubble"],["champion"]]
+    Chipper.skip_token_pattern %q{^(?:[\d\-]+)(?:am|pm|th|st|rd)?$}
+    Chipper.tokens("flubble bubble I or 123454 123-123-1111 19 on te champion 19th at 4am or 12pm").must_equal expected
+  end
+
+  it 'should skip random other chars dates and times' do
+    expected = [["flubble","bubble"],["champion","winter"],["incredible"]]
+    Chipper.skip_token_pattern %q{^(?:[\d\-]+)(?:am|pm|th|st|rd)?$}
+    Chipper.tokens("flubble_bubble ^o^ champion_winter oh_____ _______ ^incredible ").must_equal expected
+  end
+
+  it 'should be lowercase' do
+    expected = [["flubble","bubble"]]
+    Chipper.tokens("FLUBBLE BUBBLE").must_equal expected
+  end
+
+  it 'stop words should not be case sensitive' do
+    expected = [["pancakes"]]
+    Chipper.skip_tokens(%w(christmas))
+    Chipper.tokens("pancakes in Christmas").must_equal expected
+  end
+
+  it 'should filter stemmed words that are too short' do
+    expected = [["flubble","bubble"]]
+    Chipper.tokens("I am going to doing, being flubble bubble its").must_equal expected
+  end
+
+  # eg don't, won't aren't etc.  Quick 'cheats' way to do this is just sub out all single quotes with nothing.
+  it 'should remove abbreviations with single quotes' do
+    expected = [["flubble","bubble","dont"]]
+    Chipper.tokens("flubble bubble don't do it").must_equal expected
+  end
+
+  it 'should segment across urls' do
+    expected = [%w(hello world), %w(this), %w(might work)]
+    Chipper.tokens('hello world, this http://www.example.com/1 might work').must_equal expected
+  end
+
+  describe 'segment across short, stop and non-words' do
+    before do
+      @expected = [%w(flopper bopper), %w(dopper)]
+      Chipper.skip_tokens(%w(four five six stopper))
+    end
+
+    after do
+      Chipper.skip_tokens nil
+    end
+
+    it 'should segment correctly on short word' do
+      Chipper.tokens('Flopper Bopper at Dopper').must_equal @expected
+    end
+
+    it 'should segment correctly on stop word' do
+      Chipper.tokens('Flopper Bopper STOPPER Dopper').must_equal @expected
+    end
+
+    it 'should segment correctly on non-word' do
+      Chipper.tokens('Flopper Bopper. Dopper').must_equal @expected
+    end
+
+    it 'should not get stuffs from users or hashtags' do
+      Chipper.tokens("melbourne  @sydney_islame or #brisbane_humid").must_equal [["melbourne"]]
+    end
+
+    it 'should not add random newlines becasue there is underscores, full stop, space then url' do
+      Chipper.tokens("one__two.three http://t.co").must_equal [["one","two"],["three"]]
+    end
+
+    it 'should not add random newlines becasue there is leading underscore colon space then url' do
+      Chipper.tokens('bushes _purple: http://t.co/').must_equal [["bushes","purple"]]
+    end
+
+    it 'should group and segment across repeated or anchored underscores' do
+      Chipper.tokens('_foo bar._baz: hello__world! http://t.co/').must_equal [%w(foo bar), %w(baz), %w(hello world)]
+      Chipper.tokens('_foo bar._baz:_hello #hi,world').must_equal [%w(foo bar), %w(baz), %w(hello), %w(world)]
+    end
+  end
+
+  describe 'unicode' do
+    it 'should skip quotes and handle fullwidth @' do
+      text = "hello world, ain\u2019t this \uff20cool huh\u201d"
+      Chipper.tokens(text).must_equal [["hello", "world"], ["aint", "this"], ["huh"]]
+    end
+
+    it 'should kill tokens that start with unicode' do
+      text = "hello world, \u2020\uff26\u201f \u2021\uff36\u210fcool 12TH 123345134 awesome\u2020\uff26\u201f"
+      Chipper.skip_token_pattern %q{^(?:[\d\-]+)(?:am|pm|th|st|rd)?$|^\W.*$}
+      Chipper.tokens(text).must_equal [["hello", "world"], ["awesome\u2020\uff26"]]
+    end
+
+    it 'should segment on unicode puncutation' do
+      Chipper.tokens("\u2039Hello\u203aWorld").must_equal [["hello"], ["world"]]
+    end
+  end
+end

metadata

@@ -0,0 +1,199 @@
+--- !ruby/object:Gem::Specification 
+name: chipper
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 4
+  - 2
+  version: 0.4.2
+platform: ruby
+authors: 
+- Bharanee Rathna
+- John Barratt
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-01-16 00:00:00 +11:00
+default_executable: 
+dependencies: []
+
+description: twitter text extraction utilities
+email: 
+- deepfryed@gmail.com
+- djon00@gmail.com
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- ext/src/chipper.cc
+- ext/re2/filtered_re2.cc
+- ext/re2/unicode_casefold.cc
+- ext/re2/prefilter.cc
+- ext/re2/prefilter_tree.cc
+- ext/re2/re2.cc
+- ext/re2/valgrind.cc
+- ext/re2/hash.cc
+- ext/re2/parse.cc
+- ext/re2/stringpiece.cc
+- ext/re2/set.cc
+- ext/re2/bitstate.cc
+- ext/re2/prog.cc
+- ext/re2/simplify.cc
+- ext/re2/rune.cc
+- ext/re2/dfa.cc
+- ext/re2/onepass.cc
+- ext/re2/unicode_groups.cc
+- ext/re2/regexp.cc
+- ext/re2/nfa.cc
+- ext/re2/perl_groups.cc
+- ext/re2/mimics_pcre.cc
+- ext/re2/compile.cc
+- ext/re2/tostring.cc
+- ext/libstemmer_c/src_c/stem_UTF_8_russian.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_french.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_english.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.c
+- ext/libstemmer_c/src_c/stem_UTF_8_danish.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_spanish.c
+- ext/libstemmer_c/src_c/stem_UTF_8_swedish.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_italian.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_dutch.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_2_romanian.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_danish.c
+- ext/libstemmer_c/src_c/stem_UTF_8_german.c
+- ext/libstemmer_c/src_c/stem_UTF_8_romanian.c
+- ext/libstemmer_c/src_c/stem_UTF_8_turkish.c
+- ext/libstemmer_c/src_c/stem_UTF_8_italian.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_swedish.c
+- ext/libstemmer_c/src_c/stem_KOI8_R_russian.c
+- ext/libstemmer_c/src_c/stem_UTF_8_norwegian.c
+- ext/libstemmer_c/src_c/stem_UTF_8_hungarian.c
+- ext/libstemmer_c/src_c/stem_UTF_8_portuguese.c
+- ext/libstemmer_c/src_c/stem_UTF_8_french.c
+- ext/libstemmer_c/src_c/stem_UTF_8_english.c
+- ext/libstemmer_c/src_c/stem_UTF_8_finnish.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_porter.c
+- ext/libstemmer_c/src_c/stem_UTF_8_porter.c
+- ext/libstemmer_c/src_c/stem_UTF_8_spanish.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_finnish.c
+- ext/libstemmer_c/src_c/stem_UTF_8_dutch.c
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_german.c
+- ext/libstemmer_c/libstemmer/libstemmer.c
+- ext/libstemmer_c/libstemmer/libstemmer_utf8.c
+- ext/libstemmer_c/runtime/api.c
+- ext/libstemmer_c/runtime/utilities.c
+- ext/libstemmer_c/examples/stemwords.c
+- ext/libstemmer_c/mkinc_utf8.mak
+- ext/libstemmer_c/mkinc.mak
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_hungarian.h
+- ext/libstemmer_c/src_c/stem_UTF_8_porter.h
+- ext/libstemmer_c/src_c/stem_UTF_8_danish.h
+- ext/libstemmer_c/src_c/stem_UTF_8_english.h
+- ext/libstemmer_c/src_c/stem_UTF_8_hungarian.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.h
+- ext/libstemmer_c/src_c/stem_UTF_8_romanian.h
+- ext/libstemmer_c/src_c/stem_UTF_8_norwegian.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_spanish.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_german.h
+- ext/libstemmer_c/src_c/stem_UTF_8_portuguese.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_french.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_swedish.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_italian.h
+- ext/libstemmer_c/src_c/stem_UTF_8_finnish.h
+- ext/libstemmer_c/src_c/stem_UTF_8_italian.h
+- ext/libstemmer_c/src_c/stem_UTF_8_russian.h
+- ext/libstemmer_c/src_c/stem_UTF_8_spanish.h
+- ext/libstemmer_c/src_c/stem_UTF_8_swedish.h
+- ext/libstemmer_c/src_c/stem_UTF_8_french.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_english.h
+- ext/libstemmer_c/src_c/stem_UTF_8_turkish.h
+- ext/libstemmer_c/src_c/stem_KOI8_R_russian.h
+- ext/libstemmer_c/src_c/stem_UTF_8_german.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_2_romanian.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_dutch.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_danish.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_porter.h
+- ext/libstemmer_c/src_c/stem_UTF_8_dutch.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_finnish.h
+- ext/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.h
+- ext/libstemmer_c/include/libstemmer.h
+- ext/libstemmer_c/libstemmer/modules_utf8.h
+- ext/libstemmer_c/libstemmer/modules.h
+- ext/libstemmer_c/runtime/header.h
+- ext/libstemmer_c/runtime/api.h
+- ext/src/version.h
+- ext/util/benchmark.h
+- ext/util/arena.h
+- ext/util/thread.h
+- ext/util/mutex.h
+- ext/util/atomicops.h
+- ext/util/valgrind.h
+- ext/util/test.h
+- ext/util/random.h
+- ext/util/sparse_set.h
+- ext/util/utf.h
+- ext/util/pcre.h
+- ext/util/flags.h
+- ext/util/util.h
+- ext/util/sparse_array.h
+- ext/util/logging.h
+- ext/re2/stringpiece.h
+- ext/re2/prefilter_tree.h
+- ext/re2/prefilter.h
+- ext/re2/walker-inl.h
+- ext/re2/variadic_function.h
+- ext/re2/set.h
+- ext/re2/re2.h
+- ext/re2/regexp.h
+- ext/re2/filtered_re2.h
+- ext/re2/unicode_casefold.h
+- ext/re2/prog.h
+- ext/re2/unicode_groups.h
+- ext/extconf.rb
+- ext/stemmer.rb
+- test/test_tokens.rb
+- test/helper.rb
+- test/test_entities.rb
+- ext/libstemmer_c/Makefile
+- README.rdoc
+has_rdoc: true
+homepage: http://github.com/deepfryed/chipper
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: twitter text extractor
+test_files: []
+