lda-ruby 0.3.5 → 0.3.6

data/CHANGELOG

@@ -1,3 +1,13 @@
+version 0.3.6
+=============
+
+- added stopwords list and included downcasing to improve performance
+
+version 0.3.5
+=============
+
+- Bug fix for text documents by Rio Akasaka
+
 Version 0.3.4
 =============
 
@@ -20,7 +30,6 @@ Version 0.2.3
 
 - Bug fixes by Todd Foster
 
-
 Version 0.2.2
 =============
 

data/Rakefile

@@ -12,6 +12,7 @@ begin
     gem.homepage = "http://github.com/ealdent/lda-ruby"
     gem.authors = ['David Blei', 'Jason Adams', 'Rio Akasaka']
     gem.extensions = ['ext/lda-ruby/extconf.rb']
+    gem.files.include 'stopwords.txt'
     gem.require_paths = ['lib', 'ext']
     gem.add_dependency 'shoulda'
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
 :major: 0
 :minor: 3
-:patch: 5
+:patch: 6
 :build: 

data/lda-ruby.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{lda-ruby}
-  s.version = "0.3.5"
+  s.version = "0.3.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David Blei", "Jason Adams", "Rio Akasaka"]
-  s.date = %q{2011-08-03}
+  s.date = %q{2011-08-05}
   s.description = %q{Ruby port of Latent Dirichlet Allocation by David M. Blei. See http://www.cs.princeton.edu/~blei/lda-c/.}
   s.email = %q{jasonmadams@gmail.com}
   s.extensions = ["ext/lda-ruby/extconf.rb"]
@@ -40,6 +40,7 @@ Gem::Specification.new do |s|
     "ext/lda-ruby/utils.h",
     "lda-ruby.gemspec",
     "lib/lda-ruby.rb",
+    "lib/lda-ruby/config/stopwords.yml",
     "lib/lda-ruby/corpus/corpus.rb",
     "lib/lda-ruby/corpus/data_corpus.rb",
     "lib/lda-ruby/corpus/directory_corpus.rb",
@@ -54,6 +55,8 @@ Gem::Specification.new do |s|
     "test/data/sample.rb",
     "test/data/wiki-test-docs.yml",
     "test/lda_ruby_test.rb",
+    "test/simple_test.rb",
+    "test/simple_yaml.rb",
     "test/test_helper.rb"
   ]
   s.homepage = %q{http://github.com/ealdent/lda-ruby}

data/lib/lda-ruby/config/stopwords.yml

@@ -0,0 +1,571 @@
+--- 
+- a
+- a's
+- able
+- about
+- above
+- according
+- accordingly
+- across
+- actually
+- after
+- afterwards
+- again
+- against
+- ain't
+- all
+- allow
+- allows
+- almost
+- alone
+- along
+- already
+- also
+- although
+- always
+- am
+- among
+- amongst
+- an
+- and
+- another
+- any
+- anybody
+- anyhow
+- anyone
+- anything
+- anyway
+- anyways
+- anywhere
+- apart
+- appear
+- appreciate
+- appropriate
+- are
+- aren't
+- around
+- as
+- aside
+- ask
+- asking
+- associated
+- at
+- available
+- away
+- awfully
+- b
+- be
+- became
+- because
+- become
+- becomes
+- becoming
+- been
+- before
+- beforehand
+- behind
+- being
+- believe
+- below
+- beside
+- besides
+- best
+- better
+- between
+- beyond
+- both
+- brief
+- but
+- by
+- c
+- c'mon
+- c's
+- came
+- can
+- can't
+- cannot
+- cant
+- cause
+- causes
+- certain
+- certainly
+- changes
+- clearly
+- co
+- com
+- come
+- comes
+- concerning
+- consequently
+- consider
+- considering
+- contain
+- containing
+- contains
+- corresponding
+- could
+- couldn't
+- course
+- currently
+- d
+- definitely
+- described
+- despite
+- did
+- didn't
+- different
+- do
+- does
+- doesn't
+- doing
+- don't
+- done
+- down
+- downwards
+- during
+- e
+- each
+- edu
+- eg
+- eight
+- either
+- else
+- elsewhere
+- enough
+- entirely
+- especially
+- et
+- etc
+- even
+- ever
+- every
+- everybody
+- everyone
+- everything
+- everywhere
+- ex
+- exactly
+- example
+- except
+- f
+- far
+- few
+- fifth
+- first
+- five
+- followed
+- following
+- follows
+- for
+- former
+- formerly
+- forth
+- four
+- from
+- further
+- furthermore
+- g
+- get
+- gets
+- getting
+- given
+- gives
+- go
+- goes
+- going
+- gone
+- got
+- gotten
+- greetings
+- h
+- had
+- hadn't
+- happens
+- hardly
+- has
+- hasn't
+- have
+- haven't
+- having
+- he
+- he's
+- hello
+- help
+- hence
+- her
+- here
+- here's
+- hereafter
+- hereby
+- herein
+- hereupon
+- hers
+- herself
+- hi
+- him
+- himself
+- his
+- hither
+- hopefully
+- how
+- howbeit
+- however
+- i
+- i'd
+- i'll
+- i'm
+- i've
+- ie
+- if
+- ignored
+- immediate
+- in
+- inasmuch
+- inc
+- indeed
+- indicate
+- indicated
+- indicates
+- inner
+- insofar
+- instead
+- into
+- inward
+- is
+- isn't
+- it
+- it'd
+- it'll
+- it's
+- its
+- itself
+- j
+- just
+- k
+- keep
+- keeps
+- kept
+- know
+- knows
+- known
+- l
+- last
+- lately
+- later
+- latter
+- latterly
+- least
+- less
+- lest
+- let
+- let's
+- like
+- liked
+- likely
+- little
+- look
+- looking
+- looks
+- ltd
+- m
+- mainly
+- many
+- may
+- maybe
+- me
+- mean
+- meanwhile
+- merely
+- might
+- more
+- moreover
+- most
+- mostly
+- much
+- must
+- my
+- myself
+- n
+- name
+- namely
+- nd
+- near
+- nearly
+- necessary
+- need
+- needs
+- neither
+- never
+- nevertheless
+- new
+- next
+- nine
+- "no"
+- nobody
+- non
+- none
+- noone
+- nor
+- normally
+- not
+- nothing
+- novel
+- now
+- nowhere
+- o
+- obviously
+- of
+- "off"
+- often
+- oh
+- ok
+- okay
+- old
+- "on"
+- once
+- one
+- ones
+- only
+- onto
+- or
+- other
+- others
+- otherwise
+- ought
+- our
+- ours
+- ourselves
+- out
+- outside
+- over
+- overall
+- own
+- p
+- particular
+- particularly
+- per
+- perhaps
+- placed
+- please
+- plus
+- possible
+- presumably
+- probably
+- provides
+- q
+- que
+- quite
+- qv
+- r
+- rather
+- rd
+- re
+- really
+- reasonably
+- regarding
+- regardless
+- regards
+- relatively
+- respectively
+- right
+- s
+- said
+- same
+- saw
+- say
+- saying
+- says
+- second
+- secondly
+- see
+- seeing
+- seem
+- seemed
+- seeming
+- seems
+- seen
+- self
+- selves
+- sensible
+- sent
+- serious
+- seriously
+- seven
+- several
+- shall
+- she
+- should
+- shouldn't
+- since
+- six
+- so
+- some
+- somebody
+- somehow
+- someone
+- something
+- sometime
+- sometimes
+- somewhat
+- somewhere
+- soon
+- sorry
+- specified
+- specify
+- specifying
+- still
+- sub
+- such
+- sup
+- sure
+- t
+- t's
+- take
+- taken
+- tell
+- tends
+- th
+- than
+- thank
+- thanks
+- thanx
+- that
+- that's
+- thats
+- the
+- their
+- theirs
+- them
+- themselves
+- then
+- thence
+- there
+- there's
+- thereafter
+- thereby
+- therefore
+- therein
+- theres
+- thereupon
+- these
+- they
+- they'd
+- they'll
+- they're
+- they've
+- think
+- third
+- this
+- thorough
+- thoroughly
+- those
+- though
+- three
+- through
+- throughout
+- thru
+- thus
+- to
+- together
+- too
+- took
+- toward
+- towards
+- tried
+- tries
+- truly
+- try
+- trying
+- twice
+- two
+- u
+- un
+- under
+- unfortunately
+- unless
+- unlikely
+- until
+- unto
+- up
+- upon
+- us
+- use
+- used
+- useful
+- uses
+- using
+- usually
+- v
+- value
+- various
+- very
+- via
+- viz
+- vs
+- w
+- want
+- wants
+- was
+- wasn't
+- way
+- we
+- we'd
+- we'll
+- we're
+- we've
+- welcome
+- well
+- went
+- were
+- weren't
+- what
+- what's
+- whatever
+- when
+- whence
+- whenever
+- where
+- where's
+- whereafter
+- whereas
+- whereby
+- wherein
+- whereupon
+- wherever
+- whether
+- which
+- while
+- whither
+- who
+- who's
+- whoever
+- whole
+- whom
+- whose
+- why
+- will
+- willing
+- wish
+- with
+- within
+- without
+- won't
+- wonder
+- would
+- would
+- wouldn't
+- x
+- y
+- "yes"
+- yet
+- you
+- you'd
+- you'll
+- you're
+- you've
+- your
+- yours
+- yourself
+- yourselves
+- z
+- zero

data/lib/lda-ruby/corpus/corpus.rb

@@ -2,13 +2,15 @@ require 'set'
 
 module Lda
   class Corpus
-    attr_reader :documents, :num_docs, :num_terms, :vocabulary
+    attr_reader :documents, :num_docs, :num_terms, :vocabulary, :stopwords
 
     def initialize
       @documents = Array.new
       @all_terms = Set.new
       @num_terms = @num_docs = 0
       @vocabulary = Vocabulary.new
+      @stopwords = YAML.load_file(File.join(File.dirname(__FILE__), '..', 'config', 'stopwords.yml'))
+      @stopwords.map! { |w| w.strip }
     end
 
     def add_document(doc)
@@ -21,10 +23,13 @@ module Lda
       @num_terms = @all_terms.size
 
       update_vocabulary(doc)
-
       nil
     end
-
+	
+	def remove_word(word)
+		@vocabulary.words.delete word
+	end
+	
     protected
 
     def update_vocabulary(doc)

data/lib/lda-ruby/corpus/text_corpus.rb

@@ -19,4 +19,4 @@ module Lda
       end
     end
   end
-end
+end

data/lib/lda-ruby/document/document.rb

@@ -1,3 +1,5 @@
+require 'yaml'
+
 module Lda
   class Document
     attr_reader :corpus, :words, :counts, :length, :total, :tokens
@@ -29,8 +31,9 @@ module Lda
     end
 
     def tokenize(text)
-      clean_text = text.gsub(/[^A-Za-z'\s]+/, ' ').gsub(/\s+/, ' ')        # remove everything but letters and ' and leave only single spaces
+      clean_text = text.gsub(/[^A-Za-z'\s]+/, ' ').gsub(/\s+/, ' ').downcase        # remove everything but letters and ' and leave only single spaces
       @tokens = handle(clean_text.split(' '))
+      nil
     end
   end
-end
+end

data/lib/lda-ruby/document/text_document.rb

@@ -7,6 +7,7 @@ module Lda
       @filename = nil
 
       tokenize(text)
+      @tokens.reject! { |w| @corpus.stopwords.include?(w) }
       build_from_tokens
     end
 

data/test/lda_ruby_test.rb

@@ -1,4 +1,11 @@
-require 'test_helper'
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'yaml'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'lda-ruby'
 
 class LdaRubyTest < Test::Unit::TestCase
   context "A Document instance" do
@@ -66,7 +73,7 @@ class LdaRubyTest < Test::Unit::TestCase
 
     context "A typical TextDocument" do
       setup do
-        @text = 'what is that which is what is else what is'
+        @text = 'stop words stop stop masterful stoppage buffalo buffalo buffalo'
         @document = Lda::TextDocument.new(@corpus, @text)
       end
 
@@ -104,7 +111,7 @@ class LdaRubyTest < Test::Unit::TestCase
 
       should "update vocabulary with words in the document" do
         @corpus.add_document(@document2)
-        assert_equal @corpus.vocabulary.words.member?('second'), true
+        assert_equal @corpus.vocabulary.words.member?('lame'), true
       end
     end
 

data/test/simple_test.rb

@@ -0,0 +1,26 @@
+require 'rubygems'
+require 'shoulda'
+require 'yaml'
+require 'lda-ruby'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+class Test::Unit::TestCase
+
+  @corpus = Lda::Corpus.new
+  @document1 = Lda::TextDocument.new(@corpus, 'Dom Cobb is a skilled thief, the absolute best in the dangerous art of extraction, stealing valuable secrets from deep within the subconscious during the dream state, when the mind is at its most vulnerable. Cobb\'s rare ability has made him a coveted player in this treacherous new world of corporate espionage, but it has also made him an international fugitive and cost him everything he has ever loved. Now Cobb is being offered a chance at redemption. One last job could give him his life back but only if he can accomplish the impossible-inception. Instead of the perfect heist, Cobb and his team of specialists have to pull off the reverse: their task is not to steal an idea but to plant one. If they succeed, it could be the perfect crime. But no amount of careful planning or expertise can prepare the team for the dangerous enemy that seems to predict their every move. An enemy that only Cobb could have seen coming.')
+  @document2 = Lda::TextDocument.new(@corpus, 'When his brother is killed in a robbery, paraplegic Marine Jake Sully decides to take his place in a mission on the distant world of Pandora. There he learns of greedy corporate figurehead Parker Selfridge\'s intentions of driving off the native humanoid \"Na\'vi\" in order to mine for the precious material scattered throughout their rich woodland. In exchange for the spinal surgery that will fix his legs, Jake gathers intel for the cooperating military unit spearheaded by gung-ho Colonel Quaritch, while simultaneously attempting to infiltrate the Na\'vi people with the use of an \"avatar\" identity. While Jake begins to bond with the native tribe and quickly falls in love with the beautiful alien Neytiri, the restless Colonel moves forward with his ruthless extermination tactics, forcing the soldier to take a stand - and fight back in an epic battle for the fate of Pandora.')
+
+  @corpus.add_document(@document1)
+  @corpus.add_document(@document2)
+  @corpus.remove_word("cobb")
+  @lda = Lda::Lda.new(@corpus)
+
+  @lda.verbose = false
+  @lda.num_topics = 2
+  @lda.em('random')
+  topics = @lda.top_words(5)
+  puts topics
+
+end

data/test/simple_yaml.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+require 'shoulda'
+require 'yaml'
+require 'lda-ruby'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+class Test::Unit::TestCase
+
+  @filename = File.join(File.dirname(__FILE__), 'data', 'wiki-test-docs.yml')
+  @filedocs = YAML::load_file(@filename)
+  @corpus = Lda::TextCorpus.new(@filename)
+
+  @lda = Lda::Lda.new(@corpus)
+
+  @lda.verbose = false
+  @lda.num_topics = 20
+  @lda.em('random')
+  @lda.print_topics(20)
+
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lda-ruby
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
   prerelease: 
 platform: ruby
 authors:
@@ -11,12 +11,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-03 00:00:00.000000000 -04:00
+date: 2011-08-05 00:00:00.000000000 -04:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &2153196880 !ruby/object:Gem::Requirement
+  requirement: &2153224820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -24,7 +24,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153196880
+  version_requirements: *2153224820
 description: Ruby port of Latent Dirichlet Allocation by David M. Blei. See http://www.cs.princeton.edu/~blei/lda-c/.
 email: jasonmadams@gmail.com
 executables: []
@@ -56,6 +56,7 @@ files:
 - ext/lda-ruby/utils.h
 - lda-ruby.gemspec
 - lib/lda-ruby.rb
+- lib/lda-ruby/config/stopwords.yml
 - lib/lda-ruby/corpus/corpus.rb
 - lib/lda-ruby/corpus/data_corpus.rb
 - lib/lda-ruby/corpus/directory_corpus.rb
@@ -70,6 +71,8 @@ files:
 - test/data/sample.rb
 - test/data/wiki-test-docs.yml
 - test/lda_ruby_test.rb
+- test/simple_test.rb
+- test/simple_yaml.rb
 - test/test_helper.rb
 has_rdoc: true
 homepage: http://github.com/ealdent/lda-ruby