lda-ruby 0.4.0-x86_64-linux

data/lib/lda-ruby/config/stopwords.yml

@@ -0,0 +1,571 @@
+--- 
+- a
+- a's
+- able
+- about
+- above
+- according
+- accordingly
+- across
+- actually
+- after
+- afterwards
+- again
+- against
+- ain't
+- all
+- allow
+- allows
+- almost
+- alone
+- along
+- already
+- also
+- although
+- always
+- am
+- among
+- amongst
+- an
+- and
+- another
+- any
+- anybody
+- anyhow
+- anyone
+- anything
+- anyway
+- anyways
+- anywhere
+- apart
+- appear
+- appreciate
+- appropriate
+- are
+- aren't
+- around
+- as
+- aside
+- ask
+- asking
+- associated
+- at
+- available
+- away
+- awfully
+- b
+- be
+- became
+- because
+- become
+- becomes
+- becoming
+- been
+- before
+- beforehand
+- behind
+- being
+- believe
+- below
+- beside
+- besides
+- best
+- better
+- between
+- beyond
+- both
+- brief
+- but
+- by
+- c
+- c'mon
+- c's
+- came
+- can
+- can't
+- cannot
+- cant
+- cause
+- causes
+- certain
+- certainly
+- changes
+- clearly
+- co
+- com
+- come
+- comes
+- concerning
+- consequently
+- consider
+- considering
+- contain
+- containing
+- contains
+- corresponding
+- could
+- couldn't
+- course
+- currently
+- d
+- definitely
+- described
+- despite
+- did
+- didn't
+- different
+- do
+- does
+- doesn't
+- doing
+- don't
+- done
+- down
+- downwards
+- during
+- e
+- each
+- edu
+- eg
+- eight
+- either
+- else
+- elsewhere
+- enough
+- entirely
+- especially
+- et
+- etc
+- even
+- ever
+- every
+- everybody
+- everyone
+- everything
+- everywhere
+- ex
+- exactly
+- example
+- except
+- f
+- far
+- few
+- fifth
+- first
+- five
+- followed
+- following
+- follows
+- for
+- former
+- formerly
+- forth
+- four
+- from
+- further
+- furthermore
+- g
+- get
+- gets
+- getting
+- given
+- gives
+- go
+- goes
+- going
+- gone
+- got
+- gotten
+- greetings
+- h
+- had
+- hadn't
+- happens
+- hardly
+- has
+- hasn't
+- have
+- haven't
+- having
+- he
+- he's
+- hello
+- help
+- hence
+- her
+- here
+- here's
+- hereafter
+- hereby
+- herein
+- hereupon
+- hers
+- herself
+- hi
+- him
+- himself
+- his
+- hither
+- hopefully
+- how
+- howbeit
+- however
+- i
+- i'd
+- i'll
+- i'm
+- i've
+- ie
+- if
+- ignored
+- immediate
+- in
+- inasmuch
+- inc
+- indeed
+- indicate
+- indicated
+- indicates
+- inner
+- insofar
+- instead
+- into
+- inward
+- is
+- isn't
+- it
+- it'd
+- it'll
+- it's
+- its
+- itself
+- j
+- just
+- k
+- keep
+- keeps
+- kept
+- know
+- knows
+- known
+- l
+- last
+- lately
+- later
+- latter
+- latterly
+- least
+- less
+- lest
+- let
+- let's
+- like
+- liked
+- likely
+- little
+- look
+- looking
+- looks
+- ltd
+- m
+- mainly
+- many
+- may
+- maybe
+- me
+- mean
+- meanwhile
+- merely
+- might
+- more
+- moreover
+- most
+- mostly
+- much
+- must
+- my
+- myself
+- n
+- name
+- namely
+- nd
+- near
+- nearly
+- necessary
+- need
+- needs
+- neither
+- never
+- nevertheless
+- new
+- next
+- nine
+- "no"
+- nobody
+- non
+- none
+- noone
+- nor
+- normally
+- not
+- nothing
+- novel
+- now
+- nowhere
+- o
+- obviously
+- of
+- "off"
+- often
+- oh
+- ok
+- okay
+- old
+- "on"
+- once
+- one
+- ones
+- only
+- onto
+- or
+- other
+- others
+- otherwise
+- ought
+- our
+- ours
+- ourselves
+- out
+- outside
+- over
+- overall
+- own
+- p
+- particular
+- particularly
+- per
+- perhaps
+- placed
+- please
+- plus
+- possible
+- presumably
+- probably
+- provides
+- q
+- que
+- quite
+- qv
+- r
+- rather
+- rd
+- re
+- really
+- reasonably
+- regarding
+- regardless
+- regards
+- relatively
+- respectively
+- right
+- s
+- said
+- same
+- saw
+- say
+- saying
+- says
+- second
+- secondly
+- see
+- seeing
+- seem
+- seemed
+- seeming
+- seems
+- seen
+- self
+- selves
+- sensible
+- sent
+- serious
+- seriously
+- seven
+- several
+- shall
+- she
+- should
+- shouldn't
+- since
+- six
+- so
+- some
+- somebody
+- somehow
+- someone
+- something
+- sometime
+- sometimes
+- somewhat
+- somewhere
+- soon
+- sorry
+- specified
+- specify
+- specifying
+- still
+- sub
+- such
+- sup
+- sure
+- t
+- t's
+- take
+- taken
+- tell
+- tends
+- th
+- than
+- thank
+- thanks
+- thanx
+- that
+- that's
+- thats
+- the
+- their
+- theirs
+- them
+- themselves
+- then
+- thence
+- there
+- there's
+- thereafter
+- thereby
+- therefore
+- therein
+- theres
+- thereupon
+- these
+- they
+- they'd
+- they'll
+- they're
+- they've
+- think
+- third
+- this
+- thorough
+- thoroughly
+- those
+- though
+- three
+- through
+- throughout
+- thru
+- thus
+- to
+- together
+- too
+- took
+- toward
+- towards
+- tried
+- tries
+- truly
+- try
+- trying
+- twice
+- two
+- u
+- un
+- under
+- unfortunately
+- unless
+- unlikely
+- until
+- unto
+- up
+- upon
+- us
+- use
+- used
+- useful
+- uses
+- using
+- usually
+- v
+- value
+- various
+- very
+- via
+- viz
+- vs
+- w
+- want
+- wants
+- was
+- wasn't
+- way
+- we
+- we'd
+- we'll
+- we're
+- we've
+- welcome
+- well
+- went
+- were
+- weren't
+- what
+- what's
+- whatever
+- when
+- whence
+- whenever
+- where
+- where's
+- whereafter
+- whereas
+- whereby
+- wherein
+- whereupon
+- wherever
+- whether
+- which
+- while
+- whither
+- who
+- who's
+- whoever
+- whole
+- whom
+- whose
+- why
+- will
+- willing
+- wish
+- with
+- within
+- without
+- won't
+- wonder
+- would
+- would
+- wouldn't
+- x
+- y
+- "yes"
+- yet
+- you
+- you'd
+- you'll
+- you're
+- you've
+- your
+- yours
+- yourself
+- yourselves
+- z
+- zero

data/lib/lda-ruby/corpus/corpus.rb

@@ -0,0 +1,45 @@
+require "set"
+require "yaml"
+
+module Lda
+  class Corpus
+    attr_reader :documents, :num_docs, :num_terms, :vocabulary, :stopwords
+
+    def initialize(stop_word_list = nil)
+      @documents = []
+      @all_terms = Set.new
+      @num_terms = @num_docs = 0
+      @vocabulary = Vocabulary.new
+      @stopwords =  if stop_word_list.nil?
+                      File.join(File.dirname(__FILE__), '..', 'config', 'stopwords.yml')
+                    else
+                      stop_word_list
+                    end
+      @stopwords = YAML.load_file(@stopwords)
+      @stopwords.map!(&:strip)
+    end
+
+    def add_document(doc)
+      raise 'Parameter +doc+ must be of type Document' unless doc.is_a?(Document)
+
+      @documents << doc
+
+      @all_terms += doc.words
+      @num_docs += 1
+      @num_terms = @all_terms.size
+
+      update_vocabulary(doc)
+      nil
+    end
+
+    def remove_word(word)
+      @vocabulary.words.delete word
+    end
+
+    protected
+
+    def update_vocabulary(doc)
+      doc.tokens.each { |w| @vocabulary.check_word(w) }
+    end
+  end
+end

data/lib/lda-ruby/corpus/data_corpus.rb

@@ -0,0 +1,22 @@
+module Lda
+  class DataCorpus < Corpus
+    attr_reader :filename
+
+    def initialize(filename)
+      super()
+
+      @filename = filename
+      load_from_file
+    end
+
+    protected
+
+    def load_from_file
+      txt = File.open(@filename, 'r', &:read)
+      lines = txt.split(/[\r\n]+/)
+      lines.each do |line|
+        add_document(DataDocument.new(self, line))
+      end
+    end
+  end
+end

data/lib/lda-ruby/corpus/directory_corpus.rb

@@ -0,0 +1,25 @@
+module Lda
+  class DirectoryCorpus < Corpus
+    attr_reader :path, :extension
+
+    # load documents from a directory
+    def initialize(path, extension = nil)
+      super()
+
+      @path = path.dup.freeze
+      @extension = extension ? extension.dup.freeze : nil
+
+      load_from_directory
+    end
+
+    protected
+
+    def load_from_directory
+      dir_glob = File.join(@path, (@extension ? "*.#{@extension}" : '*'))
+
+      Dir.glob(dir_glob).each do |filename|
+        add_document(TextDocument.build_from_file(self, filename))
+      end
+    end
+  end
+end

data/lib/lda-ruby/corpus/text_corpus.rb

@@ -0,0 +1,27 @@
+module Lda
+  class TextCorpus < Corpus
+    attr_reader :filename
+
+    # Loads text documents from a YAML file or an array of strings
+    def initialize(input_data)
+      super()
+
+      docs = if input_data.is_a?(String) && File.exist?(input_data)
+        # yaml file containing an array of strings representing each document
+        YAML.load_file(input_data)
+      elsif input_data.is_a?(Array)
+        # an array of strings representing each document
+        input_data.dup
+      elsif input_data.is_a?(String)
+        # a single string representing one document
+        [input_data]
+      else
+        raise 'Unknown input type: please pass in a valid filename or an array of strings.'
+      end
+
+      docs.each do |doc|
+        add_document(TextDocument.new(self, doc))
+      end
+    end
+  end
+end

data/lib/lda-ruby/document/data_document.rb

@@ -0,0 +1,30 @@
+#
+# Create the Document using the svmlight-style text line:
+#
+#   num_words w1:freq1 w2:freq2 ... w_n:freq_n
+#
+# Ex.
+#   5 1:2 3:1 4:2 7:3 12:1
+#
+# The value for the number of words should equal the number of pairs
+# following it, though this isn't at all enforced.  Order of word-pair
+# indices is not important.
+#
+
+module Lda
+  class DataDocument < Document
+    def initialize(corpus, data)
+      super(corpus)
+
+      items = data.split(/\s+/)
+      pairs = items[1..items.size].map { |item| item.split(':') }
+
+      pairs.each do |feature_identifier, feature_weight|
+        @words << feature_identifier.to_i
+        @counts << feature_weight.to_i
+      end
+
+      recompute
+    end
+  end
+end