lda-ruby 0.3.8 → 0.3.9

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    M2IzN2IwMmE1YWFhNTJjYjk0ZjMyYzhjMWQyZDdjZjY2YTA0OGNjZA==
+  data.tar.gz: !binary |-
+    OTNjZjE5MGNmOGI2YzY3YzhlNDRiYTBlNDM5NmUwYmY4Mjc2ZmNkNQ==
+SHA512:
+  metadata.gz: !binary |-
+    MDAxZmQ1N2U5MGQ1MDA2MGE4YTU0N2NkN2Y4N2Q4YzlmNTdmYTkxN2FlYzU0
+    ZmFjZDZiZWIwZmRiMDJlYjNjYTg5YWQ0N2RlOTY1MTg0YzZjZTc0NGQ0YmI1
+    ZDljMjljNTA3ODdlZjBjNjNjMTc0MGRlMjBhODQzZTg3YWM5OWE=
+  data.tar.gz: !binary |-
+    OGFiMzVhMzY4OTFmZTkzMmM3YjQxMzNkM2JlMjVjOTRjM2ZhNTc1YmUxZTRj
+    Y2M3YzZiNzNiYmVkNDI1ZTUxODhkMjhlYTQ4MmRhZjVkZjQxMDhjOTk5Njc4
+    OTZhOTM5ZTQ3OTFhY2U1YjRhODQyYzBkZWNlYzhjNzBjMWFlNGU=

data/{CHANGELOG → CHANGELOG.md}

@@ -1,3 +1,15 @@
+version 0.3.9
+=============
+
+- merge pull request from @rishabh-tripathi allowing text corpus objects to also be built with an array of strings
+- couple minor code refinements
+
+version 0.3.8
+=============
+
+- tokenization changes to support German (courtesy of @LeFnord)
+- user defined stop word list (also via @LeFnord)
+
 version 0.3.7
 =============
 

data/{README.markdown → README.md}

@@ -13,7 +13,7 @@ The original C code relied on files for the input and output. We felt it was nec
     lda = Lda::Lda.new(corpus)    # create an Lda object for training
     lda.em("random")              # run EM algorithm using random starting points
     lda.load_vocabulary("data/vocab.txt")
-    lda.print_topics(20)          # print the topic 20 words per topic
+    lda.print_topics(20)          # print all topics with up to 20 words per topic
 
 If you have general questions about Latent Dirichlet Allocation, I urge you to use the [topic models mailing list][topic-models], since the people who monitor that are very knowledgeable.  If you encounter bugs specific to lda-ruby, please post an issue on the Github project.
 
@@ -34,4 +34,4 @@ Blei, David M., Ng, Andrew Y., and Jordan, Michael I. 2003. Latent dirichlet all
 [wikipedia]: http://en.wikipedia.org/wiki/Latent_Dirichlet_allocation
 [ap-data]: http://www.cs.princeton.edu/~blei/lda-c/ap.tgz
 [pdf]: http://www.cs.princeton.edu/picasso/mats/BleiNgJordan2003_blei.pdf
-[topic-models]: https://lists.cs.princeton.edu/mailman/listinfo/topic-models
+[topic-models]: https://lists.cs.princeton.edu/mailman/listinfo/topic-models

data/VERSION.yml

@@ -1,5 +1,5 @@
---- 
-:build: 
+---
 :major: 0
 :minor: 3
-:patch: 8
+:patch: 9
+:build: 

data/lib/lda-ruby/corpus/text_corpus.rb

@@ -2,18 +2,23 @@ module Lda
   class TextCorpus < Corpus
     attr_reader :filename
 
-    # Load text documents from YAML file if filename is given.
-    def initialize(filename)
+    # Loads text documents from a YAML file or an array of strings
+    def initialize(input_data)
       super()
 
-      @filename = filename
-      load_from_file
-    end
-
-    protected
+      docs = if input_data.is_a?(String) && File.exists?(input_data)
+        # yaml file containing an array of strings representing each document
+        YAML.load_file(input_data)
+      elsif input_data.is_a?(Array)
+        # an array of strings representing each document
+        input_data.dup
+      elsif input_data.is_a?(String)
+        # a single string representing one document
+        [input_data]
+      else
+        raise "Unknown input type: please pass in a valid filename or an array of strings."
+      end
 
-    def load_from_file
-      docs = YAML.load_file(@filename)
       docs.each do |doc|
         add_document(TextDocument.new(self, doc))
       end

data/lib/lda-ruby/document/document.rb

@@ -32,9 +32,7 @@ module Lda
     end
 
     def tokenize(text)
-      # now respects Umlaute
       clean_text = text.gsub(/[^a-zäöüß'-]+/i, ' ').gsub(/\s+/, ' ').downcase  # remove everything but letters and ' and leave only single spaces
-      # clean_text = text.gsub(/[^A-Za-z'\s]+/, ' ').gsub(/\s+/, ' ').downcase        # remove everything but letters and ' and leave only single spaces
       @tokens = handle(clean_text.split(' '))
       nil
     end

metadata

@@ -1,52 +1,41 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: lda-ruby
-version: !ruby/object:Gem::Version 
-  hash: 3
-  prerelease: 
-  segments: 
-  - 0
-  - 3
-  - 8
-  version: 0.3.8
+version: !ruby/object:Gem::Version
+  version: 0.3.9
 platform: ruby
-authors: 
+authors:
 - David Blei
 - Jason Adams
 - Rio Akasaka
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-10-18 00:00:00 -04:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2015-02-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: shoulda
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Ruby port of Latent Dirichlet Allocation by David M. Blei. See http://www.cs.princeton.edu/~blei/lda-c/.
 email: jasonmadams@gmail.com
 executables: []
-
-extensions: 
+extensions:
 - ext/lda-ruby/extconf.rb
-extra_rdoc_files: 
-- README
-- README.markdown
-files: 
-- CHANGELOG
-- README
-- README.markdown
+extra_rdoc_files:
+- README.md
+files:
+- CHANGELOG.md
+- README.md
 - Rakefile
 - VERSION.yml
 - ext/lda-ruby/Makefile
@@ -84,40 +73,28 @@ files:
 - test/simple_test.rb
 - test/simple_yaml.rb
 - test/test_helper.rb
-has_rdoc: true
 homepage: http://github.com/ealdent/lda-ruby
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
 - ext
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.5.2
+rubygems_version: 2.4.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Ruby port of Latent Dirichlet Allocation by David M. Blei.
 test_files: []
-

data/README

@@ -1,21 +0,0 @@
-Latent Dirichlet Allocation – Ruby Wrapper
-
-This wrapper is based on C-code by David M. Blei. In a nutshell, it can be used to automatically cluster documents into topics. The number of topics are chosen beforehand and the topics found are usually fairly intuitive. Details of the implementation can be found in the paper by Blei, Ng, and Jordan.
-
-The original C code relied on files for the input and output. We felt it was necessary to depart from that model and use Ruby objects for these steps instead. The only file necessary will be the data file (in a format similar to that used by SVMlight). Optionally you may need a vocabulary file to be able to extract the words belonging to topics.
-
-Example usage:
-
-require 'lda'
-corpus = Lda::DataCorpus.new("data/data_file.dat")
-lda = Lda::Lda.new(corpus)      # create an Lda object for training
-lda.em("random")                # run EM algorithm using random starting points
-lda.load_vocabulary("data/vocab.txt")
-lda.print_topics(20)            # print the topic 20 words per topic
-
-You can check out the mailing list for this project if you have any questions or mail lda-ruby@groups.google.com [email link]. If you have general questions about Latent Dirichlet Allocation, I urge you to use the topic models mailing list, since the people who monitor that are very knowledgeable.
-
-
-References
-
-Blei, David M., Ng, Andrew Y., and Jordan, Michael I. 2003. Latent dirichlet allocation. Journal of Machine Learning Research. 3 (Mar. 2003), 993-1022.