lda-ruby 0.3.7 → 0.3.8

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
+:build: 
 :major: 0
 :minor: 3
-:patch: 7
-:build: 
+:patch: 8

data/lda-ruby.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{lda-ruby}
-  s.version = "0.3.7"
+  s.version = "0.3.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David Blei", "Jason Adams", "Rio Akasaka"]
-  s.date = %q{2011-08-06}
+  s.date = %q{2011-10-18}
   s.description = %q{Ruby port of Latent Dirichlet Allocation by David M. Blei. See http://www.cs.princeton.edu/~blei/lda-c/.}
   s.email = %q{jasonmadams@gmail.com}
   s.extensions = ["ext/lda-ruby/extconf.rb"]
@@ -61,7 +61,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = %q{http://github.com/ealdent/lda-ruby}
   s.require_paths = ["lib", "ext"]
-  s.rubygems_version = %q{1.6.2}
+  s.rubygems_version = %q{1.5.2}
   s.summary = %q{Ruby port of Latent Dirichlet Allocation by David M. Blei.}
 
   if s.respond_to? :specification_version then

data/lib/lda-ruby/corpus/corpus.rb

@@ -4,15 +4,19 @@ module Lda
   class Corpus
     attr_reader :documents, :num_docs, :num_terms, :vocabulary, :stopwords
 
-    def initialize
+    def initialize(stop_word_list = nil)
       @documents = Array.new
       @all_terms = Set.new
       @num_terms = @num_docs = 0
       @vocabulary = Vocabulary.new
-      @stopwords = YAML.load_file(File.join(File.dirname(__FILE__), '..', 'config', 'stopwords.yml'))
+      if stop_word_list.nil?
+        @stopwords = YAML.load_file(File.join(File.dirname(__FILE__), '..', 'config', 'stopwords.yml'))
+      else
+        @stopwords = YAML.load_file(stop_word_list)
+      end
       @stopwords.map! { |w| w.strip }
     end
-
+    
     def add_document(doc)
       raise 'Parameter +doc+ must be of type Document' unless doc.kind_of?(Document)
 

data/lib/lda-ruby/document/document.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require 'yaml'
 
 module Lda
@@ -31,7 +32,9 @@ module Lda
     end
 
     def tokenize(text)
-      clean_text = text.gsub(/[^A-Za-z'\s]+/, ' ').gsub(/\s+/, ' ').downcase        # remove everything but letters and ' and leave only single spaces
+      # now respects Umlaute
+      clean_text = text.gsub(/[^a-zäöüß'-]+/i, ' ').gsub(/\s+/, ' ').downcase  # remove everything but letters and ' and leave only single spaces
+      # clean_text = text.gsub(/[^A-Za-z'\s]+/, ' ').gsub(/\s+/, ' ').downcase        # remove everything but letters and ' and leave only single spaces
       @tokens = handle(clean_text.split(' '))
       nil
     end

metadata

@@ -1,39 +1,49 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: lda-ruby
-version: !ruby/object:Gem::Version
-  version: 0.3.7
+version: !ruby/object:Gem::Version 
+  hash: 3
   prerelease: 
+  segments: 
+  - 0
+  - 3
+  - 8
+  version: 0.3.8
 platform: ruby
-authors:
+authors: 
 - David Blei
 - Jason Adams
 - Rio Akasaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-06 00:00:00.000000000 -04:00
+
+date: 2011-10-18 00:00:00 -04:00
 default_executable: 
-dependencies:
-- !ruby/object:Gem::Dependency
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: shoulda
-  requirement: &2153174960 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :runtime
-  prerelease: false
-  version_requirements: *2153174960
+  version_requirements: *id001
 description: Ruby port of Latent Dirichlet Allocation by David M. Blei. See http://www.cs.princeton.edu/~blei/lda-c/.
 email: jasonmadams@gmail.com
 executables: []
-extensions:
+
+extensions: 
 - ext/lda-ruby/extconf.rb
-extra_rdoc_files:
+extra_rdoc_files: 
 - README
 - README.markdown
-files:
+files: 
 - CHANGELOG
 - README
 - README.markdown
@@ -77,27 +87,37 @@ files:
 has_rdoc: true
 homepage: http://github.com/ealdent/lda-ruby
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
 - ext
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
-rubygems_version: 1.6.2
+rubygems_version: 1.5.2
 signing_key: 
 specification_version: 3
 summary: Ruby port of Latent Dirichlet Allocation by David M. Blei.
 test_files: []
+