rsemantic 0.1.4 → 0.2.0

data/README.md

@@ -17,11 +17,14 @@ Documentation: http://github.com/josephwilk/rsemantic/wikis/home
 ## INSTALL:
 
 Note 'brew install GSL' installs 1.15 which is not supported yet by the gsl gem. So you have to switch your GSL version to 1.14.
-See here for details on how to do that with homebrew: http://bretthard.in/2012/03/getting-related_posts-lsi-and-gsl-to-work-in-jekyll/
+With homebrew try this:
 
-<pre><code>git clone git://github.com/josephwilk/rsemantic.git
+<pre><code>
+git clone git://github.com/josephwilk/rsemantic.git
 cd rsemantic
-brew install GSL
+
+brew tap homebrew/versions
+brew install gsl114
 bundle install
 </code></pre>
 

data/TODO.txt

@@ -1,9 +1,12 @@
 == FEATURES/PROBLEMS:
 
 * Applying transforms to query vectors
-* Detect the optimal dimension reduction in LSA.
 * Allow objects to be passed in as transforms.
+  * Hashes might be enough, but a faster data structure might be a good option.
+* Detect the optimal dimension reduction in LSA.
+  * This needs some benchmark. Low number of dimensions can be effective enough.
+  * http://nlp.stanford.edu/IR-book/html/htmledition/latent-semantic-indexing-1.html
 * Implement Probabilistic latent semantic analysis
 * Implement Latent Dirichlet Allocation
 
-* Matrix transformer has to popout the matrix of VectorSpace::Model and reassign it, get rid of this.
+* Matrix transformer has to popout the matrix of VectorSpace::Model and reassign it, get rid of this.

data/lib/semantic/parser.rb

@@ -1,5 +1,5 @@
-require 'stemmer'
 require "set"
+
 module Semantic
   class Parser
 
@@ -8,8 +8,10 @@ module Semantic
       # TODO: nicer way to reference stop file location?
       @filter_stop_words = options[:filter_stop_words]
       @stem_words        = options[:stem_words]
+      locale             = options[:locale] || 'en'
+
       if @filter_stop_words
-        File.open(File.dirname(__FILE__) + '/../../resources/english.stop', 'r') do |file|
+        File.open("#{File.dirname(__FILE__)}/../../resources/#{locale}.stop", 'r') do |file|
           @stopwords = Set.new(file.read().split())
         end
       end
@@ -42,7 +44,7 @@ module Semantic
       words = string.split(" ")
 
       if @stem_words
-        words.map {|word| Stemmer.stem_word(word) }
+        words.map(&:stem)
       else
         words
       end

data/lib/semantic/transform/tf_idf_transform.rb

@@ -8,13 +8,10 @@ module Semantic
         @@number_of_documents_with_term = []
 
         matrix.transpose.enum_for(:each_row).with_index do |document, column_index|
-          document_term_total = document.sum
-
           document.enum_for(:each).with_index do |term_weight, row_index|
             unless term_weight == 0.0
-              inverse_document_frequency = GSL::Sf.log((number_of_documents /
-                                                        number_of_documents_with_term(row_index, matrix).to_f).abs)
-              term_frequency = (term_weight / document_term_total)
+              inverse_document_frequency = 1 + GSL::Sf.log(number_of_documents / (number_of_documents_with_term(row_index, matrix).to_f + 1))
+              term_frequency = Math.sqrt(term_weight)
 
               matrix[row_index, column_index] = term_frequency * inverse_document_frequency
             end

data/lib/semantic/vector_space/builder.rb

@@ -69,7 +69,7 @@ module Semantic
           end
         }
 
-        vector
+        vector.respond_to?(:to_v) ? vector.to_v : vector
       end
     end
   end

data/lib/semantic/vector_space/model.rb

@@ -22,7 +22,7 @@ module Semantic
 
       def to_s
         out = StringIO.new
-        out.print " " * 9
+        out.print " " * 12
 
         matrix.size2.times do |id|
           out.print "  D#{id+1}  "
@@ -30,8 +30,13 @@ module Semantic
         out.puts
 
         matrix.to_a.each_with_index do |terms, index|
-          # TODO fix for 1.8.7
-          out.print "#{@keywords.key(index).ljust(6)}" if @keywords.has_value?(index)
+
+          if @keywords.has_value?(index)
+            index_position = @keywords.values.index(index)
+            key = @keywords.keys[index_position]
+
+            out.print "#{key.ljust(10)}"
+          end
           out.print "[ "
 
           terms.each do |document|
@@ -40,6 +45,7 @@ module Semantic
           out.print "]"
           out.puts
         end
+
         out.string
       end
 

data/lib/semantic/version.rb

@@ -1,8 +1,8 @@
 module Semantic #:nodoc:
   class VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 1
-    TINY  = 4
+    MINOR = 2
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/tasks/rspec.rake

@@ -1,11 +1,15 @@
 require 'rspec/core/rake_task'
 
 desc 'Default: run specs.'
-task :default => :spec
+task :default => [:spec, :integration]
 
 desc "Run specs"
 RSpec::Core::RakeTask.new 
 
+RSpec::Core::RakeTask.new(:integration) do |t|
+  t.pattern = 'spec_integration/*_spec.rb'
+end
+
 desc "Generate code coverage"
 RSpec::Core::RakeTask.new(:coverage) do |t|
   t.rcov = true

data/resources/ar.stop

@@ -0,0 +1,162 @@
+ب
+ا
+أ
+،
+عشر
+عدد
+عدة
+عشرة
+عدم
+عام
+عاما
+عن
+عند
+عندما
+على
+عليه
+عليها
+زيارة
+سنة
+سنوات
+تم
+ضد
+بعد
+بعض
+اعادة
+اعلنت
+بسبب
+حتى
+اذا
+احد
+اثر
+برس
+باسم
+غدا
+شخصا
+صباح
+اطار
+اربعة
+اخرى
+بان
+اجل
+غير
+بشكل
+حاليا
+بن
+به
+ثم
+اف
+ان
+او
+اي
+بها
+صفر
+حيث
+اكد
+الا
+اما
+امس
+السابق
+التى
+التي
+اكثر
+ايار
+ايضا
+ثلاثة
+الذاتي
+الاخيرة
+الثاني
+الثانية
+الذى
+الذي
+الان
+امام
+ايام
+خلال
+حوالى
+الذين
+الاول
+الاولى
+بين
+ذلك
+دون
+حول
+حين
+الف
+الى
+انه
+اول
+ضمن
+انها
+جميع
+الماضي
+الوقت
+المقبل
+اليوم
+ـ
+ف
+و
+و6
+قد
+لا
+ما
+مع
+مساء
+هذا
+واحد
+واضاف
+واضافت
+فان
+قبل
+قال
+كان
+لدى
+نحو
+هذه
+وان
+واكد
+كانت
+واوضح
+مايو
+فى
+في
+كل
+لم
+لن
+له
+من
+هو
+هي
+قوة
+كما
+لها
+منذ
+وقد
+ولا
+نفسه
+لقاء
+مقابل
+هناك
+وقال
+وكان
+نهاية
+وقالت
+وكانت
+للامم
+فيه
+كلم
+لكن
+وفي
+وقف
+ولم
+ومن
+وهو
+وهي
+يوم
+فيها
+منها
+مليار
+لوكالة
+يكون
+يمكن
+مليون

data/resources/ca.stop

@@ -0,0 +1,124 @@
+de
+es
+i
+a
+o
+un
+una
+unes
+uns
+un
+tot
+també
+altre
+algun
+alguna
+alguns
+algunes
+ser
+és
+soc
+ets
+som
+estic
+està
+estem
+esteu
+estan
+com
+en
+per
+perquè
+per que
+estat
+estava
+ans
+abans
+éssent
+ambdós
+però
+per
+poder
+potser
+puc
+podem
+podeu
+poden
+vaig
+va
+van
+fer	faig
+fa
+fem
+feu
+fan
+cada
+fi
+inclòs
+primer
+des de
+conseguir
+consegueixo
+consigueix
+consigueixes
+conseguim
+consigueixen
+anar
+haver
+tenir
+tinc
+te
+tenim
+teniu
+tene
+el
+la
+les
+els
+seu
+aquí
+meu
+teu
+ells
+elles
+ens
+nosaltres
+vosaltres
+si
+dins
+sols
+solament
+saber
+saps
+sap
+sabem
+sabeu
+saben
+últim
+llarg
+bastant	fas
+molts
+aquells
+aquelles
+seus
+llavors
+sota
+dalt
+ús
+molt
+era
+eres
+erem
+eren
+mode
+bé
+quant
+quan
+on
+mentre
+qui
+amb
+entre
+sense
+jo
+aquell

data/resources/cs.stop

@@ -0,0 +1,136 @@
+dnes
+cz
+timto
+budes
+budem
+byli
+jses
+muj
+svym
+ta
+tomto
+tohle
+tuto
+tyto
+jej
+zda
+proc
+mate
+tato
+kam
+tohoto
+kdo
+kteri
+mi
+nam
+tom
+tomuto
+mit
+nic
+proto
+kterou
+byla
+toho
+protoze
+asi
+ho
+nasi
+napiste
+re
+coz
+tim
+takze
+svych
+jeji
+svymi
+jste
+aj
+tu
+tedy
+teto	bylo
+kde
+ke
+prave
+ji
+nad
+nejsou
+ci
+pod
+tema
+mezi
+pres
+ty
+pak
+vam
+ani
+kdyz
+vsak
+ne
+jsem
+tento
+clanku
+clanky
+aby
+jsme
+pred
+pta
+jejich
+byl
+jeste
+az
+bez
+take
+pouze
+prvni
+vase
+ktera
+nas
+novy
+tipy
+pokud
+muze
+design
+strana
+jeho
+sve
+jine
+zpravy
+nove
+neni	vas
+jen
+podle
+zde
+clanek
+uz
+email
+byt
+vice
+bude
+jiz
+nez
+ktery
+by
+ktere
+co
+nebo
+ten
+tak
+ma
+pri
+od
+po
+jsou
+jak
+dalsi
+ale
+si
+ve
+to
+jako
+za
+zpet
+ze
+do
+pro
+je
+na