tfidf 0.0.0 → 0.0.1

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/examples/demo_tf.rb

@@ -127,10 +127,10 @@ puts "Terms in the corpus"
 tfidf.terms.each {|k,v| puts "Term ID: #{k} => term: #{v}"}
 
 puts "Document-Term Matrix, sparse List of lists(LIL)"
-tfidf.sparse_matrix_doc_idx.each {|e| puts e}
+tfidf.dtm.each {|e| puts e}
 
 puts "Term-Document Matrix, sparse LIL"
-tfidf.sparse_matrix_term_idx.each {|e| puts e}
+tfidf.tdm.each {|e| puts e}
 
 puts "Term Frequency of word: video in document"
 puts tfidf.tf
@@ -140,3 +140,9 @@ puts tfidf.idf("octocat")
 
 puts "TF-IDF of word : octocats in document 7e38fa195cee92d2e7d834095d6938a89b5fdd58"
 puts tfidf.tfidf("octocat","7e38fa195cee92d2e7d834095d6938a89b5fdd58")
+
+puts "DTM in dense matrix"
+puts tfidf.dense
+
+
+

data/lib/tfidf.rb

@@ -1,6 +1,8 @@
 require 'set'
 require 'fast_stemmer'
 require 'digest'
+require 'active_support'
+require 'gsl'
 
 
 class TFIDF
@@ -9,7 +11,7 @@ class TFIDF
   @@split_pattern = /[\W]/
   
   #Hash function used for generating id for documents as well as terms
-  def hash_func(obj)
+  def self.hash_func(obj)
     return Digest::SHA1.hexdigest obj
   end
 
@@ -26,9 +28,8 @@ class TFIDF
   def initialize(corpus)
     @cardinality = 0
     @docs = {}
-    @terms = {}
-    @sparse_matrix_term_idx = {}
-    @sparse_matrix_doc_idx = {}
+    @tdm = {}
+    @dtm = {}
     @idf = {}
 
     #not in use
@@ -41,33 +42,34 @@ class TFIDF
     else
       @cardinality = corpus.length
     end
-    memo = corpus.reduce({:terms => {}, :docs => {}, :sparse_matrix_doc_idx => {}, :sparse_matrix_term_idx => {}}) do |memo, doc|
-      doc_id = hash_func doc
+    memo = corpus.reduce({:terms => SortedSet.new, :docs => ActiveSupport::OrderedHash.new({}), :dtm => {}, :tdm => {}}) do |memo, doc|
+      doc_id = TFIDF.hash_func doc
       memo[:docs][doc_id] = doc
       tf_single_doc = TFIDF.tf_single(doc)
-      memo[:sparse_matrix_doc_idx][doc_id] = tf_single_doc
+      memo[:dtm][doc_id] = tf_single_doc
       tf_single_doc.each do |keyvalue| 
         term, freq = keyvalue
-        term_id = hash_func term
-        lambda {|x|
-          if !x.has_key?(term_id)
-            x[term_id] = term
-          end}.call memo[:terms]
+#        term_id = TFIDF.hash_func term
+        memo[:terms].add term
         lambda {|x|
           if x[term] != nil
             x[term][doc_id] = freq
           else
             x[term] = {doc_id => freq}
           end
-        }.call memo[:sparse_matrix_term_idx]
+        }.call memo[:tdm]
       end
       memo
     end
     @docs = memo[:docs]
-    @terms = memo[:terms]
-    @sparse_matrix_term_idx = memo[:sparse_matrix_term_idx]
-    @sparse_matrix_doc_idx = memo[:sparse_matrix_doc_idx]
-    @sparse_matrix_term_idx.each {|k, v|
+    @terms = ActiveSupport::OrderedHash.new({})
+    memo[:terms].each do |term|
+      term_id = TFIDF.hash_func term
+      @terms[term_id] = term
+    end
+    @tdm = memo[:tdm]
+    @dtm = memo[:dtm]
+    @tdm.each {|k, v|
       @idf[k] = TFIDF.idf(v.size, @cardinality)}
   end
 
@@ -112,14 +114,14 @@ class TFIDF
     return @terms
   end
   
-  #Aka DTM, in sparse List of lists(LIL)
-  def sparse_matrix_doc_idx
-    return @sparse_matrix_doc_idx
+  #Document Term Matrix, in sparse List of lists(LIL)
+  def dtm
+    return @dtm
   end
 
-  #Aka TDM, in sparse List of lists(LIL)
-  def sparse_matrix_term_idx
-    return @sparse_matrix_term_idx
+  #Term Document Matrix, in sparse List of lists(LIL)
+  def tdm
+    return @tdm
   end
 
   #=Arguments
@@ -143,11 +145,11 @@ class TFIDF
   #  Everything
   def tf(term=nil, doc=nil)
     if term == nil || doc == nil
-      return @sparse_matrix_doc_idx
+      return @dtm
     elsif term == nil
-      return @sparse_matrix_doc_idx[doc]
+      return @dtm[doc]
     else
-      return lambda {|x| (x == nil)?0:x}.call(@sparse_matrix_doc_idx[doc][term])
+      return lambda {|x| (x == nil)?0:x}.call(@dtm[doc][term])
     end
   end
 
@@ -167,6 +169,28 @@ class TFIDF
   def tfidf(term, doc)
     return tf(term,doc) * idf(term)
   end
+
+  #Access, or calculate if not present, a dense DTM in GSL::Matrix
+  #Each row corresponds to a document, each column a term
+  #Use TFIDF#terms.values and TFIDF#docs.values to find column/row index of a specific document/term
+  def dense()
+    return @dense_matrix unless @dense_matrix.nil?
+    dense_matrix = GSL::Matrix.alloc(@docs.size, @terms.size)
+    (0...@docs.size).each do |i|
+      doc_id = @docs.keys[i]
+      @dtm[doc_id].each do |term,freq|
+        idx = @terms.values.index term
+        dense_matrix.set([i,idx], freq)
+      end
+    end
+    @dense_matrix = dense_matrix
+    return @dense_matrix
+  end
+
+  #TODO: Merge 2 TFIDF objects
+  def self.merge()
+
+  end
     
   #Simply the formula for tf*idf
   def self.idf(x,cardinality)

data/test/test_TFIDF.rb

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 require 'test/unit'
 require 'tfidf'
-=begin
+
 require 'ruby-debug'
 Debugger.start(:post_mortem => true)
 Debugger.settings[:autoeval] = true
-=end
+
 
 class TFIDFTest < Test::Unit::TestCase
   def setup
@@ -16,7 +16,7 @@ class TFIDFTest < Test::Unit::TestCase
   def test_arbitrary_text
     #Just an arbitrary test on a single text, the number isn't definitive
     tfidf = TFIDF.new @@text1
-    pp tfidf.terms.size > 50
+    assert tfidf.terms.size > 50
   end
 
 
@@ -38,6 +38,19 @@ class TFIDFTest < Test::Unit::TestCase
     assert TFIDF.idf(1,2) == 0
     assert TFIDF.idf(0,2) == 1
   end
+
+  def test_terms_should_be_sorted
+    it = @tfidf.terms.values.each
+    begin
+      while true
+        e1 = it.next
+        e2 = it.peek
+        assert e2 > e1
+      end
+    rescue StopIteration => stop_it
+      #do nothing
+    end
+  end
   
   def teardown
     #I don't do nothing

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tfidf
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-03-17 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast-stemmer
-  requirement: &16818040 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,12 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *16818040
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Calculate TF-IDF out of a text, resulting in a hash with term as key,
   frequency as value. Sorry for taking the convenient name for myself! See examples/demo_tf.rb
   for usage
@@ -30,6 +35,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- Rakefile
 - lib/tfidf.rb
 - examples/demo_tf.rb
 - test/test_TFIDF.rb
@@ -53,7 +59,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.19
 signing_key: 
 specification_version: 3
 summary: A W.I.P implementation of TF-IDF