tf_idf 0.0.1 → 0.0.2

data/README.rdoc

@@ -11,10 +11,9 @@ http://en.wikipedia.org/wiki/Tf–idf
   require 'rubygems'
   require 'tf_idf'
   
-  data = ['a a a a a a a a b b', 'a a']
+  data = [%w{a a a a a a a a b b}, %w{a a}]
   
-  # 1 is the ngram setter => http://en.wikipedia.org/wiki/N-gram
-  a = TfIdf.new(data, 1)
+  a = TfIdf.new(data)
   
   # To find the term frequencies
   a.tf

data/Rakefile

@@ -11,7 +11,6 @@ begin
     gem.homepage = "http://github.com/reddavis/TF-IDF"
     gem.authors = ["reddavis"]
     gem.add_development_dependency "rspec", ">= 1.2.9"
-    gem.add_dependency('n_gram', ">= 0.0.0")
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/lib/tf_idf.rb

@@ -1,11 +1,8 @@
-require 'n_gram'
-
 class TfIdf
   
   # n the n-grams of the data http://en.wikipedia.org/wiki/N-gram
-  def initialize(data, n=1)
+  def initialize(data)
     @data = data
-    @n = n
   end
   
   def tf
@@ -28,54 +25,62 @@ class TfIdf
     
     tf_idf
   end
+    
+  private
   
   def total_documents
     @data.size.to_f
   end
   
-  private
+  # Returns all terms, once
+  def terms
+    @data.flatten.uniq
+  end
   
   # IDF = total_documents / number_of_document_term_appears_in
   # This calculates how important a term is.
   def calculate_inverse_document_frequency
-    original_ngrams = n_gram.ngrams_of_all_data[@n].clone
+    results = {}
     
-    original_ngrams.each_key do |term|
-      
-      # Calculate how many documents the term appears in
+    terms.each do |term|
       count = 0.0
-      n_gram.ngrams_of_inputs.each do |document|
-        count += 1 if document[@n].key?(term)
+      
+      @data.each do |document|
+        count += 1 if document.include?(term)
       end
-    
-      original_ngrams[term] = Math.log10(total_documents / count)
+      
+      results[term] = Math.log10(total_documents / count)
     end
     
-    original_ngrams
+    results
   end
   
   # TF = number_of_n_term_in_document / number_of_terms_in_document
   # Calculates the number of times a term appears in the document
   # It is then normalized (as some documents are longer than others)
   def calculate_term_frequencies
-    original_ngrams = n_gram.ngrams_of_inputs.clone
-            
-    original_ngrams.each_with_index do |document, index|
+    results = []
+    
+    @data.each do |document|
+      document_result = {}
+      
+      document.each do |term|
+        if document_result.key?(term)
+          document_result[term] += 1.0
+        else
+          document_result[term] = 1.0
+        end
+      end
       
-      # Calculate the total number of terms
-      total_terms = 0.0
-      document[@n].each_value {|v| total_terms += v} 
-            
-      document[@n].each_pair do |key, value|
-        original_ngrams[index][@n][key] = (value.to_f / total_terms)
+      # Normalize the count
+      document_result.each_key do |term|
+        document_result[term] /= document.size
       end
+      
+      results << document_result
     end
     
-    original_ngrams.map {|x| x.map {|y| y[1] }}.flatten
+    results
   end
-  
-  def n_gram
-    @n_gram ||= NGram.new(@data, :n => @n)
-  end
-  
+    
 end

data/spec/tf_idf_spec.rb

@@ -3,7 +3,7 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 describe "TfIdf" do
   describe "Term Frequency" do
     before do
-      @a = TfIdf.new(data, 1)
+      @a = TfIdf.new(data)
     end
     
     it "should return 0.2" do
@@ -13,16 +13,16 @@ describe "TfIdf" do
   
   describe "Inverse Document Frequency" do
     before do
-      @a = TfIdf.new(data, 1)
+      @a = TfIdf.new(data)
     end
     
     it "should return 0.3010" do
       @a.idf['b'].to_s.should match(/0.30102999/)
     end
   end
-  
+    
   before do
-    @a = TfIdf.new(data, 1)
+    @a = TfIdf.new(data)
   end
   
   it "should return 0.0602" do
@@ -32,6 +32,6 @@ describe "TfIdf" do
   private
   
   def data
-    ['a a a a a a a a b b', 'a a']
+    [%w{a a a a a a a a b b}, %w{a a}]
   end
 end

data/tf_idf.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{tf_idf}
-  s.version = "0.0.1"
+  s.version = "0.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["reddavis"]
-  s.date = %q{2009-12-21}
+  s.date = %q{2010-01-06}
   s.description = %q{A TF-IDF in ruby - http://en.wikipedia.org/wiki/Tf–idf}
   s.email = %q{reddavis@gmail.com}
   s.extra_rdoc_files = [
@@ -45,14 +45,11 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
-      s.add_runtime_dependency(%q<n_gram>, [">= 0.0.0"])
     else
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
-      s.add_dependency(%q<n_gram>, [">= 0.0.0"])
     end
   else
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
-    s.add_dependency(%q<n_gram>, [">= 0.0.0"])
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: tf_idf
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors: 
 - reddavis
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-21 00:00:00 +00:00
+date: 2010-01-06 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -22,16 +22,6 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 1.2.9
     version: 
-- !ruby/object:Gem::Dependency 
-  name: n_gram
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 0.0.0
-    version: 
 description: "A TF-IDF in ruby - http://en.wikipedia.org/wiki/Tf\xE2\x80\x93idf"
 email: reddavis@gmail.com
 executables: []