tf-idf_csv 0.1.0 → 0.2.0

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+gem "csv"
+

data/Rakefile

@@ -9,7 +9,7 @@ begin
     gem.email = "jburgess@ap.org"
     gem.homepage = "http://github.com/aubergene/tf-idf_csv"
     gem.authors = ["Julian Burgess"]
-    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+    # gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/bin/tf-idf_csv

@@ -2,7 +2,21 @@
 
 $LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
 require 'tf-idf_csv'
+require 'csv'
+
+begin
+  csv_file = ARGV[0]
+  csv = CSV.open(csv_file)  
+rescue
+  puts "Please specify a valid CSV file"
+  Process.exit(1)
+end
+
+tf_idf = Tf_Idf_CSV.new()
+tf_idf.add_csv(csv)
+
+output_csv_file = csv_file.sub(/\.csv$/,'-tf-idf.csv')
+tf_idf.write(output_csv_file)
 
-tf = TfIdf_CSV.new(ARGV[0])
 
 

data/lib/tf-idf_csv.rb

@@ -1,46 +1,53 @@
 require 'csv'
-require 'set'
 
 # This class expects a CSV input
 # One row per document, 
 # the first cell should be a document identifier
 # each subsequent cell contains one term. 
 # TF-IDF will be returned based on the number of times the term appears in each document, relative to the total number of documents it appears in
+class Tf_Idf_CSV
 
-class TfIdf_CSV
-
-  DECIMAL_PLACES = 20 # number of decimal places to use in output
-  
-  # n the n-grams of the data http://en.wikipedia.org/wiki/N-gram
-  def initialize(csv_file)
-    @output_csv_file = csv_file.sub(/\.csv$/,'-tf-idf.csv')
-        
+  def initialize      
     @tf_idf = {}
     @total_number_of_docs = 0 
     @doc_count_per_term = Hash.new(0)
-    @term_freq_per_doc = Hash.new
-    
-    add_csv(csv_file)
-    calculate_tf_idf
-    puts "Finished calculations"
+    @term_freq_per_doc = Hash.new  
+  end
 
-    save_output
-    puts "Finished saving to #{@output_csv_file}"
+  def add_csv(csv)
+    csv.each do |row|
+      name = row[0] 
+      terms = row[1..-1]
+      add_document(name, terms)
+    end
+    calculate_tf_idf
   end
 
-  def add_csv(csv_file)
-    begin
-      CSV.foreach(csv_file) do |row|
-        add_doc(row[0], row[1..-1])
-        @total_number_of_docs += 1
+  # Save the results as CSV
+  # Term, Doc1, Doc2, Doc3...
+  # Eggs, 0.04535,,0.02
+  def write(csv_file_name, options = {})
+    decimal_places = options[:decimal_places] || 20
+        
+    CSV.open(csv_file_name,"w") do |f|
+      f << ["term", docs].flatten
+      @tf_idf.each do |term, values|
+        tmp_row = [term]
+        docs.each do |doc|
+          value = values[doc] ? ("%.#{decimal_places}f" % values[doc]) : nil
+          value = nil if value =~ /^0\.0+$/
+          tmp_row << value
+        end
+        f << tmp_row
       end
-    rescue Exception
-      puts "Error opening #{csv_file}. Please specify a valid CSV file"
-      Process.exit(1  )
     end
   end
 
-  def add_doc(doc, terms)
+  private
+
+  def add_document(doc, terms)
+    @total_number_of_docs += 1
+    
     term_counts_doc = Hash.new(0.0)
 
     # Count the number of times each term appears in this document
@@ -56,7 +63,7 @@ class TfIdf_CSV
     
     @term_freq_per_doc[doc] = term_counts_doc    
   end
-
+  
   def docs
     @term_freq_per_doc.keys
   end
@@ -78,23 +85,5 @@ class TfIdf_CSV
     end
   end
 
-  # Save the results as CSV
-  # Term, Doc1, Doc2, Doc3...
-  # Eggs, 0.04535,,0.02
-  def save_output
-    CSV.open(@output_csv_file,"w") do |f|
-      f << ["term", docs].flatten
-      @tf_idf.each do |term, values|
-        tmp_row = [term]
-        docs.each do |doc|
-          value = values[doc] ? ("%.#{DECIMAL_PLACES}f" % values[doc]) : nil
-          value = nil if value =~ /^0\.0+$/
-          tmp_row << value
-        end
-        f << tmp_row
-      end
-    end
-  end
- 
 end
 

data/tf-idf_csv.gemspec

@@ -0,0 +1,56 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{tf-idf_csv}
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Julian Burgess"]
+  s.date = %q{2010-11-02}
+  s.default_executable = %q{tf-idf_csv}
+  s.email = %q{jburgess@ap.org}
+  s.executables = ["tf-idf_csv"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "Gemfile",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "bin/tf-idf_csv",
+     "lib/tf-idf_csv.rb",
+     "sample-tf-idf.csv",
+     "sample.csv",
+     "test/helper.rb",
+     "test/test_tf-idf_csv.rb",
+     "tf-idf_csv.gemspec"
+  ]
+  s.homepage = %q{http://github.com/aubergene/tf-idf_csv}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Calculate the TF-IDF for terms in a CSV file}
+  s.test_files = [
+    "test/helper.rb",
+     "test/test_tf-idf_csv.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Julian Burgess
@@ -16,20 +16,8 @@ cert_chain: []
 
 date: 2010-11-02 00:00:00 -04:00
 default_executable: tf-idf_csv
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: thoughtbot-shoulda
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
-  type: :development
-  version_requirements: *id001
+dependencies: []
+
 description: 
 email: jburgess@ap.org
 executables: 
@@ -42,6 +30,7 @@ extra_rdoc_files:
 files: 
 - .document
 - .gitignore
+- Gemfile
 - LICENSE
 - README.rdoc
 - Rakefile
@@ -52,6 +41,7 @@ files:
 - sample.csv
 - test/helper.rb
 - test/test_tf-idf_csv.rb
+- tf-idf_csv.gemspec
 has_rdoc: true
 homepage: http://github.com/aubergene/tf-idf_csv
 licenses: []