similarity 0.2.3

data/lib/similarity.rb

@@ -0,0 +1,4 @@
+require 'similarity/corpus'
+require 'similarity/document'
+require 'similarity/term_document_matrix'
+

data/lib/similarity/corpus.rb

@@ -0,0 +1,85 @@
+require 'gsl'
+
+class Corpus
+  attr_reader :terms, :documents
+
+  def initialize
+    @terms = {}
+    @documents = []
+    @term_document_matrix = nil
+    @similarity_matrix = nil
+  end
+
+  def document_count
+    @documents.size
+  end
+
+  def <<(document)
+    document.terms.uniq.each do |term|
+      if @terms[term]
+        @terms[term] += 1
+      else
+        @terms[term] = 1
+      end
+    end
+    @documents << document
+  end
+
+  def remove_infrequent_terms!(percentage)
+    number_of_docs = document_count.to_f
+    @terms = terms.delete_if {|term, count| (count.to_f / number_of_docs) < percentage}
+    @term_document_matrix = nil
+    @similarity_matrix = nil
+  end
+
+  def remove_frequent_terms!(percentage)
+    number_of_docs = document_count.to_f
+    @terms = terms.delete_if {|term, count| (count.to_f / number_of_docs) > percentage}
+    @term_document_matrix = nil
+    @similarity_matrix = nil
+  end
+
+  def inverse_document_frequency(term)
+    puts "#{document_count} / (1 + #{document_count_for_term(term)})" if $DEBUG
+    Math.log(document_count.to_f / (1 + document_count_for_term(term)))
+  end
+
+  def document_count_for_term(term)
+    if @terms[term]
+      @terms[term]
+    else
+      0
+    end
+  end
+
+  def similarity_matrix
+    if @similarity_matrix
+      return @similarity_matrix
+    else
+      @similarity_matrix = term_document_matrix.similarity_matrix
+    end
+  end
+
+  def term_document_matrix
+    if @term_document_matrix
+      return @term_document_matrix
+    else
+      @term_document_matrix = TermDocumentMatrix.new(self)
+    end
+  end
+
+  def weights(document)
+    idx = @documents.index(document)
+    terms = @terms.to_a.map {|term| term.first}
+    weights = term_document_matrix.col(idx).to_a
+
+    # create array of array pairs of terms and weights
+    term_weight_pairs = terms.zip(weights)
+
+    # remove zero weights
+    term_weight_pairs.reject! {|pair| pair[1].zero?}
+
+    # sort in descending order
+    term_weight_pairs.sort {|x,y| y[1] <=> x[1]}
+  end
+end

data/lib/similarity/document.rb

@@ -0,0 +1,56 @@
+class Document
+  attr_reader :content, :id
+
+  def initialize(hash_args)
+    content = hash_args[:content]
+    if content && !content.empty?
+      @content = content
+      @term_frequency = nil
+      @terms = nil
+    else
+      raise ArgumentError, "text cannot be nil or blank"
+    end
+
+    id = hash_args[:id]
+    if id && !id.nil?
+      @id = id
+    else
+      @id = self.object_id
+    end
+  end
+
+  def terms
+    @terms ||=
+      @content.gsub(/(\d|\s|\W)+/, ' ').
+      split(/\s/).map { |term| term.downcase }
+  end
+
+  def term_frequencies
+    @term_frequencies ||= calculate_term_frequencies
+  end
+
+  def calculate_term_frequencies
+    tf = {}
+    terms.each do |term|
+      if tf[term]
+        tf[term] += 1
+      else
+        tf[term] = 1
+      end
+    end
+    total_number_of_terms = terms.size.to_f
+    tf.each_pair { |k,v| tf[k] = (tf[k] / total_number_of_terms) }
+  end
+
+  def term_frequency(term)
+    if tf = term_frequencies[term]
+      tf
+    else
+      0
+    end
+  end
+
+  def has_term?(term)
+    terms.include? term
+  end
+end

data/lib/similarity/term_document_matrix.rb

@@ -0,0 +1,33 @@
+require 'gsl'
+
+class TermDocumentMatrix
+  attr_reader :matrix, :labels
+
+  def initialize(corpus)
+    @matrix = GSL::Matrix.alloc(corpus.terms.size, corpus.document_count)
+
+    corpus.documents.each_with_index do |document, document_index|
+      corpus.terms.each_with_index do |term, term_index|
+        term = term.first
+        idf = corpus.inverse_document_frequency(term)
+        weight = document.term_frequency(term) * idf
+        @matrix[term_index, document_index] = weight
+      end
+    end
+
+    @matrix.each_col { |col| col.div!(col.norm) }
+    @labels = corpus.terms.to_a.map {|e| e[0]}
+  end
+
+  def similarity_matrix
+    self.matrix.transpose * self.matrix
+  end
+
+  def col(idx)
+    @matrix.col(idx)
+  end
+
+  def to_a
+    @matrix.to_a
+  end
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification 
+name: similarity
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.2.3
+platform: ruby
+authors: 
+- Chris Lowis
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-25 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: gsl
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: faker
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: ruby-graphviz
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: |
+  Pure-ruby document similarity calculations using cosine similarity and TF-IDF weights
+
+email: chris.lowis@bbc.co.uk
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/similarity/corpus.rb
+- lib/similarity/document.rb
+- lib/similarity/term_document_matrix.rb
+- lib/similarity.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: similarity
+rubygems_version: 1.5.0
+signing_key: 
+specification_version: 3
+summary: Document similarity calculations using cosine similarity and TF-IDF weights
+test_files: []
+