tf-idf-similarity 0.1.3 → 0.1.4

data/lib/tf-idf-similarity/matrix_methods.rb

@@ -1,164 +1,166 @@
-module TfIdfSimilarity::MatrixMethods
-private
+module TfIdfSimilarity
+  module MatrixMethods
+  private
 
-  # @return [GSL::Matrix,NArray,NMatrix,Matrix] all document vectors as unit vectors
-  #
-  # @note Lucene normalizes document length differently.
-  def normalize
-    case @library
-    when :gsl
-      @matrix.clone.each_col do |column|
-        unless column.isnull?
-          column.normalize!
+    # @return [GSL::Matrix,NArray,NMatrix,Matrix] all document vectors as unit vectors
+    #
+    # @note Lucene normalizes document length differently.
+    def normalize
+      case @library
+      when :gsl
+        @matrix.clone.each_col do |column|
+          unless column.isnull?
+            column.normalize!
+          end
         end
-      end
-    when :narray # @see https://github.com/masa16/narray/issues/21
-      norm = NMath.sqrt((@matrix ** 2).sum(1).reshape(@matrix.shape[0], 1))
-      norm[norm.where2[1]] = 1.0 # avoid division by zero
-      NMatrix.refer(@matrix / norm) # must be NMatrix for matrix multiplication
-    when :nmatrix # @see https://github.com/SciRuby/nmatrix/issues/38
-      normal = NMatrix.new(:dense, @matrix.shape, 0, :float64)
-      (0...@matrix.shape[1]).each do |j|
-        column = @matrix.column(j)
-        norm = Math.sqrt(column.transpose.dot(column)[0, 0])
-        (0...@matrix.shape[0]).each do |i|
-          normal[i, j] = norm.zero? ? 0 : @matrix[i, j] / norm
+      when :narray # @see https://github.com/masa16/narray/issues/21
+        norm = NMath.sqrt((@matrix ** 2).sum(1).reshape(@matrix.shape[0], 1))
+        norm[norm.where2[1]] = 1.0 # avoid division by zero
+        NMatrix.refer(@matrix / norm) # must be NMatrix for matrix multiplication
+      when :nmatrix # @see https://github.com/SciRuby/nmatrix/issues/38
+        normal = NMatrix.new(:dense, @matrix.shape, 0, :float64)
+        (0...@matrix.shape[1]).each do |j|
+          column = @matrix.column(j)
+          norm = Math.sqrt(column.transpose.dot(column)[0, 0])
+          (0...@matrix.shape[0]).each do |i|
+            normal[i, j] = norm.zero? ? 0 : @matrix[i, j] / norm
+          end
         end
+        normal
+      else
+        Matrix.columns(@matrix.column_vectors.map do |column|
+          if column.to_a.all?(&:zero?)
+            column
+          elsif column.respond_to?(:normalize)
+            column.normalize
+          else
+            column * (1 / Math.sqrt(column.inner_product(column))) # 1.8 does define division
+          end
+        end)
       end
-      normal
-    else
-      Matrix.columns(@matrix.column_vectors.map do |column|
-        if column.to_a.all?(&:zero?)
-          column
-        elsif column.respond_to?(:normalize)
-          column.normalize
-        else
-          column * (1 / Math.sqrt(column.inner_product(column))) # 1.8 does define division
-        end
-      end)
     end
-  end
 
-  # @param [Integer] row index
-  # @param [Integer] column index
-  def get(i, j)
-    case @library
-    when :narray
-      @matrix[j, i]
-    else
-      @matrix[i, j]
+    # @param [Integer] row index
+    # @param [Integer] column index
+    def get(i, j)
+      case @library
+      when :narray
+        @matrix[j, i]
+      else
+        @matrix[i, j]
+      end
     end
-  end
 
-  # @param [Integer] index the row index
-  # @return [GSL::Vector::View,NArray,NMatrix,Vector] a row
-  def row(index)
-    case @library
-    when :narray
-      @matrix[true, index]
-    else
-      @matrix.row(index)
+    # @param [Integer] index the row index
+    # @return [GSL::Vector::View,NArray,NMatrix,Vector] a row
+    def row(index)
+      case @library
+      when :narray
+        @matrix[true, index]
+      else
+        @matrix.row(index)
+      end
     end
-  end
 
-  # @param [Integer] index the column index
-  # @return [GSL::Vector::View,NArray,NMatrix,Vector] a column
-  def column(index)
-    case @library
-    when :narray
-      @matrix[index, true]
-    else
-      @matrix.column(index)
+    # @param [Integer] index the column index
+    # @return [GSL::Vector::View,NArray,NMatrix,Vector] a column
+    def column(index)
+      case @library
+      when :narray
+        @matrix[index, true]
+      else
+        @matrix.column(index)
+      end
     end
-  end
 
-  # @return [Float] the number of rows in the matrix
-  def row_size
-    case @library
-    when :gsl, :nmatrix
-      @matrix.shape[0]
-    when :narray
-      @matrix.shape[1]
-    else
-      @matrix.row_size
+    # @return [Float] the number of rows in the matrix
+    def row_size
+      case @library
+      when :gsl, :nmatrix
+        @matrix.shape[0]
+      when :narray
+        @matrix.shape[1]
+      else
+        @matrix.row_size
+      end
     end
-  end
 
-  # @return [Float] the number of columns in the matrix
-  def column_size
-    case @library
-    when :gsl, :nmatrix
-      @matrix.shape[1]
-    when :narray
-      @matrix.shape[0]
-    else
-      @matrix.column_size
+    # @return [Float] the number of columns in the matrix
+    def column_size
+      case @library
+      when :gsl, :nmatrix
+        @matrix.shape[1]
+      when :narray
+        @matrix.shape[0]
+      else
+        @matrix.column_size
+      end
     end
-  end
 
-  # @return [Array<Float>] the matrix's values
-  def values
-    case @library
-    when :nmatrix
-      @matrix.each.to_a # faster than NMatrix's `to_a` and `to_flat_a`
-    else
-      @matrix.to_a.flatten
+    # @return [Array<Float>] the matrix's values
+    def values
+      case @library
+      when :nmatrix
+        @matrix.each.to_a # faster than NMatrix's `to_a` and `to_flat_a`
+      else
+        @matrix.to_a.flatten
+      end
     end
-  end
 
-  # @return [Float] the sum of all values in the matrix
-  def sum
-    case @library
-    when :narray
-      @matrix.sum
-    else
-      values.reduce(0, :+)
+    # @return [Float] the sum of all values in the matrix
+    def sum
+      case @library
+      when :narray
+        @matrix.sum
+      else
+        values.reduce(0, :+)
+      end
     end
-  end
 
-  # @param [Array<Array>] array matrix rows
-  # @return [GSL::Matrix,NArray,NMatrix,Matrix] a matrix
-  def initialize_matrix(array)
-    case @library
-    when :gsl
-      GSL::Matrix[*array]
-    when :narray
-      NArray[*array]
-    when :nmatrix # @see https://github.com/SciRuby/nmatrix/issues/91#issuecomment-18870619
-      NMatrix.new(:dense, [array.size, array.empty? ? 0 : array[0].size], array.flatten, :float64)
-    else
-      Matrix[*array]
+    # @param [Array<Array>] array matrix rows
+    # @return [GSL::Matrix,NArray,NMatrix,Matrix] a matrix
+    def initialize_matrix(array)
+      case @library
+      when :gsl
+        GSL::Matrix[*array]
+      when :narray
+        NArray[*array]
+      when :nmatrix # @see https://github.com/SciRuby/nmatrix/issues/91#issuecomment-18870619
+        NMatrix.new(:dense, [array.size, array.empty? ? 0 : array[0].size], array.flatten, :float64)
+      else
+        Matrix[*array]
+      end
     end
-  end
 
-  # @param [GSL::Matrix,NArray,NMatrix,Matrix] matrix a matrix
-  # @return [GSL::Matrix,NArray,NMatrix,Matrix] the product
-  def multiply_self(matrix)
-    case @library
-    when :nmatrix
-      matrix.transpose.dot(matrix)
-    else
-      matrix.transpose * matrix
+    # @param [GSL::Matrix,NArray,NMatrix,Matrix] matrix a matrix
+    # @return [GSL::Matrix,NArray,NMatrix,Matrix] the product
+    def multiply_self(matrix)
+      case @library
+      when :nmatrix
+        matrix.transpose.dot(matrix)
+      else
+        matrix.transpose * matrix
+      end
     end
-  end
 
-  def log(number)
-    case @library
-    when :gsl
-      GSL::Sf::log(number)
-    when :narray
-      NMath.log(number)
-    else
-      Math.log(number)
+    def log(number)
+      case @library
+      when :gsl
+        GSL::Sf::log(number)
+      when :narray
+        NMath.log(number)
+      else
+        Math.log(number)
+      end
     end
-  end
 
-  def sqrt(number)
-    case @library
-    when :narray
-      NMath.sqrt(number)
-    else
-      Math.sqrt(number)
+    def sqrt(number)
+      case @library
+      when :narray
+        NMath.sqrt(number)
+      else
+        Math.sqrt(number)
+      end
     end
   end
 end

data/lib/tf-idf-similarity/model.rb

@@ -0,0 +1,66 @@
+module TfIdfSimilarity
+  class Model
+    include MatrixMethods
+
+    extend Forwardable
+    def_delegators :@model, :documents, :terms, :document_count
+
+    # @param [Array<Document>] documents documents
+    # @param [Hash] opts optional arguments
+    # @option opts [Symbol] :library :gsl, :narray, :nmatrix or :matrix (default)
+    def initialize(documents, opts = {})
+      @model = TermCountModel.new(documents, opts)
+      @library = (opts[:library] || :matrix).to_sym
+
+      array = Array.new(terms.size) do |i|
+        idf = inverse_document_frequency(terms[i])
+        Array.new(documents.size) do |j|
+          term_frequency(documents[j], terms[i]) * idf
+        end
+      end
+
+      @matrix = initialize_matrix(array)
+    end
+
+    # Return the term frequency–inverse document frequency.
+    #
+    # @param [Document] document a document
+    # @param [String] term a term
+    # @return [Float] the term frequency–inverse document frequency
+    def term_frequency_inverse_document_frequency(document, term)
+      inverse_document_frequency(term) * term_frequency(document, term)
+    end
+    alias_method :tfidf, :term_frequency_inverse_document_frequency
+
+    # Returns a similarity matrix for the documents in the corpus.
+    #
+    # @return [GSL::Matrix,NMatrix,Matrix] a similarity matrix
+    # @note Columns are normalized to unit vectors, so we can calculate the cosine
+    #   similarity of all document vectors.
+    def similarity_matrix
+      if documents.empty?
+        []
+      else
+        multiply_self(normalize)
+      end
+    end
+
+    # Return the index of the document in the corpus.
+    #
+    # @param [Document] document a document
+    # @return [Integer,nil] the index of the document
+    def document_index(document)
+      @model.documents.index(document)
+    end
+
+    # Return the index of the document with matching text.
+    #
+    # @param [String] text a text
+    # @return [Integer,nil] the index of the document
+    def text_index(text)
+      @model.documents.index do |document|
+        document.text == text
+      end
+    end
+  end
+end

data/lib/tf-idf-similarity/term_count_model.rb

@@ -1,74 +1,76 @@
 # A simple document-term matrix.
-class TfIdfSimilarity::TermCountModel
-  include TfIdfSimilarity::MatrixMethods
+module TfIdfSimilarity
+  class TermCountModel
+    include MatrixMethods
 
-  # The documents in the corpus.
-  attr_reader :documents
-  # The set of terms in the corpus.
-  attr_reader :terms
-  # The average number of tokens in a document.
-  attr_reader :average_document_size
+    # The documents in the corpus.
+    attr_reader :documents
+    # The set of terms in the corpus.
+    attr_reader :terms
+    # The average number of tokens in a document.
+    attr_reader :average_document_size
 
-  # @param [Array<TfIdfSimilarity::Document>] documents documents
-  # @param [Hash] opts optional arguments
-  # @option opts [Symbol] :library :gsl, :narray, :nmatrix or :matrix (default)
-  def initialize(documents, opts = {})
-    @documents = documents
-    @terms = Set.new(documents.map(&:terms).flatten).to_a
-    @library = (opts[:library] || :matrix).to_sym
+    # @param [Array<Document>] documents documents
+    # @param [Hash] opts optional arguments
+    # @option opts [Symbol] :library :gsl, :narray, :nmatrix or :matrix (default)
+    def initialize(documents, opts = {})
+      @documents = documents
+      @terms = Set.new(documents.map(&:terms).flatten).to_a
+      @library = (opts[:library] || :matrix).to_sym
 
-    array = Array.new(terms.size) do |i|
-      Array.new(documents.size) do |j|
-        documents[j].term_count(terms[i])
+      array = Array.new(terms.size) do |i|
+        Array.new(documents.size) do |j|
+          documents[j].term_count(terms[i])
+        end
       end
-    end
 
-    @matrix = initialize_matrix(array)
+      @matrix = initialize_matrix(array)
 
-    @average_document_size = documents.empty? ? 0 : sum / column_size.to_f
-  end
+      @average_document_size = documents.empty? ? 0 : sum / column_size.to_f
+    end
 
-  # @param [String] term a term
-  # @return [Integer] the number of documents the term appears in
-  def document_count(term)
-    index = terms.index(term)
-    if index
-      case @library
-      when :gsl, :narray
-        row(index).where.size
-      when :nmatrix
-        row(index).each.count(&:nonzero?)
-      else
-        vector = row(index)
-        unless vector.respond_to?(:count)
-          vector = vector.to_a
+    # @param [String] term a term
+    # @return [Integer] the number of documents the term appears in
+    def document_count(term)
+      index = terms.index(term)
+      if index
+        case @library
+        when :gsl, :narray
+          row(index).where.size
+        when :nmatrix
+          row(index).each.count(&:nonzero?)
+        else
+          vector = row(index)
+          unless vector.respond_to?(:count)
+            vector = vector.to_a
+          end
+          vector.count(&:nonzero?)
         end
-        vector.count(&:nonzero?)
+      else
+        0
       end
-    else
-      0
     end
-  end
 
-  # @param [String] term a term
-  # @return [Integer] the number of times the term appears in the corpus
-  def term_count(term)
-    index = terms.index(term)
-    if index
-      case @library
-      when :gsl, :narray
-        row(index).sum
-      when :nmatrix
-        row(index).each.reduce(0, :+) # NMatrix's `sum` method is slower
-      else
-        vector = row(index)
-        unless vector.respond_to?(:reduce)
-          vector = vector.to_a
+    # @param [String] term a term
+    # @return [Integer] the number of times the term appears in the corpus
+    def term_count(term)
+      index = terms.index(term)
+      if index
+        case @library
+        when :gsl, :narray
+          row(index).sum
+        when :nmatrix
+          row(index).each.reduce(0, :+) # NMatrix's `sum` method is slower
+        else
+          vector = row(index)
+          unless vector.respond_to?(:reduce)
+            vector = vector.to_a
+          end
+          vector.reduce(0, :+)
         end
-        vector.reduce(0, :+)
+      else
+        0
       end
-    else
-      0
     end
   end
 end