tf-idf-similarity 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 605ac457508eaf64a7e583e8a4a71af231d3d9d2f9c30ee82b25fb9f647d1312
-  data.tar.gz: f24b89dccdcbef3c4fcaa59d15050f064455859c134c550fd6a432346883eb31
+  metadata.gz: 6264c3854dae2c9b405e6880fc35161c7e7eb05521b88fa206d7d5f7cad53d73
+  data.tar.gz: d7d6c7f38723e35c256e14edecdfd7d4d993f58163a9c533e12f591051ed80dd
 SHA512:
-  metadata.gz: a41195c6543dea206baa8ce3e2095437d1df94fabedcc76a8151fa5af5991524d96530710a7216c1fef48a7008f88a43773ce2a2323afa563fa29f5abed9909c
-  data.tar.gz: aadbb85d6bd74625088d0aa7cb58b4127337d5c1dcc2af13c22664f1562013c59d79d8b3bcc3564a2861dfd968d39770205d3b401114e8bdf870b2ac412fda26
+  metadata.gz: 4f97120bc10adc73e504503969c4ee2c486ca5909def7fc69a6ed35c4ceb3ba03294fb60a583f334a8fd83b06fe37d5b13258fa85fd699c47a812d8bdc2ce23f
+  data.tar.gz: a84fd8950933635fadd85caeb6a2684dfa989664553c8971f8f1f6d1ec7f7a299388d8aec186facedf6ab9174d125182fdcdfee16facd7317b27af5b462399b9

data/.travis.yml

@@ -2,14 +2,42 @@ sudo: false
 language: ruby
 cache: bundler
 rvm:
-  - 2.0.0
-  - 2.1.0
-  - 2.2.0
+  - 2.4
+  - 2.5
+  - 2.6
+  - 2.7
+  - 3.0
+  - 3.1
+  - 3.2
+  - ruby-head
+matrix:
+  exclude:
+    # No gem releases since 2017 and failing on new versions.
+    # https://rubygems.org/gems/gsl
+    # https://rubygems.org/gems/nmatrix
+    - rvm: 3.0
+      env: MATRIX_LIBRARY=gsl
+    - rvm: 3.1
+      env: MATRIX_LIBRARY=gsl
+    - rvm: 3.2
+      env: MATRIX_LIBRARY=gsl
+    - rvm: ruby-head
+      env: MATRIX_LIBRARY=gsl
+    - rvm: 3.2
+      env: MATRIX_LIBRARY=nmatrix
+    - rvm: ruby-head
+      env: MATRIX_LIBRARY=nmatrix
+  allow_failures:
+    - rvm: ruby-head
+      env: MATRIX_LIBRARY=matrix
+    - rvm: ruby-head
+      env: MATRIX_LIBRARY=narray
 env:
   - MATRIX_LIBRARY=gsl
   - MATRIX_LIBRARY=narray
   - MATRIX_LIBRARY=nmatrix
   - MATRIX_LIBRARY=matrix
+  - MATRIX_LIBRARY=numo
 addons:
   apt:
     packages:

data/Gemfile

@@ -1,8 +1,10 @@
 source 'https://rubygems.org'
 
 gem 'rb-gsl', '~> 1.16.0.2' if ENV['MATRIX_LIBRARY'] == 'gsl'
+gem 'matrix' if ENV['MATRIX_LIBRARY'] == 'matrix'
 gem 'narray', '~> 0.6.0.0' if ENV['MATRIX_LIBRARY'] == 'narray'
-gem 'nmatrix', '~> 0.1.0.rc5' if ENV['MATRIX_LIBRARY'] == 'nmatrix' && RUBY_VERSION >= '1.9'
+gem 'nmatrix', '~> 0.2' if ENV['MATRIX_LIBRARY'] == 'nmatrix'
+gem 'numo-narray', '~> 0.9.2.1' if ENV['MATRIX_LIBRARY'] == 'numo'
 
 # Specify your gem's dependencies in the gemspec
 gemspec

data/lib/tf-idf-similarity/bm25_model.rb

@@ -22,8 +22,12 @@ module TfIdfSimilarity
     #
     # @note Like Lucene, we use a b value of 0.75 and a k1 value of 1.2.
     def term_frequency(document, term)
-      tf = document.term_count(term)
-      (tf * 2.2) / (tf + 0.3 + 0.9 * documents.size / @model.average_document_size)
+      if @model.average_document_size.zero?
+        Float::NAN
+      else
+        tf = document.term_count(term)
+        (tf * 2.2) / (tf + 0.3 + 0.9 * document.size / @model.average_document_size)
+      end
     end
     alias_method :tf, :term_frequency
   end

data/lib/tf-idf-similarity/matrix_methods.rb

@@ -17,6 +17,10 @@ module TfIdfSimilarity
         norm = NMath.sqrt((@matrix ** 2).sum(1).reshape(@matrix.shape[0], 1))
         norm[norm.where2[1]] = 1.0 # avoid division by zero
         NMatrix.refer(@matrix / norm) # must be NMatrix for matrix multiplication
+      when :numo
+        norm = Numo::NMath.sqrt((@matrix ** 2).sum(0).reshape(1, @matrix.shape[1]))
+        norm[(norm.eq 0).where] = 1.0 # avoid division by zero
+        (@matrix / norm)
       when :nmatrix # @see https://github.com/SciRuby/nmatrix/issues/38
         normal = NMatrix.new(:dense, @matrix.shape, 0, :float64)
         (0...@matrix.shape[1]).each do |j|
@@ -44,7 +48,7 @@ module TfIdfSimilarity
     # @param [Integer] column index
     def get(i, j)
       case @library
-      when :narray
+      when :narray, :numo
         @matrix[j, i]
       else
         @matrix[i, j]
@@ -57,6 +61,8 @@ module TfIdfSimilarity
       case @library
       when :narray
         @matrix[true, index]
+      when :numo
+        @matrix[index, true]
       else
         @matrix.row(index)
       end
@@ -66,7 +72,7 @@ module TfIdfSimilarity
     # @return [GSL::Vector::View,NArray,NMatrix,Vector] a column
     def column(index)
       case @library
-      when :narray
+      when :narray, :numo
         @matrix[index, true]
       else
         @matrix.column(index)
@@ -78,7 +84,7 @@ module TfIdfSimilarity
       case @library
       when :gsl, :nmatrix
         @matrix.shape[0]
-      when :narray
+      when :narray, :numo
         @matrix.shape[1]
       else
         @matrix.row_size
@@ -90,7 +96,7 @@ module TfIdfSimilarity
       case @library
       when :gsl, :nmatrix
         @matrix.shape[1]
-      when :narray
+      when :narray, :numo
         @matrix.shape[0]
       else
         @matrix.column_size
@@ -110,7 +116,7 @@ module TfIdfSimilarity
     # @return [Float] the sum of all values in the matrix
     def sum
       case @library
-      when :narray
+      when :narray, :numo
         @matrix.sum
       else
         values.reduce(0, :+)
@@ -125,6 +131,8 @@ module TfIdfSimilarity
         GSL::Matrix[*array]
       when :narray
         NArray[*array]
+      when :numo
+        Numo::DFloat[*array]
       when :nmatrix # @see https://github.com/SciRuby/nmatrix/issues/91#issuecomment-18870619
         NMatrix.new(:dense, [array.size, array.empty? ? 0 : array[0].size], array.flatten, :float64)
       else
@@ -136,7 +144,7 @@ module TfIdfSimilarity
     # @return [GSL::Matrix,NArray,NMatrix,Matrix] the product
     def multiply_self(matrix)
       case @library
-      when :nmatrix
+      when :nmatrix, :numo
         matrix.transpose.dot(matrix)
       else
         matrix.transpose * matrix
@@ -149,6 +157,8 @@ module TfIdfSimilarity
         GSL::Sf::log(number)
       when :narray
         NMath.log(number)
+      when :numo
+        Numo::NMath.log(number)
       else
         Math.log(number)
       end
@@ -158,6 +168,8 @@ module TfIdfSimilarity
       case @library
       when :narray
         NMath.sqrt(number)
+      when :numo
+        Numo::NMath.sqrt(number)
       else
         Math.sqrt(number)
       end

data/lib/tf-idf-similarity/model.rb

@@ -15,7 +15,7 @@ module TfIdfSimilarity
       array = Array.new(terms.size) do |i|
         idf = inverse_document_frequency(terms[i])
         Array.new(documents.size) do |j|
-          term_frequency(documents[j], terms[i]) * idf
+          (term_frequency(documents[j], terms[i]) * idf).to_f
         end
       end
 

data/lib/tf-idf-similarity/term_count_model.rb

@@ -37,6 +37,8 @@ module TfIdfSimilarity
         case @library
         when :gsl, :narray
           row(index).where.size
+        when :numo
+          (row(index).ne 0).where.size
         when :nmatrix
           row(index).each.count(&:nonzero?)
         else
@@ -57,7 +59,7 @@ module TfIdfSimilarity
       index = terms.index(term)
       if index
         case @library
-        when :gsl, :narray
+        when :gsl, :narray, :numo
           row(index).sum
         when :nmatrix
           row(index).each.reduce(0, :+) # NMatrix's `sum` method is slower

data/lib/tf-idf-similarity/version.rb

@@ -1,3 +1,3 @@
 module TfIdfSimilarity
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/spec/bm25_model_spec.rb

@@ -82,7 +82,12 @@ module TfIdfSimilarity
 
       describe '#term_frequency_inverse_document_frequency' do
         it 'should return negative infinity' do
-          model.tfidf(document, 'foo').should be_nan
+          case MATRIX_LIBRARY
+          when :numo
+            model.tfidf(document, 'foo').isnan.should eq 1
+          else
+            model.tfidf(document, 'foo').should be_nan
+          end
         end
       end
 
@@ -147,7 +152,7 @@ module TfIdfSimilarity
         end
 
         it 'should return the term frequency if tokens given' do
-          model.tf(document_with_tokens, 'foo-foo').should == (1 * 2.2) / (1 + 0.3 + 0.9 * 4 / 5.5)
+          model.tf(document_with_tokens, 'foo-foo').should == (1 * 2.2) / (1 + 0.3 + 0.9 * 3 / 5.5)
         end
 
         it 'should return no term frequency if no text given' do
@@ -155,7 +160,7 @@ module TfIdfSimilarity
         end
 
         it 'should return the term frequency if term counts given' do
-          model.tf(document_with_term_counts, 'bar').should == (5 * 2.2) / (5 + 0.3 + 0.9 * 4 / 5.5)
+          model.tf(document_with_term_counts, 'bar').should == (5 * 2.2) / (5 + 0.3 + 0.9 * 15 / 5.5)
         end
 
         it 'should return the term frequency of a non-occurring term' do
@@ -163,7 +168,7 @@ module TfIdfSimilarity
         end
 
         it 'should return the term frequency in a non-occurring document' do
-          model.tf(non_corpus_document, 'foo').should == (3 * 2.2) / (3 + 0.3 + 0.9 * 4 / 5.5)
+          model.tf(non_corpus_document, 'foo').should == (3 * 2.2) / (3 + 0.3 + 0.9 * 3 / 5.5)
         end
       end
 
@@ -177,17 +182,17 @@ module TfIdfSimilarity
         end
 
         it 'should return the tf*idf in a non-occurring term' do
-          model.tfidf(non_corpus_document, 'foo').should be_within(0.001).of(Math.log((4 - 1 + 0.5) / (1 + 0.5)) * (3 * 2.2) / (3 + 0.3 + 0.9 * 4 / 5.5))
+          model.tfidf(non_corpus_document, 'foo').should be_within(0.001).of(Math.log((4 - 1 + 0.5) / (1 + 0.5)) * (3 * 2.2) / (3 + 0.3 + 0.9 * 3 / 5.5))
         end
       end
 
       describe '#similarity_matrix' do
         it 'should return the similarity matrix' do
           expected = [
-            1.0,   0.564, 0.0, 0.479,
-            0.564, 1.0,   0.0, 0.540,
+            1.0,   0.558, 0.0, 0.449,
+            0.558, 1.0,   0.0, 0.501,
             0.0,   0.0,   0.0, 0.0,
-            0.479, 0.540, 0.0, 1.0,
+            0.449, 0.501, 0.0, 1.0,
           ]
 
           similarity_matrix_values(model).each_with_index do |value,i|

data/spec/spec_helper.rb

@@ -18,6 +18,8 @@ when :gsl
   require 'gsl'
 when :narray
   require 'narray'
+when :numo
+  require 'numo/narray'
 when :nmatrix
   require 'nmatrix'
 else

data/td-idf-similarity.gemspec

@@ -9,6 +9,7 @@ Gem::Specification.new do |s|
   s.homepage    = "https://github.com/jpmckinney/tf-idf-similarity"
   s.summary     = %q{Calculates the similarity between texts using tf*idf}
   s.license     = 'MIT'
+  s.required_ruby_version = '>= 2.4.0'
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -19,6 +20,6 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency('coveralls')
   s.add_development_dependency('json', '< 2')
-  s.add_development_dependency('rake', '< 12')
-  s.add_development_dependency('rspec', '~> 2.10')
+  s.add_development_dependency('rake')
+  s.add_development_dependency('rspec', '~> 3.0')
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tf-idf-similarity
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - James McKinney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-19 00:00:00.000000000 Z
+date: 2024-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode_utils
@@ -56,30 +56,30 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '3.0'
 description: 
 email: 
 executables: []
@@ -126,15 +126,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Calculates the similarity between texts using tf*idf