bm25 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '039fdd5965e1f170b441115ddce07581551e4b210e5ff7814f34e409a836bbad'
-  data.tar.gz: cc171ad4db4e7a925c897d2ece2235089fd812a7e51cd663dc241aeb9d739716
+  metadata.gz: 23849a1ca209b9d85bddd83c0e135332af3176f72e14fe1301819107fcf25f65
+  data.tar.gz: 3a736717ba9c03c17548d2c295846daa2e71a044b167c59a8ffdd8bdfe2a0a9f
 SHA512:
-  metadata.gz: b1bd14b98cc0d801a4692471c9f5efae4b36cef4ed8d9928395ea4c2b93e4c17691564fd7a7a239b5024a2a06d0b46a243ac2436168ab93fc264700ec3b1f94f
-  data.tar.gz: 1bd71a4c0c9437122ff25c7184da839414b13dcccdef673b5fa103235e7c9cd5350584ae7025c67d5c92bb9051e5ca7532b6023f5b699e5055fc8bd747c71dc7
+  metadata.gz: 3b806bd8c54e3bd7b434c9fce996b838c0e6345b25e152458ac96b8a08c873e5405ed2304b50fa09aa706689db1cc695b10d2aa5a63dbbe607a28b36102ef855
+  data.tar.gz: 67e43c256e7dfb42f077bed16cdefc13b3f46119fbffb11ea53f26822700cbbefd4d0a253adc7c134a2837a1b62ef652220e409850aaed9e03408ebb99b9537f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    bm25 (0.1.5)
+    bm25 (0.1.6)
       natto
 
 GEM

data/lib/bm25/parser.rb

@@ -1,4 +1,5 @@
 require_relative 'utils'
+require_relative 'validator'
 require 'natto'
 require 'pp'
 
@@ -53,7 +54,6 @@ module Bm25
           end
         end
         avarage_word_length = @all_word_length / doc_list.length
-        # NDL(j) = 文書Djの総単語数 / (すべての文書の平均DL)
         @docs << {
           document: d,
           words: word_map,
@@ -66,10 +66,9 @@ module Bm25
     def create_idf_map
       words = []
       @docs.each do |d|
-        d[:words].each_pair do |k, v|
-          words << k
-        end
+        d[:words].each_pair{|k, v| words << k }
       end
+
       words = words.uniq
       words.each do |word|
         f = 0
@@ -88,14 +87,15 @@ module Bm25
         new_words = []
         k1 = 1.2
         b = 0.75
+        # [ TF(i,j) * IDF(i) * (K1 + 1) ] / [ K1 * (1 - b + (b * NDL(j)) + TF(i,j) ]
         d[:words].each_pair do |k, v|
-          # [ TF(i,j) * IDF(i) * (K1 + 1) ] / [ K1 * (1 - b + (b * NDL(j)) + TF(i,j) ]
+          tfidf = @idf_map[k][:idf] * v[:tf]
           new_words << {
             word: k,
             tf: v[:tf],
             idf: @idf_map[k][:idf],
-            val: @idf_map[k][:idf] * v[:tf],
-            bm25: (v[:tf] * @idf_map[k][:idf] + (k1 + 1)) / (k1 * (1 - b + (b * d[:dl])) + v[:tf])
+            tfidf: tfidf,
+            bm25: (tfidf + (k1 + 1)) / (k1 * (1 - b + (b * d[:dl])) + v[:tf])
           }
         end
         data << {

data/lib/bm25/utils.rb

@@ -1,38 +1,14 @@
-require 'natto'
 module Bm25
 
   module Utils
 
     class << self
 
-      def is_stopword? (word)
-        match = false
-        stopword_path = File.join( File.dirname(__FILE__), 'stopword.txt' )
-        File.open(stopword_path, "r") do |f|
-          f.each_line do |t|
-            if t.chomp === word
-              match = true
-              break
-            end
-          end
-        end
-        return match
-
-      end
-
-      def is_onechar?(word)
-        return word.size == 1
-      end
-
       def separate_words(document)
         nm = Natto::MeCab.new
         data = []
         nm.parse(document) do |n|
-          if (n.is_bos? || n.is_eos?) ||
-              n.feature.scan(/名詞|固有名詞/).length === 0 ||
-              n.surface.match(/[\/\d]/) ||
-              self.is_stopword?(n.surface) ||
-              self.is_onechar?(n.surface)
+          if Bm25::Validator.validate_word(n)
             next
           end
           data << n.surface

data/lib/bm25/validator.rb

@@ -0,0 +1,47 @@
+module Bm25
+  class Validator
+
+    class << self
+
+      # @param [String] word
+      # @return [Boolean] return True if the word is a stopword
+      def is_stopword? (word)
+        match = false
+        stopword_path = File.join( File.dirname(__FILE__), 'stopword.txt' )
+        File.open(stopword_path, "r") do |f|
+          f.each_line do |t|
+            if t.chomp === word
+              match = true
+              break
+            end
+          end
+        end
+        return match
+      end
+
+      # @param [String] word
+      # @return [Boolean] return True if the word is one character
+      def is_onechar?(word)
+        return word.size == 1
+      end
+
+      # @param [Object] mecab object
+      # @return [Boolean] return True if the mecab obj is noun
+      def validate_word(word_obj)
+        n = word_obj
+        if (n.is_bos? || n.is_eos?) ||
+            n.feature.scan(/名詞/).length === 0 ||
+            n.surface.match(/[\/\d]/) ||
+            self.is_stopword?(n.surface) ||
+            self.is_onechar?(n.surface)
+            return true
+        end
+        
+        return false
+
+      end
+
+    end
+
+  end
+end

data/lib/bm25/version.rb

@@ -1,3 +1,3 @@
 module Bm25
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bm25
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Masayuki Komatsu
@@ -89,6 +89,7 @@ files:
 - lib/bm25/parser.rb
 - lib/bm25/stopword.txt
 - lib/bm25/utils.rb
+- lib/bm25/validator.rb
 - lib/bm25/version.rb
 homepage: https://github.com/Bit-Pumpkin/bm25
 licenses: