bm25f 0.2.4 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c7af39f073c1fc55d42199f9c882caa5adf672d5c94c715fe379c58cd4a4a19
-  data.tar.gz: 1ebb197c89da4b70780d3631658a1bb3d0ce5c244abe8b5875c41ee1632f63a3
+  metadata.gz: f82699fd9a61bec9ce5a36b450860d2d64a10e35b6a81e29760067f5e04fd0be
+  data.tar.gz: 515fb6057a449c3deb0fc8fb32286114bc507212b843cb645b758a7930d0605a
 SHA512:
-  metadata.gz: 816270030afd15716f8b89f4c74d5c11dbc32f169c7a10a1eca931782e21e320017aa95bab66e32c6b500d407922e289edce8db396c956a62c245871744bb79b
-  data.tar.gz: 0ef709327eaa9c5b14e1132bafc199c4d66759b509c85d8bf3dcb23ba5b8dbe526b698c0ab1051486741fa3a3d4d21d073f8d4e3d12ced9128ec16fedd6a39e4
+  metadata.gz: 68facd20bceba46784de0db93a22d59827245632c7b848ad1ca185c4e7b8079dbbc66e3e4027531b5aa55610275694282509fd257783bc6d187dbc7401effddf
+  data.tar.gz: 4ef27f212f8bb8ea828b4dbc33dd6b726a105e1abfd10797b488d951c4f956e53bd14b37261d78c10b3bbb1342c73728b4dd767ab1fd1638b33bcbbf5d685d3e

data/lib/bm25f.rb

@@ -1,8 +1,7 @@
-require 'treat'
+require 'uea-stemmer'
+require 'pragmatic_tokenizer'
 
 class BM25F
-  include Treat::Core::DSL
-
   # Initializes a BM25F model.
   #
   # @param term_freq_weight [Float] Weight for term frequency.
@@ -10,6 +9,9 @@ class BM25F
   def initialize(term_freq_weight: 1.33, doc_length_weight: 0.8)
     @term_freq_weight = term_freq_weight
     @doc_length_weight = doc_length_weight
+
+    @tokenizer = PragmaticTokenizer::Tokenizer.new
+    @stemmer = UEAStemmer.new
   end
 
   # Fits the model to a set of documents.
@@ -56,7 +58,7 @@ class BM25F
     documents.each do |k, v|
       next unless v.instance_of? String
 
-      documents[k] = sentence(v).map(&:stem).join(' ')
+      documents[k] = v.map { |t| @stemmer.stem t }.join(' ')
     end
     documents
   end
@@ -99,7 +101,7 @@ class BM25F
   # @param query [String] The query to preprocess.
   # @return [Array<String>] An array of preprocessed query terms.
   def preprocess_query(query)
-    sentence(query).tokenize.map(&:stem)
+    @tokenizer.tokenize(query).map { |t| @stemmer.stem t }
   end
 
   # Calculates the score of a document using an array of query terms.

data/test/test_bm25f.rb

@@ -5,19 +5,20 @@ class BM25FTest < Minitest::Test
   def setup
     @bm25f = BM25F.new
     @documents = [
-      { title: 'hello world', content: 'foo bar baz' },
-      { title: 'foo bar', content: 'goodbye, world!', test: nil }
+      { url: 'https://wikimedia.org', title: 'Wikimedia',
+        content: 'Wikimedia. Wikimedia is a global movement whose mission is to bring free educational content to the world. Through various projects, chapters and the support structure of the ...' },
+      { url: 'https://twitter.com/Wikipedia', title: 'Wikipedia (@Wikipedia) · Twitter', content: nil },
+      { url: 'https://play.google.com/store/apps/details', title: 'Wikipedia - Apps on Google Play',
+        content: 'The best Wikipedia experience on your Mobile device. Ad-free and free of charge, forever. With the official Wikipedia app, you can search and explore 40+ ...' },
+      { url: 'https://www.wikipedia.org', title: 'Wikipedia',
+        content: 'Wikipedia is a free online encyclopedia, created and edited by volunteers around the world and hosted by the Wikimedia Foundation.' }
     ]
   end
 
   def test_score
     @bm25f.fit @documents
-    scores = @bm25f.score 'hello world foo bar baz'
+    scores = @bm25f.score 'wikipedia'
 
-    # Sort
-    scores = scores.to_a.sort_by { |_, v| v.to_i }
-
-    # Checks if the most matching element is the first element
-    assert scores.last[0].zero?
+    puts scores.inspect
   end
 end

metadata

@@ -1,29 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: bm25f
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.6
 platform: ruby
 authors:
 - catflip
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-10 00:00:00.000000000 Z
+date: 2024-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: treat
+  name: pragmatic_tokenizer
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 3.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 3.2.0
+- !ruby/object:Gem::Dependency
+  name: uea-stemmer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.3
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement