tantiny 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32334d17636719a204b09795443ba26989c1511e515965649af0e92aa0ee5d5a
-  data.tar.gz: 2c596b09325d57012e7987e5c8eba5eb8e9e81f93f5fa3d99859567407f10c9f
+  metadata.gz: 3aeb0d870c24b42e8bf9b607e6ccb6bccff26b0f5f671a84e248d5465008d268
+  data.tar.gz: ea8fc4e425bdeeeb16d6a7db8be220b2742ad9c035b1b1a8c4c365a4ac0cd0f1
 SHA512:
-  metadata.gz: 797d85d76769bf0165f8ecc81d652890d0603806b005de9cece8a3bb6b8b0f6866b4b53fd42caee0738cc43cc9b2e383b1f08ab28f1e706c6231c908bd7334dc
-  data.tar.gz: c683bcb69c47af11da1020cffaa40a9aad40eef358e8c87674b39a9678600f987606db0945632b0d604db4c66a7d634c980fd9446f2674a53c5d42692e4e5913
+  metadata.gz: cf150617442ea2f5829de2a1e4e514310295775cfaad414f9438815fd9a5ab5746d161b69fcb87da1d8ee9a8fbf68b3b7a033f76f49a639e35d06633cc9dddda
+  data.tar.gz: 3a29e2db6f5ba7a71385185058b082ca2420a5dec2805bab73e947c8532ef6f67fde38a61f0d4bf2ef1ced0506e29b7ec75575b58fe7cb3dcaf7e9ef6b07ef25

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.4.1](https://github.com/altertable-ai/tantiny/compare/tantiny/v0.4.0...tantiny/v0.4.1) (2025-11-03)
+
+
+### Bug Fixes
+
+* **highlighting:** ensure the same tokenizer is used for text & query + handle last term as prefix ([#7](https://github.com/altertable-ai/tantiny/issues/7)) ([e713de3](https://github.com/altertable-ai/tantiny/commit/e713de385fea5a8060e615c173e9097871e052fc))
+
 ## [0.4.0](https://github.com/altertable-ai/tantiny/compare/tantiny-v0.3.3...tantiny/v0.4.0) (2025-11-01)
 
 [Resume development & transfer ownership](https://github.com/altertable-ai/tantiny/pull/1) following https://github.com/baygeldin/tantiny/pull/24

data/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantiny"
-version = "0.4.0" # {x-release-please-version}
+version = "0.4.1" # {x-release-please-version}
 edition = "2021"
 authors = ["Sylvain Utard", "Alexander Baygeldin"]
 repository = "https://github.com/altertable-ai/tantiny"

data/lib/tantiny/query.rb

@@ -118,9 +118,8 @@ module Tantiny
         disjunction(*field_queries).boost(boost_factor)
       end
 
-      def highlight(text, query_string, fuzzy_distance: 0, tokenizer: Tantiny::Tokenizer.new(:simple))
-        terms = tokenizer.terms(query_string).map(&:to_s)
-        __highlight(text.to_s, terms, fuzzy_distance)
+      def highlight(text, query_string, fuzzy_distance: 0, tokenizer: Tantiny::Tokenizer.new(:simple), last_term_min_length_prefix_match: 3)
+        __highlight(text.to_s, query_string.to_s, fuzzy_distance.to_i, tokenizer, last_term_min_length_prefix_match.to_i)
       end
 
       private

data/lib/tantiny/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tantiny
-  VERSION = "0.4.0" # {x-release-please-version}
+  VERSION = "0.4.1" # {x-release-please-version}
 end

data/src/query.rs

@@ -7,6 +7,7 @@ use tantivy::Term;
 use time::OffsetDateTime;
 
 use crate::index::Index;
+use crate::tokenizer::Tokenizer;
 
 #[magnus::wrap(class = "Tantiny::Query", free_immediately, size)]
 pub struct Query(Box<dyn tantivy::query::Query>);
@@ -213,27 +214,38 @@ impl Query {
         Query(Box::new(query))
     }
 
-    fn highlight(text: String, terms: Vec<String>, fuzzy_distance: i64) -> Result<String, Error> {
-        use tantivy::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer, TokenStream};
-
-        // Create a simple tokenizer for highlighting
-        let mut analyzer = TextAnalyzer::builder(SimpleTokenizer::default())
-            .filter(LowerCaser)
-            .build();
+    fn highlight(
+        text: String,
+        query_string: String,
+        fuzzy_distance: i64,
+        tokenizer: &Tokenizer,
+        last_term_min_length_prefix_match: i64,
+    ) -> Result<String, Error> {
+        use tantivy::tokenizer::TokenStream;
+        let mut analyzer = tokenizer.get_analyzer();
+
+        // Tokenizer the query string
+        let query_tokens = {
+            let mut query_string_token_stream = analyzer.token_stream(&query_string);
+            let mut tokens = Vec::new();
+            while query_string_token_stream.advance() {
+                let token = query_string_token_stream.token();
+                tokens.push(token.text.clone());
+            }
+            tokens
+        };
 
         // Tokenize the input text
-        let mut token_stream = analyzer.token_stream(&text);
-
-        // Collect all tokens with their positions
+        let mut input_text_token_stream = analyzer.token_stream(&text);
         let mut tokens = Vec::new();
-        while token_stream.advance() {
-            let token = token_stream.token();
+        while input_text_token_stream.advance() {
+            let token = input_text_token_stream.token();
             tokens.push((token.text.clone(), token.offset_from, token.offset_to));
         }
 
         // Build Levenshtein automata for each term (same algorithm as Tantivy's FuzzyTermQuery)
         let lev_builder = LevenshteinAutomatonBuilder::new(fuzzy_distance as u8, true);
-        let automata: Vec<_> = terms
+        let automata: Vec<_> = query_tokens
             .iter()
             .map(|term| lev_builder.build_dfa(term))
             .collect();
@@ -244,7 +256,7 @@ impl Query {
 
         for (token_text, start, end) in tokens {
             // Check if this token matches any of the query terms (exact or fuzzy)
-            let should_highlight = terms.iter().zip(&automata).any(|(term, dfa)| {
+            let fuzzy_match = query_tokens.iter().zip(&automata).any(|(term, dfa)| {
                 // Exact match
                 if token_text.eq_ignore_ascii_case(term) {
                     return true;
@@ -254,6 +266,14 @@ impl Query {
                 matches!(dfa.eval(&token_text), Distance::Exact(_))
             });
 
+            // Check if this token is a prefix match for the last query term
+            let prefix_match = token_text.len() > last_term_min_length_prefix_match as usize
+                && query_tokens
+                    .last()
+                    .map(|last_token| token_text.starts_with(last_token))
+                    .unwrap_or(false);
+            let should_highlight = fuzzy_match || prefix_match;
+
             // Add the text before the token
             result.push_str(&text[last_pos..start]);
 
@@ -297,7 +317,7 @@ pub fn init(ruby: &Ruby, module: RModule) -> Result<(), Error> {
     class.define_singleton_method("__conjunction", magnus::function!(Query::conjunction, 1))?;
     class.define_method("__negation", magnus::method!(Query::negation, 0))?;
     class.define_method("__boost", magnus::method!(Query::boost, 1))?;
-    class.define_singleton_method("__highlight", magnus::function!(Query::highlight, 3))?;
+    class.define_singleton_method("__highlight", magnus::function!(Query::highlight, 5))?;
 
     Ok(())
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tantiny
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Sylvain Utard