isomorfeus-ferret 0.14.1 → 0.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5818fce6d84b9bd4814be3bbed270127e05297dcf85adeebc495c8f334430d88
-  data.tar.gz: 77c9c3246c7777947084b47620d3aeeeb9eb76d7b0a17a4d30a37a38547a54da
+  metadata.gz: 9954cd3c9b84e7689861a8039c8305407b218d683f9cdfc74ee77542ae6f125c
+  data.tar.gz: 8bcf947f08f84fa7c73e157caaa254a8e21e4700b0c852004613e8c9c9517716
 SHA512:
-  metadata.gz: 59632a0b46b9bd247da0f8b3908654a8027fbcef2aadc897f7681d25b03d4404191d037be323f666ef9bae679c72b135318aa853158e6bf0205b754ec3b2b18f
-  data.tar.gz: 2a037003347c6bca0900bf80410e83f43d397400f37e22f112e6ef6893a568dba29561b12594f803f3b28baee9f5f1ae67595c244d91b7dffa9d06e4e493c891
+  metadata.gz: '083b7943b6d5d1dd59d114d5db4dba0d54f1684a7057ce6358b105ac9564908310bdd2df2539791ed30142428b4ec53ab646d2980eb8549998f2cb586b8c6892'
+  data.tar.gz: e5e15c37ffa004850a18eacae06f95535b460522166a1bc4ec611d3a4dae0aaddb7180d06f78d479e397708a167a5372d093ad2173a026d5888a7c2d34b3aef0

data/ext/isomorfeus_ferret_ext/frb_index.c

@@ -2548,7 +2548,7 @@ static void Init_TVTerm(void) {
  *  == Example
  *
  *    tv = index_reader.term_vector(doc_id, :content)
- *    tv_term = tv.find {|tvt| tvt.term = "fox"}
+ *    tv_term = tv.find {|tvt| tvt.term == "fox"}
  *
  *    # get the term frequency
  *    term_freq = tv_term.positions.size

data/ext/isomorfeus_ferret_ext/frt_analysis.c

@@ -66,7 +66,8 @@ static inline int get_cp(char *start, char *end, int *cp_len, rb_encoding *enc)
 
 FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen, frt_off_t start, frt_off_t end, int pos_inc, rb_encoding *encoding) {
     if (tlen >= FRT_MAX_WORD_SIZE) {
-        tlen = FRT_MAX_WORD_SIZE - 1; // TODO: this may invalidate mbc's
+        char *head_last = rb_enc_left_char_head(text, text + FRT_MAX_WORD_SIZE - 1, text + tlen, encoding);
+        tlen = head_last - text;
     }
 
     if (encoding == utf8_encoding) {
@@ -1031,9 +1032,9 @@ static FrtToken *stemf_next(FrtTokenStream *ts) {
     stemmed = sb_stemmer_stem(stemmer, (sb_symbol *)tk->text, tk->len);
     len = sb_stemmer_length(stemmer);
     if (len >= FRT_MAX_WORD_SIZE) {
-        len = FRT_MAX_WORD_SIZE - 1;
+        char *head_last = rb_enc_left_char_head(tk->text, tk->text + FRT_MAX_WORD_SIZE - 1, tk->text + len, utf8_encoding);
+        len = head_last - tk->text;
     }
-
     memcpy(tk->text, stemmed, len);
     tk->text[len] = '\0';
     tk->len = len;

data/ext/isomorfeus_ferret_ext/frt_global.c

@@ -261,30 +261,30 @@ void frt_register_for_cleanup(void *p, frt_free_ft free_func) {
 void frt_init(int argc, const char *const argv[]) {
     atexit(&frt_hash_finalize);
 
-    utf8_encoding = rb_enc_find("UTF-8");
+    utf8_encoding = rb_utf8_encoding();
     utf8_mbmaxlen = rb_enc_mbmaxlen(utf8_encoding);
     char *p = "'";
     cp_apostrophe = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = ".";
-    cp_dot = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = ",";
-    cp_comma = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "\\";
-    cp_backslash = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "/";
-    cp_slash = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "_";
-    cp_underscore = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "-";
-    cp_dash = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "\u2010";
-    cp_hyphen = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "@";
-    cp_at = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "&";
-    cp_ampersand = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = ":";
-    cp_colon = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    char *q = ".";
+    cp_dot = rb_enc_mbc_to_codepoint(q, q + 1, utf8_encoding);
+    char *r = ",";
+    cp_comma = rb_enc_mbc_to_codepoint(r, r + 1, utf8_encoding);
+    char *s = "\\";
+    cp_backslash = rb_enc_mbc_to_codepoint(s, s + 1, utf8_encoding);
+    char *t = "/";
+    cp_slash = rb_enc_mbc_to_codepoint(t, t + 1, utf8_encoding);
+    char *u = "_";
+    cp_underscore = rb_enc_mbc_to_codepoint(u, u + 1, utf8_encoding);
+    char *v = "-";
+    cp_dash = rb_enc_mbc_to_codepoint(v, v + 1, utf8_encoding);
+    char *w = "\u2010";
+    cp_hyphen = rb_enc_mbc_to_codepoint(w, w + 1, utf8_encoding);
+    char *x = "@";
+    cp_at = rb_enc_mbc_to_codepoint(x, x + 1, utf8_encoding);
+    char *y = "&";
+    cp_ampersand = rb_enc_mbc_to_codepoint(y, y + 1, utf8_encoding);
+    char *z = ":";
+    cp_colon = rb_enc_mbc_to_codepoint(z, z + 1, utf8_encoding);
 
     FRT_SORT_FIELD_SCORE = frt_sort_field_alloc();
     FRT_SORT_FIELD_SCORE->field_index_class = NULL;               /* field_index_class */

data/ext/isomorfeus_ferret_ext/frt_index.c

@@ -842,7 +842,7 @@ static void sis_find_segments_file(FrtStore *store, FindSegmentsFile *fsf, void
     volatile frt_i64 last_gen = -1;
     volatile frt_i64 gen = 0;
 
-    /* Loop until we succeed in calling doBody() without hitting an
+    /* Loop until we succeed in calling run() without hitting an
      * IOException. An IOException most likely means a commit was in process
      * and has finished, in the time it took us to load the now-old infos
      * files (and segments files). It's also possible it's a true error
@@ -5047,6 +5047,7 @@ FrtPosting *frt_p_new(FrtMemoryPool *mp, int doc_num, int pos)
 FrtPostingList *frt_pl_new(FrtMemoryPool *mp, const char *term,
                            int term_len, FrtPosting *p)
 {
+    // TODO account for term_len as measured in the original text vs utf8 term_len of term
     FrtPostingList *pl = FRT_MP_ALLOC(mp, FrtPostingList);
     pl->term = (char *)frt_mp_memdup(mp, term, term_len + 1);
     pl->term_len = term_len;
@@ -5449,15 +5450,14 @@ FrtHash *frt_dw_invert_field(FrtDocWriter *dw, FrtFieldInverter *fld_inv, FrtDoc
         for (i = 0; i < df_size; i++) {
             int len = df->lengths[i];
             char *data_ptr = df->data[i];
+            if (len >= FRT_MAX_WORD_SIZE) {
+                char *head_last = rb_enc_left_char_head(data_ptr, data_ptr + FRT_MAX_WORD_SIZE - 1, data_ptr + len, df->encodings[i]);
+                len = head_last - data_ptr;
+            }
             if (df->encodings[i] == utf8_encoding) {
-                if (len >= FRT_MAX_WORD_SIZE) {
-                    len = FRT_MAX_WORD_SIZE - 1;  // TODO: this may invalidate mbc's
-                    data_ptr = (char *)memcpy(buf, df->data[i], len);
-                    buf[len] = '\0';
-                }
+                data_ptr = (char *)memcpy(buf, df->data[i], len);
+                buf[len] = '\0';
             } else if (df->encodings[i] != utf8_encoding) {
-                if (len >= FRT_MAX_WORD_SIZE)
-                    len = FRT_MAX_WORD_SIZE - 1;
                 const unsigned char *sp = (unsigned char *)df->data[i];
                 unsigned char *dp = (unsigned char *)&buf;
                 rb_econv_t *ec = rb_econv_open(rb_enc_name(df->encodings[i]), "UTF-8", RUBY_ECONV_INVALID_REPLACE);

data/lib/isomorfeus/ferret/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Ferret
-    VERSION = '0.14.1'
+    VERSION = '0.14.2'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: isomorfeus-ferret
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.14.2
 platform: ruby
 authors:
 - Jan Biedermann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-01 00:00:00.000000000 Z
+date: 2022-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oj
@@ -381,13 +381,7 @@ licenses:
 metadata:
   github_repo: ssh://github.com/isomorfeus/gems
   source_code_uri: https://github.com/isomorfeus/isomorfeus-ferret
-post_install_message: |2+
-
-  isomorfeus-ferret 0.13:
-  Breaking change:
-    To support Ruby string encodings, die index file format has changed.
-    Indexes created with previous versions < 0.13 must be recreated with 0.13!
-
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -407,4 +401,3 @@ signing_key:
 specification_version: 4
 summary: Indexed document store for Isomorfeus.
 test_files: []
-...