isomorfeus-ferret 0.13.1 → 0.13.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 98439b4a9e6ca849246c6e2ddd2ce1bbf117182025b3651d4f6a95593aff0eb6
-  data.tar.gz: 0e1e90c4bfce1014c9983f4bb5be0f3123ef502788f464e988c4671bdd1758ed
+  metadata.gz: a1a8509c12f5d180d38944adb53291958a9f8327a8b1706def9fecd7f9c60e73
+  data.tar.gz: fbbd38e08dd1992cd93663b1b04cc4b666b6a6dc59104414b75515bc2ec4d54d
 SHA512:
-  metadata.gz: da674772ba34175364d0d4d93023ef380eab06334b4f9c3e1956934289a7a2016387607740fb83cc11753a3bb72de62208390933ff9f4cc341524b7fe6c0c6af
-  data.tar.gz: 8261029020f33cb9fb52453e007defdfd2c1dd029f9da22a02b64ae0047bcd333e62e6d6a8d15263ee71089e203f393f5e865138a31784edf48f66a483a0ffce
+  metadata.gz: 7158cdc3f7a0624d35b668b31287d489ee3d389f17a2ab1b58235a7c2be639b6a7bbb3a8c0efee6d168804128fa5c0d7955c00f860b653a76f23d6421fc84c5d
+  data.tar.gz: 60c62db42081291a52e66be4b7e1b4c46eb12a458b0838f3d8793ffaeb013aff27a37aca06ab4831157d1dd65330395bfdaa2fa7924a278261f1187295592042

data/ext/isomorfeus_ferret_ext/frb_index.c

@@ -64,6 +64,7 @@ static ID id_fld_num_map;
 static ID id_field_num;
 static ID id_boost;
 
+extern rb_encoding *utf8_encoding;
 extern void frb_set_term(VALUE rterm, FrtTerm *t);
 extern FrtAnalyzer *frb_get_cwrapped_analyzer(VALUE ranalyzer);
 extern VALUE frb_get_analyzer(FrtAnalyzer *a);
@@ -1219,6 +1220,7 @@ static VALUE frb_get_tv_term(FrtTVTerm *tv_term) {
     VALUE rtext;
     VALUE rpositions = Qnil;
     rtext = rb_str_new2(tv_term->text);
+    rb_enc_associate(rtext, utf8_encoding);
     if (tv_term->positions) {
         int *positions = tv_term->positions;
         rpositions = rb_ary_new2(freq);
@@ -3085,10 +3087,6 @@ static void Init_TermDocEnum(void) {
     rb_define_method(cTermDocEnum, "to_json",        frb_tde_to_json, -1);
 }
 
-/* rdochack
-cTermVector = rb_define_class_under(mIndex, "TermVector", rb_cObject);
-*/
-
 /*
  *  Document-class: Ferret::Index::TermVector::TVOffsets
  *
@@ -3107,9 +3105,6 @@ cTermVector = rb_define_class_under(mIndex, "TermVector", rb_cObject);
  */
 static void Init_TVOffsets(void) {
     const char *tv_offsets_class = "TVOffsets";
-    /* rdochack
-    cTVOffsets = rb_define_class_under(cTermVector, "TVOffsets", rb_cObject);
-    */
     cTVOffsets = rb_struct_define(tv_offsets_class, "start", "end", NULL);
     rb_set_class_path(cTVOffsets, cTermVector, tv_offsets_class);
     rb_const_set(mIndex, rb_intern(tv_offsets_class), cTVOffsets);
@@ -3130,13 +3125,8 @@ static void Init_TVOffsets(void) {
  *    tv_term = tv.find {|tvt| tvt.term = "fox"}
  *    offsets = tv_term.positions.collect {|pos| tv.offsets[pos]}
  */
-static void
-Init_TVTerm(void)
-{
+static void Init_TVTerm(void) {
     const char *tv_term_class = "TVTerm";
-    /* rdochack
-    cTVTerm = rb_define_class_under(cTermVector, "TVTerm", rb_cObject);
-    */
     cTVTerm = rb_struct_define(tv_term_class, "text", "freq", "positions", NULL);
     rb_set_class_path(cTVTerm, cTermVector, tv_term_class);
     rb_const_set(mIndex, rb_intern(tv_term_class), cTVTerm);
@@ -3172,15 +3162,9 @@ Init_TVTerm(void)
  *  particular that you need to store both positions and offsets if you want
  *  to associate offsets with particular terms.
  */
-static void
-Init_TermVector(void)
-{
+static void Init_TermVector(void) {
     const char *tv_class = "TermVector";
-    /* rdochack
-    cTermVector = rb_define_class_under(mIndex, "TermVector", rb_cObject);
-    */
-    cTermVector = rb_struct_define(tv_class,
-                                   "field", "terms", "offsets", NULL);
+    cTermVector = rb_struct_define(tv_class, "field", "terms", "offsets", NULL);
     rb_set_class_path(cTermVector, mIndex, tv_class);
     rb_const_set(mIndex, rb_intern(tv_class), cTermVector);
 

data/ext/isomorfeus_ferret_ext/frt_index.c

@@ -14,6 +14,7 @@
 #undef close
 #undef read
 
+extern rb_encoding *utf8_encoding;
 extern void frt_micro_sleep(const int micro_seconds);
 
 #define GET_LOCK(lock, name, store, err_msg) do {\
@@ -1710,8 +1711,7 @@ static FrtTermVector *frt_fr_read_term_vector(FrtFieldsReader *fr, int field_num
             total_len = delta_start + delta_len;
             frt_is_read_bytes(fdt_in, buffer + delta_start, delta_len);
             buffer[total_len++] = '\0';
-            term->text = (char *)memcpy(FRT_ALLOC_N(char, total_len),
-                                        buffer, total_len);
+            term->text = (char *)memcpy(FRT_ALLOC_N(char, total_len), buffer, total_len);
 
             /* read freq */
             freq = term->freq = frt_is_read_vint(fdt_in);
@@ -1822,8 +1822,7 @@ FrtTermVector *frt_fr_get_field_tv(FrtFieldsReader *fr, int doc_num, int field_n
  *
  ****************************************************************************/
 
-FrtFieldsWriter *frt_fw_open(FrtStore *store, const char *segment, FrtFieldInfos *fis)
-{
+FrtFieldsWriter *frt_fw_open(FrtStore *store, const char *segment, FrtFieldInfos *fis) {
     FrtFieldsWriter *fw = FRT_ALLOC(FrtFieldsWriter);
     char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
     size_t segment_len = strlen(segment);
@@ -1844,8 +1843,7 @@ FrtFieldsWriter *frt_fw_open(FrtStore *store, const char *segment, FrtFieldInfos
     return fw;
 }
 
-void frt_fw_close(FrtFieldsWriter *fw)
-{
+void frt_fw_close(FrtFieldsWriter *fw) {
     frt_os_close(fw->fdt_out);
     frt_os_close(fw->fdx_out);
     frt_ram_destroy_buffer(fw->buffer);
@@ -2046,8 +2044,7 @@ void frt_fw_add_doc(FrtFieldsWriter *fw, FrtDocument *doc) {
     frt_ramo_write_to(fw->buffer, fdt_out);
 }
 
-void frt_fw_write_tv_index(FrtFieldsWriter *fw)
-{
+void frt_fw_write_tv_index(FrtFieldsWriter *fw) {
     int i;
     const int tv_cnt = frt_ary_size(fw->tv_fields);
     FrtOutStream *fdt_out = fw->fdt_out;
@@ -5548,9 +5545,24 @@ FrtHash *frt_dw_invert_field(FrtDocWriter *dw, FrtFieldInverter *fld_inv, FrtDoc
         for (i = 0; i < df_size; i++) {
             int len = df->lengths[i];
             char *data_ptr = df->data[i];
-            if (len > FRT_MAX_WORD_SIZE) {
-                len = FRT_MAX_WORD_SIZE - 1;
-                data_ptr = (char *)memcpy(buf, df->data[i], len);
+            if (df->encodings[i] == utf8_encoding) {
+                if (len >= FRT_MAX_WORD_SIZE) {
+                    len = FRT_MAX_WORD_SIZE - 1;  // TODO: this may invalidate mbc's
+                    data_ptr = (char *)memcpy(buf, df->data[i], len);
+                    buf[len] = '\0';
+                }
+            } else if (df->encodings[i] != utf8_encoding) {
+                if (len >= FRT_MAX_WORD_SIZE)
+                    len = FRT_MAX_WORD_SIZE - 1;
+                const unsigned char *sp = (unsigned char *)df->data[i];
+                unsigned char *dp = (unsigned char *)&buf;
+                rb_econv_t *ec = rb_econv_open(rb_enc_name(df->encodings[i]), "UTF-8", RUBY_ECONV_INVALID_REPLACE);
+                assert(ec != NULL);
+                rb_econv_convert(ec, &sp, (unsigned char *)df->data[i] + len, &dp, (unsigned char *)&buf + FRT_MAX_WORD_SIZE - 1, 0);
+                rb_econv_close(ec);
+                len = dp - (unsigned char *)&buf;
+                buf[len] = '\0';
+                data_ptr = buf;
             }
             dw_add_posting(mp, curr_plists, fld_plists, doc_num, data_ptr, len, i);
             if (store_offsets) {

data/lib/isomorfeus/ferret/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Ferret
-    VERSION = '0.13.1'
+    VERSION = '0.13.2'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: isomorfeus-ferret
 version: !ruby/object:Gem::Version
-  version: 0.13.1
+  version: 0.13.2
 platform: ruby
 authors:
 - Jan Biedermann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-16 00:00:00.000000000 Z
+date: 2022-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake