isomorfeus-ferret 0.12.7 → 0.13.2

data/ext/isomorfeus_ferret_ext/frt_q_parser.c

@@ -155,12 +155,15 @@
 
 #include <string.h>
 #include <ctype.h>
-#include <wctype.h>
 #include <assert.h>
 #include "frt_global.h"
 #include "frt_except.h"
 #include "frt_search.h"
 #include "frt_array.h"
+#include <ruby/encoding.h>
+
+extern rb_encoding *utf8_encoding;
+extern int utf8_mbmaxlen;
 
 typedef struct Phrase {
     int             size;
@@ -180,7 +183,7 @@ float frt_qp_default_fuzzy_min_sim = 0.5;
 int frt_qp_default_fuzzy_pre_len = 0;
 
 
-#line 184 "frt_q_parser.c"
+#line 187 "frt_q_parser.c"
 
 # ifndef YY_CAST
 #  ifdef __cplusplus
@@ -237,7 +240,7 @@ extern int yydebug;
 #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
 union YYSTYPE
 {
-#line 113 "frt_q_parser.y"
+#line 116 "frt_q_parser.y"
 
     FrtQuery *query;
     FrtBooleanClause *bcls;
@@ -246,7 +249,7 @@ union YYSTYPE
     Phrase *phrase;
     char *str;
 
-#line 250 "frt_q_parser.c"
+#line 253 "frt_q_parser.c"
 
 };
 typedef union YYSTYPE YYSTYPE;
@@ -257,7 +260,7 @@ typedef union YYSTYPE YYSTYPE;
 
 
 
-int yyparse (FrtQParser *qp);
+int yyparse (FrtQParser *qp, rb_encoding *encoding);
 
 
 
@@ -312,10 +315,10 @@ typedef enum yysymbol_kind_t yysymbol_kind_t;
 
 
 /* Second part of user prologue.  */
-#line 121 "frt_q_parser.y"
+#line 124 "frt_q_parser.y"
 
 static int yylex(YYSTYPE *lvalp, FrtQParser *qp);
-static int yyerror(FrtQParser *qp, char const *msg);
+static int yyerror(FrtQParser *qp, rb_encoding *encoding, char const *msg);
 
 #define PHRASE_INIT_CAPA 4
 static FrtQuery *get_bool_q(FrtBCArray *bca);
@@ -323,29 +326,26 @@ static FrtQuery *get_bool_q(FrtBCArray *bca);
 static FrtBCArray *first_cls(FrtBooleanClause *boolean_clause);
 static FrtBCArray *add_and_cls(FrtBCArray *bca, FrtBooleanClause *clause);
 static FrtBCArray *add_or_cls(FrtBCArray *bca, FrtBooleanClause *clause);
-static FrtBCArray *add_default_cls(FrtQParser *qp, FrtBCArray *bca,
-                                FrtBooleanClause *clause);
+static FrtBCArray *add_default_cls(FrtQParser *qp, FrtBCArray *bca, FrtBooleanClause *clause);
 static void bca_destroy(FrtBCArray *bca);
 
 static FrtBooleanClause *get_bool_cls(FrtQuery *q, FrtBCType occur);
 
-static FrtQuery *get_term_q(FrtQParser *qp, FrtSymbol field, char *word);
-static FrtQuery *get_fuzzy_q(FrtQParser *qp, FrtSymbol field, char *word,
-                          char *slop);
-static FrtQuery *get_wild_q(FrtQParser *qp, FrtSymbol field, char *pattern);
+static FrtQuery *get_term_q(FrtQParser *qp, ID field, char *word, rb_encoding *encoding);
+static FrtQuery *get_fuzzy_q(FrtQParser *qp, ID field, char *word, char *slop, rb_encoding *encoding);
+static FrtQuery *get_wild_q(FrtQParser *qp, ID field, char *pattern, rb_encoding *encoding);
 
 static FrtHashSet *first_field(FrtQParser *qp, const char *field_name);
 static FrtHashSet *add_field(FrtQParser *qp, const char *field_name);
 
-static FrtQuery *get_phrase_q(FrtQParser *qp, Phrase *phrase, char *slop);
+static FrtQuery *get_phrase_q(FrtQParser *qp, Phrase *phrase, char *slop, rb_encoding *encoding);
 
 static Phrase *ph_first_word(char *word);
 static Phrase *ph_add_word(Phrase *self, char *word);
 static Phrase *ph_add_multi_word(Phrase *self, char *word);
 static void ph_destroy(Phrase *self);
 
-static FrtQuery *get_r_q(FrtQParser *qp, FrtSymbol field, char *from, char *to,
-                      bool inc_lower, bool inc_upper);
+static FrtQuery *get_r_q(FrtQParser *qp, ID field, char *from, char *to, bool inc_lower, bool inc_upper, rb_encoding *encoding);
 
 static void qp_push_fields(FrtQParser *self, FrtHashSet *fields, bool destroy);
 static void qp_pop_fields(FrtQParser *self);
@@ -359,17 +359,17 @@ static void qp_pop_fields(FrtQParser *self);
  */
 #define FLDS(q, func) do {\
     FRT_TRY {\
-        FrtSymbol field;\
+        ID field;\
         if (qp->fields->size == 0) {\
             q = NULL;\
         } else if (qp->fields->size == 1) {\
-            field = (FrtSymbol)qp->fields->first->elem;\
+            field = (ID)qp->fields->first->elem;\
             q = func;\
         } else {\
             FrtQuery *volatile sq; FrtHashSetEntry *volatile hse;\
             q = frt_bq_new_max(false, qp->max_clauses);\
             for (hse = qp->fields->first; hse; hse = hse->next) {\
-                field = (FrtSymbol)hse->elem;\
+                field = (ID)hse->elem;\
                 sq = func;\
                 FRT_TRY\
                   if (sq) frt_bq_add_query_nr(q, sq, FRT_BC_SHOULD);\
@@ -969,7 +969,7 @@ enum { YYENOMEM = -2 };
       }                                                           \
     else                                                          \
       {                                                           \
-        yyerror (qp, YY_("syntax error: cannot back up")); \
+        yyerror (qp, encoding, YY_("syntax error: cannot back up")); \
         YYERROR;                                                  \
       }                                                           \
   while (0)
@@ -1002,7 +1002,7 @@ do {                                                                      \
     {                                                                     \
       YYFPRINTF (stderr, "%s ", Title);                                   \
       yy_symbol_print (stderr,                                            \
-                  Kind, Value, qp); \
+                  Kind, Value, qp, encoding); \
       YYFPRINTF (stderr, "\n");                                           \
     }                                                                     \
 } while (0)
@@ -1014,11 +1014,12 @@ do {                                                                      \
 
 static void
 yy_symbol_value_print (FILE *yyo,
-                       yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, FrtQParser *qp)
+                       yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, FrtQParser *qp, rb_encoding *encoding)
 {
   FILE *yyoutput = yyo;
   YY_USE (yyoutput);
   YY_USE (qp);
+  YY_USE (encoding);
   if (!yyvaluep)
     return;
   YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN
@@ -1033,12 +1034,12 @@ yy_symbol_value_print (FILE *yyo,
 
 static void
 yy_symbol_print (FILE *yyo,
-                 yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, FrtQParser *qp)
+                 yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, FrtQParser *qp, rb_encoding *encoding)
 {
   YYFPRINTF (yyo, "%s %s (",
              yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind));
 
-  yy_symbol_value_print (yyo, yykind, yyvaluep, qp);
+  yy_symbol_value_print (yyo, yykind, yyvaluep, qp, encoding);
   YYFPRINTF (yyo, ")");
 }
 
@@ -1072,7 +1073,7 @@ do {                                                            \
 
 static void
 yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp,
-                 int yyrule, FrtQParser *qp)
+                 int yyrule, FrtQParser *qp, rb_encoding *encoding)
 {
   int yylno = yyrline[yyrule];
   int yynrhs = yyr2[yyrule];
@@ -1085,7 +1086,7 @@ yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp,
       YYFPRINTF (stderr, "   $%d = ", yyi + 1);
       yy_symbol_print (stderr,
                        YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]),
-                       &yyvsp[(yyi + 1) - (yynrhs)], qp);
+                       &yyvsp[(yyi + 1) - (yynrhs)], qp, encoding);
       YYFPRINTF (stderr, "\n");
     }
 }
@@ -1093,7 +1094,7 @@ yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp,
 # define YY_REDUCE_PRINT(Rule)          \
 do {                                    \
   if (yydebug)                          \
-    yy_reduce_print (yyssp, yyvsp, Rule, qp); \
+    yy_reduce_print (yyssp, yyvsp, Rule, qp, encoding); \
 } while (0)
 
 /* Nonzero means print parse trace.  It is left uninitialized so that
@@ -1134,10 +1135,11 @@ int yydebug;
 
 static void
 yydestruct (const char *yymsg,
-            yysymbol_kind_t yykind, YYSTYPE *yyvaluep, FrtQParser *qp)
+            yysymbol_kind_t yykind, YYSTYPE *yyvaluep, FrtQParser *qp, rb_encoding *encoding)
 {
   YY_USE (yyvaluep);
   YY_USE (qp);
+  YY_USE (encoding);
   if (!yymsg)
     yymsg = "Deleting";
   YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp);
@@ -1148,67 +1150,67 @@ yydestruct (const char *yymsg,
     case YYSYMBOL_bool_q: /* bool_q  */
 #line 221 "frt_q_parser.y"
             { if (((*yyvaluep).query) && qp->destruct) frt_q_deref(((*yyvaluep).query)); }
-#line 1152 "frt_q_parser.c"
+#line 1154 "frt_q_parser.c"
         break;
 
     case YYSYMBOL_bool_clss: /* bool_clss  */
 #line 223 "frt_q_parser.y"
             { if (((*yyvaluep).bclss) && qp->destruct) bca_destroy(((*yyvaluep).bclss)); }
-#line 1158 "frt_q_parser.c"
+#line 1160 "frt_q_parser.c"
         break;
 
     case YYSYMBOL_bool_cls: /* bool_cls  */
 #line 222 "frt_q_parser.y"
             { if (((*yyvaluep).bcls) && qp->destruct) frt_bc_deref(((*yyvaluep).bcls)); }
-#line 1164 "frt_q_parser.c"
+#line 1166 "frt_q_parser.c"
         break;
 
     case YYSYMBOL_boosted_q: /* boosted_q  */
 #line 221 "frt_q_parser.y"
             { if (((*yyvaluep).query) && qp->destruct) frt_q_deref(((*yyvaluep).query)); }
-#line 1170 "frt_q_parser.c"
+#line 1172 "frt_q_parser.c"
         break;
 
     case YYSYMBOL_q: /* q  */
 #line 221 "frt_q_parser.y"
             { if (((*yyvaluep).query) && qp->destruct) frt_q_deref(((*yyvaluep).query)); }
-#line 1176 "frt_q_parser.c"
+#line 1178 "frt_q_parser.c"
         break;
 
     case YYSYMBOL_term_q: /* term_q  */
 #line 221 "frt_q_parser.y"
             { if (((*yyvaluep).query) && qp->destruct) frt_q_deref(((*yyvaluep).query)); }
-#line 1182 "frt_q_parser.c"
+#line 1184 "frt_q_parser.c"
         break;
 
     case YYSYMBOL_wild_q: /* wild_q  */
 #line 221 "frt_q_parser.y"
             { if (((*yyvaluep).query) && qp->destruct) frt_q_deref(((*yyvaluep).query)); }
-#line 1188 "frt_q_parser.c"
+#line 1190 "frt_q_parser.c"
         break;
 
     case YYSYMBOL_field_q: /* field_q  */
 #line 221 "frt_q_parser.y"
             { if (((*yyvaluep).query) && qp->destruct) frt_q_deref(((*yyvaluep).query)); }
-#line 1194 "frt_q_parser.c"
+#line 1196 "frt_q_parser.c"
         break;
 
     case YYSYMBOL_phrase_q: /* phrase_q  */
 #line 221 "frt_q_parser.y"
             { if (((*yyvaluep).query) && qp->destruct) frt_q_deref(((*yyvaluep).query)); }
-#line 1200 "frt_q_parser.c"
+#line 1202 "frt_q_parser.c"
         break;
 
     case YYSYMBOL_ph_words: /* ph_words  */
 #line 224 "frt_q_parser.y"
             { if (((*yyvaluep).phrase) && qp->destruct) ph_destroy(((*yyvaluep).phrase)); }
-#line 1206 "frt_q_parser.c"
+#line 1208 "frt_q_parser.c"
         break;
 
     case YYSYMBOL_range_q: /* range_q  */
 #line 221 "frt_q_parser.y"
             { if (((*yyvaluep).query) && qp->destruct) frt_q_deref(((*yyvaluep).query)); }
-#line 1212 "frt_q_parser.c"
+#line 1214 "frt_q_parser.c"
         break;
 
       default:
@@ -1227,7 +1229,7 @@ yydestruct (const char *yymsg,
 `----------*/
 
 int
-yyparse (FrtQParser *qp)
+yyparse (FrtQParser *qp, rb_encoding *encoding)
 {
 /* Lookahead token kind.  */
 int yychar;
@@ -1484,269 +1486,269 @@ yyreduce:
   case 2: /* bool_q: %empty  */
 #line 226 "frt_q_parser.y"
                                       {   qp->result = (yyval.query) = NULL; }
-#line 1488 "frt_q_parser.c"
+#line 1490 "frt_q_parser.c"
     break;
 
   case 3: /* bool_q: bool_clss  */
 #line 227 "frt_q_parser.y"
                                       { T qp->result = (yyval.query) = get_bool_q((yyvsp[0].bclss)); E }
-#line 1494 "frt_q_parser.c"
+#line 1496 "frt_q_parser.c"
     break;
 
   case 4: /* bool_clss: bool_cls  */
 #line 229 "frt_q_parser.y"
                                       { T (yyval.bclss) = first_cls((yyvsp[0].bcls)); E }
-#line 1500 "frt_q_parser.c"
+#line 1502 "frt_q_parser.c"
     break;
 
   case 5: /* bool_clss: bool_clss AND bool_cls  */
 #line 230 "frt_q_parser.y"
                                       { T (yyval.bclss) = add_and_cls((yyvsp[-2].bclss), (yyvsp[0].bcls)); E }
-#line 1506 "frt_q_parser.c"
+#line 1508 "frt_q_parser.c"
     break;
 
   case 6: /* bool_clss: bool_clss OR bool_cls  */
 #line 231 "frt_q_parser.y"
                                       { T (yyval.bclss) = add_or_cls((yyvsp[-2].bclss), (yyvsp[0].bcls)); E }
-#line 1512 "frt_q_parser.c"
+#line 1514 "frt_q_parser.c"
     break;
 
   case 7: /* bool_clss: bool_clss bool_cls  */
 #line 232 "frt_q_parser.y"
                                       { T (yyval.bclss) = add_default_cls(qp, (yyvsp[-1].bclss), (yyvsp[0].bcls)); E }
-#line 1518 "frt_q_parser.c"
+#line 1520 "frt_q_parser.c"
     break;
 
   case 8: /* bool_cls: REQ boosted_q  */
 #line 234 "frt_q_parser.y"
                                       { T (yyval.bcls) = get_bool_cls((yyvsp[0].query), FRT_BC_MUST); E }
-#line 1524 "frt_q_parser.c"
+#line 1526 "frt_q_parser.c"
     break;
 
   case 9: /* bool_cls: NOT boosted_q  */
 #line 235 "frt_q_parser.y"
                                       { T (yyval.bcls) = get_bool_cls((yyvsp[0].query), FRT_BC_MUST_NOT); E }
-#line 1530 "frt_q_parser.c"
+#line 1532 "frt_q_parser.c"
     break;
 
   case 10: /* bool_cls: boosted_q  */
 #line 236 "frt_q_parser.y"
                                       { T (yyval.bcls) = get_bool_cls((yyvsp[0].query), FRT_BC_SHOULD); E }
-#line 1536 "frt_q_parser.c"
+#line 1538 "frt_q_parser.c"
     break;
 
   case 12: /* boosted_q: q '^' QWRD  */
 #line 239 "frt_q_parser.y"
                                       { T if ((yyvsp[-2].query)) sscanf((yyvsp[0].str),"%f",&((yyvsp[-2].query)->boost));  (yyval.query)=(yyvsp[-2].query); E }
-#line 1542 "frt_q_parser.c"
+#line 1544 "frt_q_parser.c"
     break;
 
   case 14: /* q: '(' ')'  */
 #line 242 "frt_q_parser.y"
                                       { T (yyval.query) = frt_bq_new_max(true, qp->max_clauses); E }
-#line 1548 "frt_q_parser.c"
+#line 1550 "frt_q_parser.c"
     break;
 
   case 15: /* q: '(' bool_clss ')'  */
 #line 243 "frt_q_parser.y"
                                       { T (yyval.query) = get_bool_q((yyvsp[-1].bclss)); E }
-#line 1554 "frt_q_parser.c"
+#line 1556 "frt_q_parser.c"
     break;
 
   case 20: /* term_q: QWRD  */
 #line 249 "frt_q_parser.y"
-                                      { FLDS((yyval.query), get_term_q(qp, field, (yyvsp[0].str))); Y}
-#line 1560 "frt_q_parser.c"
+                                      { FLDS((yyval.query), get_term_q(qp, field, (yyvsp[0].str), encoding)); Y}
+#line 1562 "frt_q_parser.c"
     break;
 
   case 21: /* term_q: QWRD '~' QWRD  */
 #line 250 "frt_q_parser.y"
-                                      { FLDS((yyval.query), get_fuzzy_q(qp, field, (yyvsp[-2].str), (yyvsp[0].str))); Y}
-#line 1566 "frt_q_parser.c"
+                                      { FLDS((yyval.query), get_fuzzy_q(qp, field, (yyvsp[-2].str), (yyvsp[0].str), encoding)); Y}
+#line 1568 "frt_q_parser.c"
     break;
 
   case 22: /* term_q: QWRD '~'  */
 #line 251 "frt_q_parser.y"
-                                      { FLDS((yyval.query), get_fuzzy_q(qp, field, (yyvsp[-1].str), NULL)); Y}
-#line 1572 "frt_q_parser.c"
+                                      { FLDS((yyval.query), get_fuzzy_q(qp, field, (yyvsp[-1].str), NULL, encoding)); Y}
+#line 1574 "frt_q_parser.c"
     break;
 
   case 23: /* wild_q: WILD_STR  */
 #line 253 "frt_q_parser.y"
-                                      { FLDS((yyval.query), get_wild_q(qp, field, (yyvsp[0].str))); Y}
-#line 1578 "frt_q_parser.c"
+                                      { FLDS((yyval.query), get_wild_q(qp, field, (yyvsp[0].str), encoding)); Y}
+#line 1580 "frt_q_parser.c"
     break;
 
   case 24: /* $@1: %empty  */
 #line 255 "frt_q_parser.y"
                         { qp_pop_fields(qp); }
-#line 1584 "frt_q_parser.c"
+#line 1586 "frt_q_parser.c"
     break;
 
   case 25: /* field_q: field ':' q $@1  */
 #line 256 "frt_q_parser.y"
                                       { (yyval.query) = (yyvsp[-1].query); }
-#line 1590 "frt_q_parser.c"
+#line 1592 "frt_q_parser.c"
     break;
 
   case 26: /* $@2: %empty  */
 #line 257 "frt_q_parser.y"
                 { qp_push_fields(qp, qp->all_fields, false); }
-#line 1596 "frt_q_parser.c"
+#line 1598 "frt_q_parser.c"
     break;
 
   case 27: /* $@3: %empty  */
 #line 257 "frt_q_parser.y"
                                                                      { qp_pop_fields(qp); }
-#line 1602 "frt_q_parser.c"
+#line 1604 "frt_q_parser.c"
     break;
 
   case 28: /* field_q: '*' $@2 ':' q $@3  */
 #line 258 "frt_q_parser.y"
                                       { (yyval.query) = (yyvsp[-1].query); }
-#line 1608 "frt_q_parser.c"
+#line 1610 "frt_q_parser.c"
     break;
 
   case 29: /* field: QWRD  */
 #line 260 "frt_q_parser.y"
                                       { (yyval.hashset) = first_field(qp, (yyvsp[0].str)); }
-#line 1614 "frt_q_parser.c"
+#line 1616 "frt_q_parser.c"
     break;
 
   case 30: /* field: field '|' QWRD  */
 #line 261 "frt_q_parser.y"
                                       { (yyval.hashset) = add_field(qp, (yyvsp[0].str));}
-#line 1620 "frt_q_parser.c"
+#line 1622 "frt_q_parser.c"
     break;
 
   case 31: /* phrase_q: '"' ph_words '"'  */
 #line 263 "frt_q_parser.y"
-                                      { (yyval.query) = get_phrase_q(qp, (yyvsp[-1].phrase), NULL); }
-#line 1626 "frt_q_parser.c"
+                                      { (yyval.query) = get_phrase_q(qp, (yyvsp[-1].phrase), NULL, encoding); }
+#line 1628 "frt_q_parser.c"
     break;
 
   case 32: /* phrase_q: '"' ph_words '"' '~' QWRD  */
 #line 264 "frt_q_parser.y"
-                                      { (yyval.query) = get_phrase_q(qp, (yyvsp[-3].phrase), (yyvsp[0].str)); }
-#line 1632 "frt_q_parser.c"
+                                      { (yyval.query) = get_phrase_q(qp, (yyvsp[-3].phrase), (yyvsp[0].str), encoding); }
+#line 1634 "frt_q_parser.c"
     break;
 
   case 33: /* phrase_q: '"' '"'  */
 #line 265 "frt_q_parser.y"
                                       { (yyval.query) = NULL; }
-#line 1638 "frt_q_parser.c"
+#line 1640 "frt_q_parser.c"
     break;
 
   case 34: /* phrase_q: '"' '"' '~' QWRD  */
 #line 266 "frt_q_parser.y"
                                       { (yyval.query) = NULL; (void)(yyvsp[0].str);}
-#line 1644 "frt_q_parser.c"
+#line 1646 "frt_q_parser.c"
     break;
 
   case 35: /* ph_words: QWRD  */
 #line 268 "frt_q_parser.y"
                               { (yyval.phrase) = ph_first_word((yyvsp[0].str)); }
-#line 1650 "frt_q_parser.c"
+#line 1652 "frt_q_parser.c"
     break;
 
   case 36: /* ph_words: '<' '>'  */
 #line 269 "frt_q_parser.y"
                               { (yyval.phrase) = ph_first_word(NULL); }
-#line 1656 "frt_q_parser.c"
+#line 1658 "frt_q_parser.c"
     break;
 
   case 37: /* ph_words: ph_words QWRD  */
 #line 270 "frt_q_parser.y"
                               { (yyval.phrase) = ph_add_word((yyvsp[-1].phrase), (yyvsp[0].str)); }
-#line 1662 "frt_q_parser.c"
+#line 1664 "frt_q_parser.c"
     break;
 
   case 38: /* ph_words: ph_words '<' '>'  */
 #line 271 "frt_q_parser.y"
                               { (yyval.phrase) = ph_add_word((yyvsp[-2].phrase), NULL); }
-#line 1668 "frt_q_parser.c"
+#line 1670 "frt_q_parser.c"
     break;
 
   case 39: /* ph_words: ph_words '|' QWRD  */
 #line 272 "frt_q_parser.y"
                               { (yyval.phrase) = ph_add_multi_word((yyvsp[-2].phrase), (yyvsp[0].str));  }
-#line 1674 "frt_q_parser.c"
+#line 1676 "frt_q_parser.c"
     break;
 
   case 40: /* range_q: '[' QWRD QWRD ']'  */
 #line 274 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  true,  true)); Y}
-#line 1680 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  true,  true,  encoding)); Y}
+#line 1682 "frt_q_parser.c"
     break;
 
   case 41: /* range_q: '[' QWRD QWRD '}'  */
 #line 275 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  true,  false)); Y}
-#line 1686 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  true,  false, encoding)); Y}
+#line 1688 "frt_q_parser.c"
     break;
 
   case 42: /* range_q: '{' QWRD QWRD ']'  */
 #line 276 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  false, true)); Y}
-#line 1692 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  false, true,  encoding)); Y}
+#line 1694 "frt_q_parser.c"
     break;
 
   case 43: /* range_q: '{' QWRD QWRD '}'  */
 #line 277 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  false, false)); Y}
-#line 1698 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-2].str),  (yyvsp[-1].str),  false, false, encoding)); Y}
+#line 1700 "frt_q_parser.c"
     break;
 
   case 44: /* range_q: '<' QWRD '}'  */
 #line 278 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[-1].str),  false, false)); Y}
-#line 1704 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[-1].str),  false, false, encoding)); Y}
+#line 1706 "frt_q_parser.c"
     break;
 
   case 45: /* range_q: '<' QWRD ']'  */
 #line 279 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[-1].str),  false, true)); Y}
-#line 1710 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[-1].str),  false, true,  encoding)); Y}
+#line 1712 "frt_q_parser.c"
     break;
 
   case 46: /* range_q: '[' QWRD '>'  */
 #line 280 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-1].str),  NULL,true,  false)); Y}
-#line 1716 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-1].str),  NULL,true,  false, encoding)); Y}
+#line 1718 "frt_q_parser.c"
     break;
 
   case 47: /* range_q: '{' QWRD '>'  */
 #line 281 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-1].str),  NULL,false, false)); Y}
-#line 1722 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[-1].str),  NULL,false, false, encoding)); Y}
+#line 1724 "frt_q_parser.c"
     break;
 
   case 48: /* range_q: '<' QWRD  */
 #line 282 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[0].str),  false, false)); Y}
-#line 1728 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[0].str),  false, false, encoding)); Y}
+#line 1730 "frt_q_parser.c"
     break;
 
   case 49: /* range_q: '<' '=' QWRD  */
 #line 283 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[0].str),  false, true)); Y}
-#line 1734 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, NULL,(yyvsp[0].str),  false, true,  encoding)); Y}
+#line 1736 "frt_q_parser.c"
     break;
 
   case 50: /* range_q: '>' '=' QWRD  */
 #line 284 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[0].str),  NULL,true,  false)); Y}
-#line 1740 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[0].str),  NULL,true,  false, encoding)); Y}
+#line 1742 "frt_q_parser.c"
     break;
 
   case 51: /* range_q: '>' QWRD  */
 #line 285 "frt_q_parser.y"
-                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[0].str),  NULL,false, false)); Y}
-#line 1746 "frt_q_parser.c"
+                              { FLDS((yyval.query), get_r_q(qp, field, (yyvsp[0].str),  NULL,false, false, encoding)); Y}
+#line 1748 "frt_q_parser.c"
     break;
 
 
-#line 1750 "frt_q_parser.c"
+#line 1752 "frt_q_parser.c"
 
       default: break;
     }
@@ -1793,7 +1795,7 @@ yyerrlab:
   if (!yyerrstatus)
     {
       ++yynerrs;
-      yyerror (qp, YY_("syntax error"));
+      yyerror (qp, encoding, YY_("syntax error"));
     }
 
   if (yyerrstatus == 3)
@@ -1810,7 +1812,7 @@ yyerrlab:
       else
         {
           yydestruct ("Error: discarding",
-                      yytoken, &yylval, qp);
+                      yytoken, &yylval, qp, encoding);
           yychar = YYEMPTY;
         }
     }
@@ -1866,7 +1868,7 @@ yyerrlab1:
 
 
       yydestruct ("Error: popping",
-                  YY_ACCESSING_SYMBOL (yystate), yyvsp, qp);
+                  YY_ACCESSING_SYMBOL (yystate), yyvsp, qp, encoding);
       YYPOPSTACK (1);
       yystate = *yyssp;
       YY_STACK_PRINT (yyss, yyssp);
@@ -1904,7 +1906,7 @@ yyabortlab:
 | yyexhaustedlab -- YYNOMEM (memory exhaustion) comes here.  |
 `-----------------------------------------------------------*/
 yyexhaustedlab:
-  yyerror (qp, YY_("memory exhausted"));
+  yyerror (qp, encoding, YY_("memory exhausted"));
   yyresult = 2;
   goto yyreturnlab;
 
@@ -1919,7 +1921,7 @@ yyreturnlab:
          user semantic actions for why this is necessary.  */
       yytoken = YYTRANSLATE (yychar);
       yydestruct ("Cleanup: discarding lookahead",
-                  yytoken, &yylval, qp);
+                  yytoken, &yylval, qp, encoding);
     }
   /* Do not reclaim the symbols of the rule whose action triggered
      this YYABORT or YYACCEPT.  */
@@ -1928,7 +1930,7 @@ yyreturnlab:
   while (yyssp != yyss)
     {
       yydestruct ("Cleanup: popping",
-                  YY_ACCESSING_SYMBOL (+*yyssp), yyvsp, qp);
+                  YY_ACCESSING_SYMBOL (+*yyssp), yyvsp, qp, encoding);
       YYPOPSTACK (1);
     }
 #ifndef yyoverflow
@@ -2102,8 +2104,9 @@ static int yylex(YYSTYPE *lvalp, FrtQParser *qp)
  * It is responsible for clearing any memory that was allocated during the
  * parsing process.
  */
-static int yyerror(FrtQParser *qp, char const *msg)
+static int yyerror(FrtQParser *qp, rb_encoding *encoding, char const *msg)
 {
+    (void)encoding;
     qp->destruct = true;
     if (!qp->handle_parse_errors) {
         char buf[1024];
@@ -2133,22 +2136,21 @@ static int yyerror(FrtQParser *qp, char const *msg)
  * This method returns the query parser for a particular field and sets it up
  * with the text to be tokenized.
  */
-static FrtTokenStream *get_cached_ts(FrtQParser *qp, FrtSymbol field, char *text)
-{
+static FrtTokenStream *get_cached_ts(FrtQParser *qp, ID field, char *text, rb_encoding *encoding) {
     FrtTokenStream *ts;
     if (frt_hs_exists(qp->tokenized_fields, (void *)field)) {
         ts = (FrtTokenStream *)frt_h_get(qp->ts_cache, (void *)field);
         if (!ts) {
-            ts = frt_a_get_ts(qp->analyzer, field, text);
+            ts = frt_a_get_ts(qp->analyzer, field, text, encoding);
             frt_h_set(qp->ts_cache, (void *)field, ts);
         }
         else {
-            ts->reset(ts, text);
+            ts->reset(ts, text, encoding);
         }
     }
     else {
         ts = qp->non_tokenizer;
-        ts->reset(ts, text);
+        ts->reset(ts, text, encoding);
     }
     return ts;
 }
@@ -2305,11 +2307,10 @@ static FrtBooleanClause *get_bool_cls(FrtQuery *q, FrtBCType occur)
  * what we want as it will match any documents containing the same email
  * address and tokenized with the same tokenizer.
  */
-static FrtQuery *get_term_q(FrtQParser *qp, FrtSymbol field, char *word)
-{
+static FrtQuery *get_term_q(FrtQParser *qp, ID field, char *word, rb_encoding *encoding) {
     FrtQuery *q;
     FrtToken *token;
-    FrtTokenStream *stream = get_cached_ts(qp, field, word);
+    FrtTokenStream *stream = get_cached_ts(qp, field, word, encoding);
 
     if ((token = frt_ts_next(stream)) == NULL) {
         q = NULL;
@@ -2343,11 +2344,10 @@ static FrtQuery *get_term_q(FrtQParser *qp, FrtSymbol field, char *word)
  * will be used. If there are any more tokens after tokenization, they will be
  * ignored.
  */
-static FrtQuery *get_fuzzy_q(FrtQParser *qp, FrtSymbol field, char *word, char *slop_str)
-{
+static FrtQuery *get_fuzzy_q(FrtQParser *qp, ID field, char *word, char *slop_str, rb_encoding *encoding) {
     FrtQuery *q;
     FrtToken *token;
-    FrtTokenStream *stream = get_cached_ts(qp, field, word);
+    FrtTokenStream *stream = get_cached_ts(qp, field, word, encoding);
 
     if ((token = frt_ts_next(stream)) == NULL) {
         q = NULL;
@@ -2365,31 +2365,20 @@ static FrtQuery *get_fuzzy_q(FrtQParser *qp, FrtSymbol field, char *word, char *
 }
 
 /**
- * Downcase a string taking locale into account and works for multibyte
- * character sets.
+ * Downcase a string taking encoding into account and works for multibyte character sets.
  */
-static char *lower_str(char *str)
-{
-    const int max_len = (int)strlen(str) + 1;
-    int cnt;
-    wchar_t *wstr = FRT_ALLOC_N(wchar_t, max_len);
-    if ((cnt = mbstowcs(wstr, str, max_len)) > 0) {
-        wchar_t *w = wstr;
-        while (*w) {
-            *w = towlower(*w);
-            w++;
-        }
-        wcstombs(str, wstr, max_len);
-    }
-    else {
-        char *s = str;
-        while (*s) {
-            *s = tolower(*s);
-            s++;
-        }
-    }
-    free(wstr);
-    str[max_len] = '\0';
+static char *lower_str(char *str, int len, rb_encoding *enc) {
+    OnigCaseFoldType fold_type = ONIGENC_CASE_DOWNCASE;
+    const int max_len = len + 20; // CASE_MAPPING_ADDITIONAL_LENGTH
+    char *buf = FRT_ALLOC_N(char, max_len);
+    char *buf_end = buf + max_len + 19;
+    const OnigUChar *t = (const OnigUChar *)str;
+
+    len = enc->case_map(&fold_type, &t, (const OnigUChar *)(str + len), (OnigUChar *)buf, (OnigUChar *)buf_end, enc);
+    memcpy(str, buf, len);
+    str[len] = '\0';
+    free(buf);
+
     return str;
 }
 
@@ -2402,8 +2391,7 @@ static char *lower_str(char *str)
  * optimized to a MatchAllQuery if the pattern is '*' or a PrefixQuery if the
  * only wild char (*, ?) in the pattern is a '*' at the end of the pattern.
  */
-static FrtQuery *get_wild_q(FrtQParser *qp, FrtSymbol field, char *pattern)
-{
+static FrtQuery *get_wild_q(FrtQParser *qp, ID field, char *pattern, rb_encoding *encoding) {
     FrtQuery *q;
     bool is_prefix = false;
     char *p;
@@ -2411,7 +2399,7 @@ static FrtQuery *get_wild_q(FrtQParser *qp, FrtSymbol field, char *pattern)
 
     if (qp->wild_lower
         && (!qp->tokenized_fields || frt_hs_exists(qp->tokenized_fields, (void *)field))) {
-        lower_str(pattern);
+        lower_str(pattern, len, encoding);
     }
 
     /* simplify the wildcard query to a prefix query if possible. Basically a
@@ -2446,9 +2434,8 @@ static FrtQuery *get_wild_q(FrtQParser *qp, FrtSymbol field, char *pattern)
 /**
  * Adds another field to the top of the FieldStack.
  */
-static FrtHashSet *add_field(FrtQParser *qp, const char *field_name)
-{
-    FrtSymbol field = rb_intern(field_name);
+static FrtHashSet *add_field(FrtQParser *qp, const char *field_name) {
+    ID field = rb_intern(field_name);
     if (qp->allow_any_fields || frt_hs_exists(qp->all_fields, (void *)field)) {
         frt_hs_add(qp->fields, (void *)field);
     }
@@ -2574,8 +2561,7 @@ static Phrase *ph_add_multi_word(Phrase *self, char *word)
  * This problem can easily be solved by using the StandardTokenizer or any
  * custom tokenizer which will leave dbalmain@gmail.com as a single token.
  */
-static FrtQuery *get_phrase_query(FrtQParser *qp, FrtSymbol field, Phrase *phrase, char *slop_str)
-{
+static FrtQuery *get_phrase_query(FrtQParser *qp, ID field, Phrase *phrase, char *slop_str, rb_encoding *encoding) {
     const int pos_cnt = phrase->size;
     FrtQuery *q = NULL;
 
@@ -2583,7 +2569,7 @@ static FrtQuery *get_phrase_query(FrtQParser *qp, FrtSymbol field, Phrase *phras
         char **words = phrase->positions[0].terms;
         const int word_count = frt_ary_size(words);
         if (word_count == 1) {
-            q = get_term_q(qp, field, words[0]);
+            q = get_term_q(qp, field, words[0], encoding);
         }
         else {
             int i;
@@ -2592,7 +2578,7 @@ static FrtQuery *get_phrase_query(FrtQParser *qp, FrtSymbol field, Phrase *phras
             char *last_word = NULL;
 
             for (i = 0; i < word_count; i++) {
-                token = frt_ts_next(get_cached_ts(qp, field, words[i]));
+                token = frt_ts_next(get_cached_ts(qp, field, words[i], encoding));
                 if (token) {
                     free(words[i]);
                     last_word = words[i] = frt_estrdup(token->text);
@@ -2644,7 +2630,7 @@ static FrtQuery *get_phrase_query(FrtQParser *qp, FrtSymbol field, Phrase *phras
             pos_inc += phrase->positions[i].pos + 1; /* Actually holds pos_inc*/
 
             if (word_count == 1) {
-                stream = get_cached_ts(qp, field, words[0]);
+                stream = get_cached_ts(qp, field, words[0], encoding);
                 while ((token = frt_ts_next(stream))) {
                     if (token->pos_inc) {
                         frt_phq_add_term(q, token->text,
@@ -2661,7 +2647,7 @@ static FrtQuery *get_phrase_query(FrtQParser *qp, FrtSymbol field, Phrase *phras
                 bool added_position = false;
 
                 for (j = 0; j < word_count; j++) {
-                    stream = get_cached_ts(qp, field, words[j]);
+                    stream = get_cached_ts(qp, field, words[j], encoding);
                     if ((token = frt_ts_next(stream))) {
                         if (!added_position) {
                             frt_phq_add_term(q, token->text,
@@ -2685,10 +2671,10 @@ static FrtQuery *get_phrase_query(FrtQParser *qp, FrtSymbol field, Phrase *phras
  * the query parser as the all PhraseQuery didn't work well for this. Once the
  * PhraseQuery has been built the Phrase object needs to be destroyed.
  */
-static FrtQuery *get_phrase_q(FrtQParser *qp, Phrase *phrase, char *slop_str)
+static FrtQuery *get_phrase_q(FrtQParser *qp, Phrase *phrase, char *slop_str, rb_encoding *encoding)
 {
     FrtQuery *volatile q = NULL;
-    FLDS(q, get_phrase_query(qp, field, phrase, slop_str));
+    FLDS(q, get_phrase_query(qp, field, phrase, slop_str, encoding));
     ph_destroy(phrase);
     return q;
 }
@@ -2699,29 +2685,26 @@ static FrtQuery *get_phrase_q(FrtQParser *qp, Phrase *phrase, char *slop_str)
  * Just like with WildCardQuery, RangeQuery needs to downcase its terms if the
  * tokenizer also downcased its terms.
  */
-static FrtQuery *get_r_q(FrtQParser *qp, FrtSymbol field, char *from, char *to, bool inc_lower, bool inc_upper)
-{
+static FrtQuery *get_r_q(FrtQParser *qp, ID field, char *from, char *to, bool inc_lower, bool inc_upper, rb_encoding *encoding) {
     FrtQuery *rq;
     if (qp->wild_lower
         && (!qp->tokenized_fields || frt_hs_exists(qp->tokenized_fields, (void *)field))) {
-        if (from) {
-            lower_str(from);
-        }
-        if (to) {
-            lower_str(to);
-        }
+        if (from)
+            lower_str(from, strlen(from), encoding);
+        if (to)
+            lower_str(to, strlen(to), encoding);
     }
 /*
  * terms don't get tokenized as it doesn't really make sense to do so for
  * range queries.
 
     if (from) {
-        FrtTokenStream *stream = get_cached_ts(qp, field, from);
+        FrtTokenStream *stream = get_cached_ts(qp, field, from, encoding);
         FrtToken *token = frt_ts_next(stream);
         from = token ? frt_estrdup(token->text) : NULL;
     }
     if (to) {
-        FrtTokenStream *stream = get_cached_ts(qp, field, to);
+        FrtTokenStream *stream = get_cached_ts(qp, field, to, encoding);
         FrtToken *token = frt_ts_next(stream);
         to = token ? frt_estrdup(token->text) : NULL;
     }
@@ -2789,20 +2772,16 @@ void frt_qp_destroy(FrtQParser *self)
     assert(NULL == self->fields_top);
 
     frt_h_destroy(self->ts_cache);
-    frt_tk_destroy(self->non_tokenizer);
+    frt_ts_deref(self->non_tokenizer);
     frt_a_deref(self->analyzer);
     free(self);
 }
 
-/**
- * Creates a new QueryParser setting all boolean parameters to their defaults.
- * If +def_fields+ is NULL then +all_fields+ is used in place of +def_fields+.
- * Not also that this method ensures that all fields that exist in
- * +def_fields+ must also exist in +all_fields+. This should make sense.
- */
-FrtQParser *frt_qp_new(FrtAnalyzer *analyzer)
-{
-    FrtQParser *self = FRT_ALLOC(FrtQParser);
+FrtQParser *frt_qp_alloc() {
+    return FRT_ALLOC(FrtQParser);
+}
+
+FrtQParser *frt_qp_init(FrtQParser *self, FrtAnalyzer *analyzer) {
     self->or_default = true;
     self->wild_lower = true;
     self->clean_str = false;
@@ -2830,8 +2809,18 @@ FrtQParser *frt_qp_new(FrtAnalyzer *analyzer)
     return self;
 }
 
-void frt_qp_add_field(FrtQParser *self, FrtSymbol field, bool is_default, bool is_tokenized)
-{
+/**
+ * Creates a new QueryParser setting all boolean parameters to their defaults.
+ * If +def_fields+ is NULL then +all_fields+ is used in place of +def_fields+.
+ * Not also that this method ensures that all fields that exist in
+ * +def_fields+ must also exist in +all_fields+. This should make sense.
+ */
+FrtQParser *frt_qp_new(FrtAnalyzer *analyzer) {
+    FrtQParser *self = frt_qp_alloc();
+    return frt_qp_init(self, analyzer);
+}
+
+void frt_qp_add_field(FrtQParser *self, ID field, bool is_default, bool is_tokenized) {
     frt_hs_add(self->all_fields, (void *)field);
     if (is_default) {
         frt_hs_add(self->def_fields, (void *)field);
@@ -2961,12 +2950,12 @@ char *frt_qp_clean_str(char *str)
  * analyzer. It then turns these tokens (if any) into a boolean query. If it
  * fails to find any tokens, this method will return NULL.
  */
-static FrtQuery *qp_get_bad_query(FrtQParser *qp, char *str)
+static FrtQuery *qp_get_bad_query(FrtQParser *qp, char *str, rb_encoding *encoding)
 {
     FrtQuery *volatile q = NULL;
     qp->recovering = true;
     assert(qp->fields_top->next == NULL);
-    FLDS(q, get_term_q(qp, field, str));
+    FLDS(q, get_term_q(qp, field, str, encoding));
     return q;
 }
 
@@ -2978,40 +2967,63 @@ static FrtQuery *qp_get_bad_query(FrtQParser *qp, char *str)
  * and turns them into a boolean query on the default fields.
  */
 
-FrtQuery *qp_parse(FrtQParser *self, char *qstr)
+FrtQuery *qp_parse(FrtQParser *self, char *query_string, rb_encoding *encoding)
 {
     FrtQuery *result = NULL;
+    char *qstr;
+    unsigned char *dp_start = NULL;
+
     frt_mutex_lock(&self->mutex);
     /* if qp->fields_top->next is not NULL we have a left over field-stack
      * object that was not popped during the last query parse */
     assert(NULL == self->fields_top->next);
 
+    /* encode query_string to utf8 for futher processing unless it is utf8 encoded */
+    if (encoding == utf8_encoding) {
+        qstr = query_string;
+    } else {
+        /* assume query is sbc encoded und encoding to utf results in maximum utf mbc expansion */
+        const unsigned char *sp = (unsigned char *)query_string;
+        int query_string_len = strlen(query_string);
+        int dp_length = query_string_len * utf8_mbmaxlen + 1;
+        unsigned char *dp = FRT_ALLOC_N(unsigned char, dp_length);
+        dp_start = dp;
+        rb_econv_t *ec = rb_econv_open(rb_enc_name(encoding), rb_enc_name(utf8_encoding), RUBY_ECONV_INVALID_REPLACE);
+        assert(ec != NULL);
+        rb_econv_convert(ec, &sp, (unsigned char *)query_string + query_string_len, &dp, (unsigned char *)dp + dp_length - 1, 0);
+        rb_econv_close(ec);
+        *dp = '\0';
+        qstr = (char *)dp_start;
+    }
+
     self->recovering = self->destruct = false;
+
     if (self->clean_str) {
         self->qstrp = self->qstr = frt_qp_clean_str(qstr);
-    }
-    else {
+    } else {
         self->qstrp = self->qstr = qstr;
     }
     self->fields = self->def_fields;
     self->result = NULL;
 
-    if (0 == yyparse(self)) {
+    if (0 == yyparse(self, encoding))
       result = self->result;
-    }
+
     if (!result && self->handle_parse_errors) {
         self->destruct = false;
-        result = qp_get_bad_query(self, self->qstr);
+        result = qp_get_bad_query(self, self->qstr, encoding);
     }
-    if (self->destruct && !self->handle_parse_errors) {
+    if (self->destruct && !self->handle_parse_errors)
         FRT_RAISE(FRT_PARSE_ERROR, frt_xmsg_buffer);
-    }
-    if (!result) {
+
+    if (!result)
         result = frt_bq_new(false);
-    }
-    if (self->clean_str) {
+
+    if (self->clean_str)
         free(self->qstr);
-    }
+    if (dp_start)
+        free(dp_start);
+
     frt_mutex_unlock(&self->mutex);
     return result;
 }