isomorfeus-ferret 0.12.0

data/ext/isomorfeus_ferret_ext/q_parser.y

@@ -0,0 +1,1366 @@
+/*****************************************************************************
+ * QueryParser
+ * ===========
+ *
+ * Brief Overview
+ * --------------
+ *
+ * === Creating a QueryParser
+ *
+ *  +qp_new+ allocates a new QueryParser and assigns three very important
+ *  HashSets; +qp->def_fields+, +qp->tkz_fields+ and +qp->all_fields+. The
+ *  query language allows you to assign a field or a set of fields to each
+ *  part of the query.
+ *
+ *    - +qp->def_fields+ is the set of fields that a query is applied to by
+ *      default when no fields are specified.
+ *    - +qp->all_fields+ is the set of fields that gets searched when the user
+ *      requests a search of all fields.
+ *    - +qp->tkz_fields+ is the set of fields that gets tokenized before being
+ *      added to the query parser.
+ *
+ * === qp_parse
+ *
+ *  The main QueryParser method is +qp_parse+. It gets called with a the query
+ *  string and returns a Query object which can then be passed to the
+ *  IndexSearcher. The first thing it does is to clean the query string if
+ *  +qp->clean_str+ is set to true. The cleaning is done with the
+ *  +qp_clean_str+.
+ *
+ *  It then calls the yacc parser which will set +qp->result+ to the parsed
+ *  query. If parsing fails in any way, +qp->result+ should be set to NULL, in
+ *  which case qp_parse does one of two things depending on the value of
+ *  +qp->handle_parse_errors+;
+ *
+ *    - If it is set to true, qp_parse attempts to do a very basic parsing of
+ *      the query by ignoring all special characters and parsing the query as
+ *      a plain boolean query.
+ *    - If it is set to false, qp_parse will raise a PARSE_ERROR and hopefully
+ *      free all allocated memory.
+ *
+ * === The Lexer
+ *
+ *  +yylex+ is the lexing method called by the QueryParser. It breaks the
+ *  query up into special characters;
+ *
+ *      ( "&:()[]{}!\"~^|<>=*?+-" )
+ *
+ *  and tokens;
+ *
+ *    - QWRD
+ *    - WILD_STR
+ *    - AND['AND', '&&']
+ *    - OR['OR', '||']
+ *    - REQ['REQ', '+']
+ *    - NOT['NOT', '-', '~']
+ *
+ *  QWRD tokens are query word tokens which are made up of characters other
+ *  than the special characters. They can also contain special characters when
+ *  escaped with a backslash '\'. WILD_STR is the same as QWRD except that it
+ *  may also contain '?' and '*' characters.
+ *
+ * === The Parser
+ *
+ *  For a better understanding of the how the query parser works, it is a good
+ *  idea to study the Ferret Query Language (FQL) described below. Once you
+ *  understand FQL the one tricky part that needs to be mentioned is how
+ *  fields are handled. This is where +qp->def_fields+ and +qp->all_fields
+ *  come into play. When no fields are specified then the default fields are
+ *  used. The '*:' field specifier will search all fields contained in the
+ *  all_fields set.  Otherwise all fields specified in the field descripter
+ *  separated by '|' will be searched. For example 'title|content:' will
+ *  search the title and content fields. When fields are specified like this,
+ *  the parser will push the fields onto a stack and all queries modified by
+ *  the field specifier will be applied to the fields on top of the stack.
+ *  The parser uses the FLDS macro to handle the current fields. It takes the
+ *  current query building function in the parser and calls it for all the
+ *  current search fields (on top of the stack).
+ *
+ * Ferret Query Language (FQL)
+ * ---------------------------
+ *
+ * FIXME to be continued...
+ *****************************************************************************/
+%{
+#include <string.h>
+#include <ctype.h>
+#include <wctype.h>
+#include <assert.h>
+#include "frt_global.h"
+#include "frt_except.h"
+#include "frt_search.h"
+#include "frt_array.h"
+#include "frt_internal.h"
+
+typedef struct Phrase {
+    int             size;
+    int             capa;
+    int             pos_inc;
+    FrtPhrasePosition *positions;
+} Phrase;
+
+#define BCA_INIT_CAPA 4
+typedef struct BCArray {
+    int size;
+    int capa;
+    BooleanClause **clauses;
+} BCArray;
+
+float qp_default_fuzzy_min_sim = 0.5;
+int qp_default_fuzzy_pre_len = 0;
+
+%}
+%union {
+    FrtQuery *query;
+    BooleanClause *bcls;
+    BCArray *bclss;
+    FrtHashSet *hashset;
+    Phrase *phrase;
+    char *str;
+}
+%{
+static int yylex(YYSTYPE *lvalp, FrtQParser *qp);
+static int yyerror(QParser *qp, char const *msg);
+
+#define PHRASE_INIT_CAPA 4
+static FrtQuery *get_bool_q(BCArray *bca);
+
+static BCArray *first_cls(BooleanClause *boolean_clause);
+static BCArray *add_and_cls(BCArray *bca, BooleanClause *clause);
+static BCArray *add_or_cls(BCArray *bca, BooleanClause *clause);
+static BCArray *add_default_cls(QParser *qp, BCArray *bca,
+                                BooleanClause *clause);
+static void bca_destroy(BCArray *bca);
+
+static BooleanClause *get_bool_cls(FrtQuery *q, BCType occur);
+
+static FrtQuery *get_term_q(QParser *qp, FrtSymbol field, char *word);
+static FrtQuery *get_fuzzy_q(QParser *qp, FrtSymbol field, char *word,
+                          char *slop);
+static FrtQuery *get_wild_q(QParser *qp, FrtSymbol field, char *pattern);
+
+static FrtHashSet *first_field(QParser *qp, const char *field);
+static FrtHashSet *add_field(QParser *qp, const char *field);
+
+static FrtQuery *get_phrase_q(QParser *qp, Phrase *phrase, char *slop);
+
+static Phrase *ph_first_word(char *word);
+static Phrase *ph_add_word(Phrase *self, char *word);
+static Phrase *ph_add_multi_word(Phrase *self, char *word);
+static void ph_destroy(Phrase *self);
+
+static FrtQuery *get_r_q(QParser *qp, FrtSymbol field, char *from, char *to,
+                      bool inc_lower, bool inc_upper);
+
+static void qp_push_fields(QParser *self, FrtHashSet *fields, bool destroy);
+static void qp_pop_fields(QParser *self);
+
+/**
+ * +FLDS+ calls +func+ for all fields on top of the field stack. +func+
+ * must return a query. If there is more than one field on top of FieldStack
+ * then +FLDS+ will combing all the queries returned by +func+ into a single
+ * BooleanQuery which it than assigns to +q+. If there is only one field, the
+ * return value of +func+ is assigned to +q+ directly.
+ */
+#define FLDS(q, func) do {\
+    FRT_TRY {\
+        FrtSymbol field;\
+        if (qp->fields->size == 0) {\
+            q = NULL;\
+        } else if (qp->fields->size == 1) {\
+            field = (Symbol)qp->fields->first->elem;\
+            q = func;\
+        } else {\
+            FrtQuery *volatile sq; FrtHashSetEntry *volatile hse;\
+            q = bq_new_max(false, qp->max_clauses);\
+            for (hse = qp->fields->first; hse; hse = hse->next) {\
+                field = (Symbol)hse->elem;\
+                sq = func;\
+                FRT_TRY\
+                  if (sq) frt_bq_add_query_nr(q, sq, FRT_BC_SHOULD);\
+                FRT_XCATCHALL\
+                  if (sq) frt_q_deref(sq);\
+                FRT_XENDTRY\
+            }\
+            if (((FrtBooleanQuery *)q)->clause_cnt == 0) {\
+                frt_q_deref(q);\
+                q = NULL;\
+            }\
+        }\
+    } FRT_XCATCHALL\
+        qp->destruct = true;\
+        FRT_HANDLED();\
+    FRT_XENDTRY\
+    if (qp->destruct && !qp->recovering && q) {q_deref(q); q = NULL;}\
+} while (0)
+
+#define Y if (qp->destruct) goto yyerrorlab;
+#define T FRT_TRY
+#define E\
+  FRT_XCATCHALL\
+    qp->destruct = true;\
+    FRT_HANDLED();\
+  FRT_XENDTRY\
+  if (qp->destruct) Y;
+%}
+%expect 1
+%pure-parser
+%parse-param { FrtQParser *qp }
+%lex-param   { FrtQParser *qp }
+%token <str>    QWRD WILD_STR
+%type <query>   q bool_q boosted_q term_q wild_q field_q phrase_q range_q
+%type <bcls>    bool_cls
+%type <bclss>   bool_clss
+%type <hashset> field
+%type <phrase>  ph_words
+%nonassoc LOW
+%left AND OR
+%nonassoc REQ NOT
+%left ':'
+%nonassoc HIGH
+%destructor { if ($$ && qp->destruct) frt_q_deref($$); } q bool_q boosted_q term_q wild_q field_q phrase_q range_q
+%destructor { if ($$ && qp->destruct) bc_deref($$); } bool_cls
+%destructor { if ($$ && qp->destruct) bca_destroy($$); } bool_clss
+%destructor { if ($$ && qp->destruct) ph_destroy($$); } ph_words
+%%
+bool_q    : /* Nothing */             {   qp->result = $$ = NULL; }
+          | bool_clss                 { T qp->result = $$ = get_bool_q($1); E }
+          ;
+bool_clss : bool_cls                  { T $$ = first_cls($1); E }
+          | bool_clss AND bool_cls    { T $$ = add_and_cls($1, $3); E }
+          | bool_clss OR bool_cls     { T $$ = add_or_cls($1, $3); E }
+          | bool_clss bool_cls        { T $$ = add_default_cls(qp, $1, $2); E }
+          ;
+bool_cls  : REQ boosted_q             { T $$ = get_bool_cls($2, FRT_BC_MUST); E }
+          | NOT boosted_q             { T $$ = get_bool_cls($2, FRT_BC_MUST_NOT); E }
+          | boosted_q                 { T $$ = get_bool_cls($1, FRT_BC_SHOULD); E }
+          ;
+boosted_q : q
+          | q '^' QWRD                { T if ($1) sscanf($3,"%f",&($1->boost));  $$=$1; E }
+          ;
+q         : term_q
+          | '(' ')'                   { T $$ = bq_new_max(true, qp->max_clauses); E }
+          | '(' bool_clss ')'         { T $$ = get_bool_q($2); E }
+          | field_q
+          | phrase_q
+          | range_q
+          | wild_q
+          ;
+term_q    : QWRD                      { FLDS($$, get_term_q(qp, field, $1)); Y}
+          | QWRD '~' QWRD %prec HIGH  { FLDS($$, get_fuzzy_q(qp, field, $1, $3)); Y}
+          | QWRD '~' %prec LOW        { FLDS($$, get_fuzzy_q(qp, field, $1, NULL)); Y}
+          ;
+wild_q    : WILD_STR                  { FLDS($$, get_wild_q(qp, field, $1)); Y}
+          ;
+field_q   : field ':' q { qp_pop_fields(qp); }
+                                      { $$ = $3; }
+          | '*' { qp_push_fields(qp, qp->all_fields, false); } ':' q { qp_pop_fields(qp); }
+                                      { $$ = $4; }
+          ;
+field     : QWRD                      { $$ = first_field(qp, $1); }
+          | field '|' QWRD            { $$ = add_field(qp, $3);}
+          ;
+phrase_q  : '"' ph_words '"'          { $$ = get_phrase_q(qp, $2, NULL); }
+          | '"' ph_words '"' '~' QWRD { $$ = get_phrase_q(qp, $2, $5); }
+          | '"' '"'                   { $$ = NULL; }
+          | '"' '"' '~' QWRD          { $$ = NULL; (void)$4;}
+          ;
+ph_words  : QWRD              { $$ = ph_first_word($1); }
+          | '<' '>'           { $$ = ph_first_word(NULL); }
+          | ph_words QWRD     { $$ = ph_add_word($1, $2); }
+          | ph_words '<' '>'  { $$ = ph_add_word($1, NULL); }
+          | ph_words '|' QWRD { $$ = ph_add_multi_word($1, $3);  }
+          ;
+range_q   : '[' QWRD QWRD ']' { FLDS($$, get_r_q(qp, field, $2,  $3,  true,  true)); Y}
+          | '[' QWRD QWRD '}' { FLDS($$, get_r_q(qp, field, $2,  $3,  true,  false)); Y}
+          | '{' QWRD QWRD ']' { FLDS($$, get_r_q(qp, field, $2,  $3,  false, true)); Y}
+          | '{' QWRD QWRD '}' { FLDS($$, get_r_q(qp, field, $2,  $3,  false, false)); Y}
+          | '<' QWRD '}'      { FLDS($$, get_r_q(qp, field, NULL,$2,  false, false)); Y}
+          | '<' QWRD ']'      { FLDS($$, get_r_q(qp, field, NULL,$2,  false, true)); Y}
+          | '[' QWRD '>'      { FLDS($$, get_r_q(qp, field, $2,  NULL,true,  false)); Y}
+          | '{' QWRD '>'      { FLDS($$, get_r_q(qp, field, $2,  NULL,false, false)); Y}
+          | '<' QWRD          { FLDS($$, get_r_q(qp, field, NULL,$2,  false, false)); Y}
+          | '<' '=' QWRD      { FLDS($$, get_r_q(qp, field, NULL,$3,  false, true)); Y}
+          | '>' '='  QWRD     { FLDS($$, get_r_q(qp, field, $3,  NULL,true,  false)); Y}
+          | '>' QWRD          { FLDS($$, get_r_q(qp, field, $2,  NULL,false, false)); Y}
+          ;
+%%
+
+static const char *special_char = "&:()[]{}!\"~^|<>=*?+-";
+static const char *not_word =   " \t()[]{}!\"~^|<>=";
+
+/**
+ * +get_word+ gets the next query-word from the query string. A query-word is
+ * basically a string of non-special or escaped special characters. It is
+ * FrtAnalyzer agnostic. It is up to the get_*_q methods to tokenize the word and
+ * turn it into a +Query+. See the documentation for each get_*_q method to
+ * see how it handles tokenization.
+ *
+ * Note that +get_word+ is also responsible for returning field names and
+ * matching the special tokens 'AND', 'NOT', 'REQ' and 'OR'.
+ */
+static int get_word(YYSTYPE *lvalp, FrtQParser *qp)
+{
+    bool is_wild = false;
+    int len;
+    char c;
+    char *buf = qp->buf[qp->buf_index];
+    char *bufp = buf;
+    qp->buf_index = (qp->buf_index + 1) % QP_CONC_WORDS;
+
+    if (qp->dynbuf) {
+        free(qp->dynbuf);
+        qp->dynbuf = NULL;
+    }
+
+    qp->qstrp--; /* need to back up one character */
+
+    while (!strchr(not_word, (c = *qp->qstrp++))) {
+        switch (c) {
+            case '\\':
+                if ((c = *qp->qstrp) == '\0') {
+                    *bufp++ = '\\';
+                }
+                else {
+                    *bufp++ = c;
+                    qp->qstrp++;
+                }
+                break;
+            case ':':
+                if ((*qp->qstrp) == ':') {
+                    qp->qstrp++;
+                    *bufp++ = ':';
+                    *bufp++ = ':';
+                }
+                else {
+                   goto get_word_done;
+                }
+                break;
+            case '*': case '?':
+                is_wild = true;
+                /* fall through */
+            default:
+                *bufp++ = c;
+        }
+        /* we've exceeded the static buffer. switch to the dynamic one. The
+         * dynamic buffer is allocated enough space to hold the whole query
+         * string so it's capacity doesn't need to be checked again once
+         * allocated. */
+        if (!qp->dynbuf && ((bufp - buf) == MAX_WORD_SIZE)) {
+            qp->dynbuf = FRT_ALLOC_AND_ZERO_N(char, strlen(qp->qstr) + 1);
+            strncpy(qp->dynbuf, buf, MAX_WORD_SIZE);
+            buf = qp->dynbuf;
+            bufp = buf + MAX_WORD_SIZE;
+        }
+    }
+get_word_done:
+    qp->qstrp--;
+    /* check for keywords. There are only four so we have a bit of a hack
+     * which just checks for all of them. */
+    *bufp = '\0';
+    len = (int)(bufp - buf);
+    if (qp->use_keywords) {
+        if (len == 3) {
+            if (buf[0] == 'A' && buf[1] == 'N' && buf[2] == 'D') return AND;
+            if (buf[0] == 'N' && buf[1] == 'O' && buf[2] == 'T') return NOT;
+            if (buf[0] == 'R' && buf[1] == 'E' && buf[2] == 'Q') return REQ;
+        }
+        if (len == 2 && buf[0] == 'O' && buf[1] == 'R') return OR;
+    }
+
+    /* found a word so return it. */
+    lvalp->str = buf;
+    if (is_wild) {
+        return WILD_STR;
+    }
+    return QWRD;
+}
+
+/**
+ * +yylex+ is the lexing method called by the QueryParser. It breaks the
+ * query up into special characters;
+ *
+ *     ( "&:()[]{}!\"~^|<>=*?+-" )
+ *
+ * and tokens;
+ *
+ *   - QWRD
+ *   - WILD_STR
+ *   - AND['AND', '&&']
+ *   - OR['OR', '||']
+ *   - REQ['REQ', '+']
+ *   - NOT['NOT', '-', '~']
+ *
+ * QWRD tokens are query word tokens which are made up of characters other
+ * than the special characters. They can also contain special characters when
+ * escaped with a backslash '\'. WILD_STR is the same as QWRD except that it
+ * may also contain '?' and '*' characters.
+ *
+ * If any of the special chars are seen they will usually be returned straight
+ * away. The exceptions are the wild chars '*' and '?', and '&' which will be
+ * treated as a plain old word character unless followed by another '&'.
+ *
+ * If no special characters or tokens are found then yylex delegates to
+ * +get_word+ which will fetch the next query-word.
+ */
+static int yylex(YYSTYPE *lvalp, FrtQParser *qp)
+{
+    char c, nc;
+
+    while ((c=*qp->qstrp++) == ' ' || c == '\t') {
+    }
+
+    if (c == '\0') return 0;
+
+    if (strchr(special_char, c)) {   /* comment */
+        nc = *qp->qstrp;
+        switch (c) {
+            case '-': case '!': return NOT;
+            case '+': return REQ;
+            case '*':
+                if (nc == ':') return c;
+                break;
+            case '?':
+                break;
+            case '&':
+                if (nc == '&') {
+                    qp->qstrp++;
+                    return AND;
+                }
+                break; /* Don't return single & character. Use in word. */
+            case '|':
+                if (nc == '|') {
+                    qp->qstrp++;
+                    return OR;
+                }
+            default:
+                return c;
+        }
+    }
+
+    return get_word(lvalp, qp);
+}
+
+/**
+ * yyerror gets called if there is an parse error with the yacc parser.
+ * It is responsible for clearing any memory that was allocated during the
+ * parsing process.
+ */
+static int yyerror(QParser *qp, char const *msg)
+{
+    qp->destruct = true;
+    if (!qp->handle_parse_errors) {
+        char buf[1024];
+        buf[1023] = '\0';
+        strncpy(buf, qp->qstr, 1023);
+        if (qp->clean_str) {
+            free(qp->qstr);
+        }
+        frt_mutex_unlock(&qp->mutex);
+        snprintf(xmsg_buffer, XMSG_BUFFER_SIZE,
+                 "couldn't parse query ``%s''. Error message "
+                 " was %s", buf, (char *)msg);
+    }
+    while (qp->fields_top->next != NULL) {
+        qp_pop_fields(qp);
+    }
+    return 0;
+}
+
+#define BQ(query) ((FrtBooleanQuery *)(query))
+
+/**
+ * The QueryParser caches a tokenizer for each field so that it doesn't need
+ * to allocate a new tokenizer for each term in the query. This would be quite
+ * expensive as tokenizers use quite a large hunk of memory.
+ *
+ * This method returns the query parser for a particular field and sets it up
+ * with the text to be tokenized.
+ */
+static FrtTokenStream *get_cached_ts(QParser *qp, FrtSymbol field, char *text)
+{
+    FrtTokenStream *ts;
+    if (frt_hs_exists(qp->tokenized_fields, field)) {
+        ts = (FrtTokenStream *)frt_h_get(qp->ts_cache, field);
+        if (!ts) {
+            ts = frt_a_get_ts(qp->analyzer, field, text);
+            frt_h_set(qp->ts_cache, field, ts);
+        }
+        else {
+            ts->reset(ts, text);
+        }
+    }
+    else {
+        ts = qp->non_tokenizer;
+        ts->reset(ts, text);
+    }
+    return ts;
+}
+
+/**
+ * Turns a BooleanClause array into a BooleanQuery. It will optimize the query
+ * if 0 or 1 clauses are present to NULL or the actual query in the clause
+ * respectively.
+ */
+static FrtQuery *get_bool_q(BCArray *bca)
+{
+    FrtQuery *q;
+    const int clause_count = bca->size;
+
+    if (clause_count == 0) {
+        q = NULL;
+        free(bca->clauses);
+    }
+    else if (clause_count == 1) {
+        BooleanClause *bc = bca->clauses[0];
+        if (bc->is_prohibited) {
+            q = frt_bq_new(false);
+            frt_bq_add_query_nr(q, bc->query, FRT_BC_MUST_NOT);
+            frt_bq_add_query_nr(q, frt_maq_new(), FRT_BC_MUST);
+        }
+        else {
+            q = bc->query;
+        }
+        free(bc);
+        free(bca->clauses);
+    }
+    else {
+        q = frt_bq_new(false);
+        /* copy clauses into query */
+
+        BQ(q)->clause_cnt = clause_count;
+        BQ(q)->clause_capa = bca->capa;
+        free(BQ(q)->clauses);
+        BQ(q)->clauses = bca->clauses;
+    }
+    free(bca);
+    return q;
+}
+
+/**
+ * Base method for appending BooleanClauses to a BooleanClause array. This
+ * method doesn't care about the type of clause (MUST, SHOULD, MUST_NOT).
+ */
+static void bca_add_clause(BCArray *bca, BooleanClause *clause)
+{
+    if (bca->size >= bca->capa) {
+        bca->capa <<= 1;
+        FRT_REALLOC_N(bca->clauses, BooleanClause *, bca->capa);
+    }
+    bca->clauses[bca->size] = clause;
+    bca->size++;
+}
+
+/**
+ * Add the first clause to a BooleanClause array. This method is also
+ * responsible for allocating a new BooleanClause array.
+ */
+static BCArray *first_cls(BooleanClause *clause)
+{
+    BCArray *bca = FRT_ALLOC_AND_ZERO(BCArray);
+    bca->capa = BCA_INIT_CAPA;
+    bca->clauses = FRT_ALLOC_N(BooleanClause *, BCA_INIT_CAPA);
+    if (clause) {
+        bca_add_clause(bca, clause);
+    }
+    return bca;
+}
+
+/**
+ * Add AND clause to the BooleanClause array. The means that it will set the
+ * clause being added and the previously added clause from SHOULD clauses to
+ * MUST clauses. (If they are currently MUST_NOT clauses they stay as they
+ * are.)
+ */
+static BCArray *add_and_cls(BCArray *bca, BooleanClause *clause)
+{
+    if (clause) {
+        if (bca->size == 1) {
+            if (!bca->clauses[0]->is_prohibited) {
+                bc_set_occur(bca->clauses[0], FRT_BC_MUST);
+            }
+        }
+        if (!clause->is_prohibited) {
+            bc_set_occur(clause, FRT_BC_MUST);
+        }
+        bca_add_clause(bca, clause);
+    }
+    return bca;
+}
+
+/**
+ * Add SHOULD clause to the BooleanClause array.
+ */
+static BCArray *add_or_cls(BCArray *bca, BooleanClause *clause)
+{
+    if (clause) {
+        bca_add_clause(bca, clause);
+    }
+    return bca;
+}
+
+/**
+ * Add AND or OR clause to the BooleanClause array, depending on the default
+ * clause type.
+ */
+static BCArray *add_default_cls(QParser *qp, BCArray *bca,
+                                BooleanClause *clause)
+{
+    if (qp->or_default) {
+        add_or_cls(bca, clause);
+    }
+    else {
+        add_and_cls(bca, clause);
+    }
+    return bca;
+}
+
+/**
+ * destroy array of BooleanClauses
+ */
+static void bca_destroy(BCArray *bca)
+{
+    int i;
+    for (i = 0; i < bca->size; i++) {
+        bc_deref(bca->clauses[i]);
+    }
+    free(bca->clauses);
+    free(bca);
+}
+
+/**
+ * Turn a query into a BooleanClause for addition to a BooleanQuery.
+ */
+static BooleanClause *get_bool_cls(FrtQuery *q, BCType occur)
+{
+    if (q) {
+        return bc_new(q, occur);
+    }
+    else {
+        return NULL;
+    }
+}
+
+/**
+ * Create a TermQuery. The word will be tokenized and if the tokenization
+ * produces more than one token, a PhraseQuery will be returned. For example,
+ * if the word is dbalmain@gmail.com and a LetterTokenizer is used then a
+ * PhraseQuery "dbalmain gmail com" will be returned which is actually exactly
+ * what we want as it will match any documents containing the same email
+ * address and tokenized with the same tokenizer.
+ */
+static FrtQuery *get_term_q(QParser *qp, FrtSymbol field, char *word)
+{
+    FrtQuery *q;
+    FrtToken *token;
+    FrtTokenStream *stream = get_cached_ts(qp, field, word);
+
+    if ((token = frt_ts_next(stream)) == NULL) {
+        q = NULL;
+    }
+    else {
+        q = frt_tq_new(field, token->text);
+        if ((token = frt_ts_next(stream)) != NULL) {
+            /* Less likely case, destroy the term query and create a
+             * phrase query instead */
+            FrtQuery *phq = frt_phq_new(field);
+            frt_phq_add_term(phq, ((TermQuery *)q)->term, 0);
+            q->destroy_i(q);
+            q = phq;
+            do {
+                if (token->pos_inc) {
+                    frt_phq_add_term(q, token->text, token->pos_inc);
+                    /* add some slop since single term  was expected */
+                    ((FrtPhraseQuery *)q)->slop++;
+                }
+                else {
+                    frt_phq_append_multi_term(q, token->text);
+                }
+            } while ((token = frt_ts_next(stream)) != NULL);
+        }
+    }
+    return q;
+}
+
+/**
+ * Create a FuzzyQuery. The word will be tokenized and only the first token
+ * will be used. If there are any more tokens after tokenization, they will be
+ * ignored.
+ */
+static FrtQuery *get_fuzzy_q(QParser *qp, FrtSymbol field, char *word,
+                          char *slop_str)
+{
+    FrtQuery *q;
+    FrtToken *token;
+    FrtTokenStream *stream = get_cached_ts(qp, field, word);
+
+    if ((token = frt_ts_next(stream)) == NULL) {
+        q = NULL;
+    }
+    else {
+        /* it only makes sense to find one term in a fuzzy query */
+        float slop = qp_default_fuzzy_min_sim;
+        if (slop_str) {
+            sscanf(slop_str, "%f", &slop);
+        }
+        q = frt_fuzq_new_conf(field, token->text, slop, qp_default_fuzzy_pre_len,
+                          qp->max_clauses);
+    }
+    return q;
+}
+
+/**
+ * Downcase a string taking locale into account and works for multibyte
+ * character sets.
+ */
+static char *lower_str(char *str)
+{
+    const int max_len = (int)strlen(str) + 1;
+    int cnt;
+    wchar_t *wstr = FRT_ALLOC_N(wchar_t, max_len);
+    if ((cnt = mbstowcs(wstr, str, max_len)) > 0) {
+        wchar_t *w = wstr;
+        while (*w) {
+            *w = towlower(*w);
+            w++;
+        }
+        wcstombs(str, wstr, max_len);
+    }
+    else {
+        char *s = str;
+        while (*s) {
+            *s = tolower(*s);
+            s++;
+        }
+    }
+    free(wstr);
+    str[max_len] = '\0';
+    return str;
+}
+
+/**
+ * Create a WildCardQuery. No tokenization will be performed on the pattern
+ * but the pattern will be downcased if +qp->wild_lower+ is set to true and
+ * the field in question is a tokenized field.
+ *
+ * Note: this method will not always return a WildCardQuery. It could be
+ * optimized to a MatchAllQuery if the pattern is '*' or a PrefixQuery if the
+ * only wild char (*, ?) in the pattern is a '*' at the end of the pattern.
+ */
+static FrtQuery *get_wild_q(QParser *qp, FrtSymbol field, char *pattern)
+{
+    FrtQuery *q;
+    bool is_prefix = false;
+    char *p;
+    int len = (int)strlen(pattern);
+
+    if (qp->wild_lower
+        && (!qp->tokenized_fields || frt_hs_exists(qp->tokenized_fields, field))) {
+        lower_str(pattern);
+    }
+
+    /* simplify the wildcard query to a prefix query if possible. Basically a
+     * prefix query is any wildcard query that has a '*' as the last character
+     * and no other wildcard characters before it. "*" by itself will expand
+     * to a MatchAllQuery */
+    if (strcmp(pattern, "*") == 0) {
+        return frt_maq_new();
+    }
+    if (pattern[len - 1] == '*') {
+        is_prefix = true;
+        for (p = &pattern[len - 2]; p >= pattern; p--) {
+            if (*p == '*' || *p == '?') {
+                is_prefix = false;
+                break;
+            }
+        }
+    }
+    if (is_prefix) {
+        /* chop off the '*' temporarily to create the query */
+        pattern[len - 1] = 0;
+        q = frt_prefixq_new(field, pattern);
+        pattern[len - 1] = '*';
+    }
+    else {
+        q = frt_wcq_new(field, pattern);
+    }
+    MTQMaxTerms(q) = qp->max_clauses;
+    return q;
+}
+
+/**
+ * Adds another field to the top of the FieldStack.
+ */
+static FrtHashSet *add_field(QParser *qp, const char *field_name)
+{
+    FrtSymbol field = field_name;
+    if (qp->allow_any_fields || frt_hs_exists(qp->all_fields, field)) {
+        frt_hs_add(qp->fields, field);
+    }
+    return qp->fields;
+}
+
+/**
+ * The method gets called when a field modifier ("field1|field2:") is seen. It
+ * will push a new FieldStack object onto the stack and add +field+ to its
+ * fields set.
+ */
+static FrtHashSet *first_field(QParser *qp, const char *field)
+{
+    qp_push_fields(qp, frt_hs_new_ptr(NULL), true);
+    return add_field(qp, field);
+}
+
+/**
+ * Destroy a phrase object freeing all allocated memory.
+ */
+static void ph_destroy(Phrase *self)
+{
+    int i;
+    for (i = 0; i < self->size; i++) {
+        frt_ary_destroy(self->positions[i].terms, &free);
+    }
+    free(self->positions);
+    free(self);
+}
+
+
+/**
+ * Allocate a new Phrase object
+ */
+static Phrase *ph_new()
+{
+  Phrase *self = FRT_ALLOC_AND_ZERO(Phrase);
+  self->capa = PHRASE_INIT_CAPA;
+  self->positions = FRT_ALLOC_AND_ZERO_N(PhrasePosition, PHRASE_INIT_CAPA);
+  return self;
+}
+
+/**
+ * Add the first word to the phrase. This method is also in charge of
+ * allocating a new Phrase object.
+ */
+static Phrase *ph_first_word(char *word)
+{
+    Phrase *self = ph_new();
+    if (word) { /* no point in adding NULL in start */
+        self->positions[0].terms = frt_ary_new_type_capa(char *, 1);
+        frt_ary_push(self->positions[0].terms, frt_estrdup(word));
+        self->size = 1;
+    }
+    return self;
+}
+
+/**
+ * Add a new word to the Phrase
+ */
+static Phrase *ph_add_word(Phrase *self, char *word)
+{
+    if (word) {
+        const int index = self->size;
+        FrtPhrasePosition *pp = self->positions;
+        if (index >= self->capa) {
+            self->capa <<= 1;
+            FRT_REALLOC_N(pp, PhrasePosition, self->capa);
+            self->positions = pp;
+        }
+        pp[index].pos = self->pos_inc;
+        pp[index].terms = frt_ary_new_type_capa(char *, 1);
+        frt_ary_push(pp[index].terms, frt_estrdup(word));
+        self->size++;
+        self->pos_inc = 0;
+    }
+    else {
+        self->pos_inc++;
+    }
+    return self;
+}
+
+/**
+ * Adds a word to the Phrase object in the same position as the previous word
+ * added to the Phrase. This will later be turned into a multi-PhraseQuery.
+ */
+static Phrase *ph_add_multi_word(Phrase *self, char *word)
+{
+    const int index = self->size - 1;
+    FrtPhrasePosition *pp = self->positions;
+
+    if (word) {
+        frt_ary_push(pp[index].terms, frt_estrdup(word));
+    }
+    return self;
+}
+
+/**
+ * Build a phrase query for a single field. It might seem like a better idea
+ * to build the PhraseQuery once and duplicate it for each field but this
+ * would be buggy in the case of PerFieldAnalyzers in which case a different
+ * tokenizer could be used for each field.
+ *
+ * Note that the query object returned by this method is not always a
+ * PhraseQuery. If there is only one term in the query then the query is
+ * simplified to a TermQuery. If there are multiple terms but only a single
+ * position, then a MultiTermQuery is retured.
+ *
+ * Note that each word in the query gets tokenized. Unlike get_term_q, if the
+ * word gets tokenized into more than one token, the rest of the tokens are
+ * ignored. For example, if you have the phrase;
+ *
+ *      "email: dbalmain@gmail.com"
+ *
+ * the Phrase object will contain to positions with the words 'email:' and
+ * 'dbalmain@gmail.com'. Now, if you are using a LetterTokenizer then the
+ * second word will be tokenized into the tokens ['dbalmain', 'gmail', 'com']
+ * and only the first token will be used, so the resulting phrase query will
+ * actually look like this;
+ *
+ *      "email dbalmain"
+ *
+ * This problem can easily be solved by using the StandardTokenizer or any
+ * custom tokenizer which will leave dbalmain@gmail.com as a single token.
+ */
+static FrtQuery *get_phrase_query(QParser *qp, FrtSymbol field,
+                               Phrase *phrase, char *slop_str)
+{
+    const int pos_cnt = phrase->size;
+    FrtQuery *q = NULL;
+
+    if (pos_cnt == 1) {
+        char **words = phrase->positions[0].terms;
+        const int word_count = frt_ary_size(words);
+        if (word_count == 1) {
+            q = get_term_q(qp, field, words[0]);
+        }
+        else {
+            int i;
+            int term_cnt = 0;
+            FrtToken *token;
+            char *last_word = NULL;
+
+            for (i = 0; i < word_count; i++) {
+                token = frt_ts_next(get_cached_ts(qp, field, words[i]));
+                if (token) {
+                    free(words[i]);
+                    last_word = words[i] = frt_estrdup(token->text);
+                    ++term_cnt;
+                }
+                else {
+                    /* empty words will later be ignored */
+                    words[i][0] = '\0';
+                }
+            }
+
+            switch (term_cnt) {
+                case 0:
+                    q = frt_bq_new(false);
+                    break;
+                case 1:
+                    q = frt_tq_new(field, last_word);
+                    break;
+                default:
+                    q = frt_multi_tq_new_conf(field, term_cnt, 0.0);
+                    for (i = 0; i < word_count; i++) {
+                        /* ignore empty words */
+                        if (words[i][0]) {
+                            frt_multi_tq_add_term(q, words[i]);
+                        }
+                    }
+                    break;
+            }
+        }
+    }
+    else if (pos_cnt > 1) {
+        FrtToken *token;
+        FrtTokenStream *stream;
+        int i, j;
+        int pos_inc = 0;
+        q = frt_phq_new(field);
+        if (slop_str) {
+            int slop;
+            sscanf(slop_str,"%d",&slop);
+            ((FrtPhraseQuery *)q)->slop = slop;
+        }
+
+        for (i = 0; i < pos_cnt; i++) {
+            char **words = phrase->positions[i].terms;
+            const int word_count = frt_ary_size(words);
+            if (pos_inc) {
+                ((FrtPhraseQuery *)q)->slop++;
+            }
+            pos_inc += phrase->positions[i].pos + 1; /* Actually holds pos_inc*/
+
+            if (word_count == 1) {
+                stream = get_cached_ts(qp, field, words[0]);
+                while ((token = frt_ts_next(stream))) {
+                    if (token->pos_inc) {
+                        frt_phq_add_term(q, token->text,
+                                     pos_inc ? pos_inc : token->pos_inc);
+                    }
+                    else {
+                        frt_phq_append_multi_term(q, token->text);
+                        ((FrtPhraseQuery *)q)->slop++;
+                    }
+                    pos_inc = 0;
+                }
+            }
+            else {
+                bool added_position = false;
+
+                for (j = 0; j < word_count; j++) {
+                    stream = get_cached_ts(qp, field, words[j]);
+                    if ((token = frt_ts_next(stream))) {
+                        if (!added_position) {
+                            frt_phq_add_term(q, token->text,
+                                         pos_inc ? pos_inc : token->pos_inc);
+                            added_position = true;
+                            pos_inc = 0;
+                        }
+                        else {
+                            frt_phq_append_multi_term(q, token->text);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return q;
+}
+
+/**
+ * Get a phrase query from the Phrase object. The Phrase object is built up by
+ * the query parser as the all PhraseQuery didn't work well for this. Once the
+ * PhraseQuery has been built the Phrase object needs to be destroyed.
+ */
+static FrtQuery *get_phrase_q(QParser *qp, Phrase *phrase, char *slop_str)
+{
+    FrtQuery *volatile q = NULL;
+    FLDS(q, get_phrase_query(qp, field, phrase, slop_str));
+    ph_destroy(phrase);
+    return q;
+}
+
+/**
+ * Gets a RangeQuery object.
+ *
+ * Just like with WildCardQuery, RangeQuery needs to downcase its terms if the
+ * tokenizer also downcased its terms.
+ */
+static FrtQuery *get_r_q(QParser *qp, FrtSymbol field, char *from, char *to,
+                      bool inc_lower, bool inc_upper)
+{
+    FrtQuery *rq;
+    if (qp->wild_lower
+        && (!qp->tokenized_fields || frt_hs_exists(qp->tokenized_fields, field))) {
+        if (from) {
+            lower_str(from);
+        }
+        if (to) {
+            lower_str(to);
+        }
+    }
+/*
+ * terms don't get tokenized as it doesn't really make sense to do so for
+ * range queries.
+
+    if (from) {
+        FrtTokenStream *stream = get_cached_ts(qp, field, from);
+        FrtToken *token = frt_ts_next(stream);
+        from = token ? frt_estrdup(token->text) : NULL;
+    }
+    if (to) {
+        FrtTokenStream *stream = get_cached_ts(qp, field, to);
+        FrtToken *token = frt_ts_next(stream);
+        to = token ? frt_estrdup(token->text) : NULL;
+    }
+*/
+
+    rq = qp->use_typed_range_query ?
+        frt_trq_new(field, from, to, inc_lower, inc_upper) :
+        frt_rq_new(field, from, to, inc_lower, inc_upper);
+    return rq;
+}
+
+/**
+ * Every time the query parser sees a new field modifier ("field1|field2:")
+ * it pushes a new FieldStack object onto the stack and sets its fields to the
+ * fields specified in the fields modifier. If the field modifier is '*',
+ * fs->fields is set to all_fields. fs->fields is set to +qp->def_field+ at
+ * the bottom of the stack (ie the very first set of fields pushed onto the
+ * stack).
+ */
+static void qp_push_fields(QParser *self, FrtHashSet *fields, bool destroy)
+{
+    FieldStack *fs = FRT_ALLOC(FieldStack);
+
+    fs->next    = self->fields_top;
+    fs->fields  = fields;
+    fs->destroy = destroy;
+
+    self->fields_top = fs;
+    self->fields = fields;
+}
+
+/**
+ * Pops the top of the fields stack and frees any memory used by it. This will
+ * get called when query modified by a field modifier ("field1|field2:") has
+ * been fully parsed and the field specifier no longer applies.
+ */
+static void qp_pop_fields(QParser *self)
+{
+    FieldStack *fs = self->fields_top;
+
+    if (fs->destroy) {
+        frt_hs_destroy(fs->fields);
+    }
+    self->fields_top = fs->next;
+    if (self->fields_top) {
+        self->fields = self->fields_top->fields;
+    }
+    free(fs);
+}
+
+/**
+ * Free all memory allocated by the QueryParser.
+ */
+void frt_qp_destroy(QParser *self)
+{
+    if (self->tokenized_fields != self->all_fields) {
+        frt_hs_destroy(self->tokenized_fields);
+    }
+    if (self->def_fields != self->all_fields) {
+        frt_hs_destroy(self->def_fields);
+    }
+    frt_hs_destroy(self->all_fields);
+
+    qp_pop_fields(self);
+    assert(NULL == self->fields_top);
+
+    frt_h_destroy(self->ts_cache);
+    frt_tk_destroy(self->non_tokenizer);
+    frt_a_deref(self->analyzer);
+    free(self);
+}
+
+/**
+ * Creates a new QueryParser setting all boolean parameters to their defaults.
+ * If +def_fields+ is NULL then +all_fields+ is used in place of +def_fields+.
+ * Not also that this method ensures that all fields that exist in
+ * +def_fields+ must also exist in +all_fields+. This should make sense.
+ */
+QParser *qp_new(FrtAnalyzer *analyzer)
+{
+    FrtQParser *self = FRT_ALLOC(QParser);
+    self->or_default = true;
+    self->wild_lower = true;
+    self->clean_str = false;
+    self->max_clauses = QP_MAX_CLAUSES;
+    self->handle_parse_errors = false;
+    self->allow_any_fields = false;
+    self->use_keywords = true;
+    self->use_typed_range_query = false;
+    self->def_slop = 0;
+
+    self->tokenized_fields = frt_hs_new_ptr(NULL);
+    self->all_fields = frt_hs_new_ptr(NULL);
+    self->def_fields = frt_hs_new_ptr(NULL);
+
+    self->fields_top = NULL;
+    qp_push_fields(self, self->def_fields, false);
+
+    /* make sure all_fields contains the default fields */
+    self->analyzer = analyzer;
+    self->ts_cache = frt_h_new_ptr((frt_free_ft)&ts_deref);
+    self->buf_index = 0;
+    self->dynbuf = NULL;
+    self->non_tokenizer = non_tokenizer_new();
+    frt_mutex_init(&self->mutex, NULL);
+    return self;
+}
+
+void frt_qp_add_field(QParser *self,
+                  FrtSymbol field,
+                  bool is_default,
+                  bool is_tokenized)
+{
+    frt_hs_add(self->all_fields, field);
+    if (is_default) {
+        frt_hs_add(self->def_fields, field);
+    }
+    if (is_tokenized) {
+        frt_hs_add(self->tokenized_fields, field);
+    }
+}
+
+/* these chars have meaning within phrases */
+static const char *PHRASE_CHARS = "<>|\"";
+
+/**
+ * +str_insert_char+ inserts a character at the beginning of a string by
+ * shifting the rest of the string right.
+ */
+static void str_insert_char(char *str, int len, char chr)
+{
+    memmove(str+1, str, len*sizeof(char));
+    *str = chr;
+}
+
+/**
+ * +qp_clean_str+ basically scans the query string and ensures that all open
+ * and close parentheses '()' and quotes '"' are balanced. It does this by
+ * inserting or appending extra parentheses or quotes to the string. This
+ * obviously won't necessarily be exactly what the user wanted but we are
+ * never going to know that anyway. The main job of this method is to help the
+ * query at least parse correctly.
+ *
+ * It also checks that all special characters within phrases (ie between
+ * quotes) are escaped correctly unless they have meaning within a phrase
+ * ( <>,|," ). Note that '<' and '>' will also be escaped unless the appear
+ * together like so; '<>'.
+ */
+char *qp_clean_str(char *str)
+{
+    int b, pb = -1;
+    int br_cnt = 0;
+    bool quote_open = false;
+    char *sp, *nsp;
+
+    /* leave a little extra */
+    char *new_str = FRT_ALLOC_N(char, strlen(str)*2 + 1);
+
+    for (sp = str, nsp = new_str; *sp; sp++) {
+        b = *sp;
+        /* ignore escaped characters */
+        if (pb == '\\') {
+            if (quote_open && strrchr(PHRASE_CHARS, b)) {
+                *nsp++ = '\\'; /* this was left off the first time through */
+            }
+            *nsp++ = b;
+            /* \ has escaped itself so has no power. Assign pb random char 'r' */
+            pb = ((b == '\\') ? 'r' : b);
+            continue;
+        }
+        switch (b) {
+            case '\\':
+                if (!quote_open) { /* We do our own escaping below */
+                    *nsp++ = b;
+                }
+                break;
+            case '"':
+                quote_open = !quote_open;
+                *nsp++ = b;
+                break;
+            case '(':
+              if (!quote_open) {
+                  br_cnt++;
+              }
+              else {
+                  *nsp++ = '\\';
+              }
+              *nsp++ = b;
+              break;
+            case ')':
+                if (!quote_open) {
+                    if (br_cnt == 0) {
+                        str_insert_char(new_str, (int)(nsp - new_str), '(');
+                        nsp++;
+                    }
+                    else {
+                        br_cnt--;
+                    }
+                }
+                else {
+                    *nsp++ = '\\';
+                }
+                *nsp++ = b;
+                break;
+            case '>':
+                if (quote_open) {
+                    if (pb == '<') {
+                        /* remove the escape character */
+                        nsp--;
+                        nsp[-1] = '<';
+                    }
+                    else {
+                        *nsp++ = '\\';
+                    }
+                }
+                *nsp++ = b;
+                break;
+            default:
+                if (quote_open) {
+                    if (strrchr(special_char, b) && b != '|') {
+                        *nsp++ = '\\';
+                    }
+                }
+                *nsp++ = b;
+        }
+        pb = b;
+    }
+    if (quote_open) {
+        *nsp++ = '"';
+    }
+    for (;br_cnt > 0; br_cnt--) {
+      *nsp++ = ')';
+    }
+    *nsp = '\0';
+    return new_str;
+}
+
+/**
+ * Takes a string and finds whatever tokens it can using the QueryParser's
+ * analyzer. It then turns these tokens (if any) into a boolean query. If it
+ * fails to find any tokens, this method will return NULL.
+ */
+static FrtQuery *qp_get_bad_query(QParser *qp, char *str)
+{
+    FrtQuery *volatile q = NULL;
+    qp->recovering = true;
+    assert(qp->fields_top->next == NULL);
+    FLDS(q, get_term_q(qp, field, str));
+    return q;
+}
+
+/**
+ * +qp_parse+ takes a string and turns it into a Query object using Ferret's
+ * query language. It must either raise an error or return a query object. It
+ * must not return NULL. If the yacc parser fails it will use a very basic
+ * boolean query parser which takes whatever tokens it can find in the query
+ * and terns them into a boolean query on the default fields.
+ */
+extern VALUE cQueryParseException;
+Query *qp_parse(QParser *self, char *qstr)
+{
+    FrtQuery *result = NULL;
+    frt_mutex_lock(&self->mutex);
+    /* if qp->fields_top->next is not NULL we have a left over field-stack
+     * object that was not popped during the last query parse */
+    assert(NULL == self->fields_top->next);
+
+    self->recovering = self->destruct = false;
+    if (self->clean_str) {
+        self->qstrp = self->qstr = frt_qp_clean_str(qstr);
+    }
+    else {
+        self->qstrp = self->qstr = qstr;
+    }
+    self->fields = self->def_fields;
+    self->result = NULL;
+
+    if (0 == yyparse(self)) result = self->result;
+    if (!result && self->handle_parse_errors) {
+        self->destruct = false;
+        result = qp_get_bad_query(self, self->qstr);
+    }
+    if (self->destruct && !self->handle_parse_errors) {
+        rb_raise(cQueryParseException, xmsg_buffer);
+    }
+    if (!result) {
+        result = frt_bq_new(false);
+    }
+    if (self->clean_str) {
+        free(self->qstr);
+    }
+
+    frt_mutex_unlock(&self->mutex);
+    return result;
+}