ferret 0.11.6 → 0.11.8.4

data/ext/q_prefix.c

@@ -1,5 +1,7 @@
 #include <string.h>
 #include "search.h"
+#include "symbol.h"
+#include "internal.h"
 
 /****************************************************************************
  *
@@ -9,23 +11,20 @@
 
 #define PfxQ(query) ((PrefixQuery *)(query))
 
-static char *prq_to_s(Query *self, const char *current_field) 
+static char *prq_to_s(Query *self, Symbol default_field)
 {
     char *buffer, *bptr;
     const char *prefix = PfxQ(self)->prefix;
-    const char *field = PfxQ(self)->field;
     size_t plen = strlen(prefix);
-    size_t flen = strlen(field);
+    size_t flen = sym_len(PfxQ(self)->field);
 
     bptr = buffer = ALLOC_N(char, plen + flen + 35);
 
-    if (strcmp(field, current_field) != 0) {
-        sprintf(bptr, "%s:", field);
-        bptr += flen + 1;
+    if (PfxQ(self)->field != default_field) {
+        bptr += sprintf(bptr, "%s:", S(PfxQ(self)->field));
     }
 
-    sprintf(bptr, "%s*", prefix);
-    bptr += plen + 1;
+    bptr += sprintf(bptr, "%s*", prefix);
     if (self->boost != 1.0) {
         *bptr = '^';
         dbl_to_s(++bptr, self->boost);
@@ -36,9 +35,9 @@ static char *prq_to_s(Query *self, const char *current_field)
 
 static Query *prq_rewrite(Query *self, IndexReader *ir)
 {
-    const char *field = PfxQ(self)->field;
-    const int field_num = fis_get_field_num(ir->fis, field);
-    Query *volatile q = multi_tq_new_conf(field, MTQMaxTerms(self), 0.0);
+    const int field_num = fis_get_field_num(ir->fis, PfxQ(self)->field);
+    Query *volatile q = multi_tq_new_conf(PfxQ(self)->field,
+                                          MTQMaxTerms(self), 0.0);
     q->boost = self->boost;        /* set the boost */
 
     if (field_num >= 0) {
@@ -48,7 +47,7 @@ static Query *prq_rewrite(Query *self, IndexReader *ir)
         size_t prefix_len = strlen(prefix);
 
         TRY
-            do { 
+            do {
                 if (strncmp(term, prefix, prefix_len) != 0) {
                     break;
                 }
@@ -64,27 +63,26 @@ static Query *prq_rewrite(Query *self, IndexReader *ir)
 
 static void prq_destroy(Query *self)
 {
-    free(PfxQ(self)->field);
     free(PfxQ(self)->prefix);
     q_destroy_i(self);
 }
 
 static unsigned long prq_hash(Query *self)
 {
-    return str_hash(PfxQ(self)->field) ^ str_hash(PfxQ(self)->prefix);
+    return sym_hash(PfxQ(self)->field) ^ str_hash(PfxQ(self)->prefix);
 }
 
 static int prq_eq(Query *self, Query *o)
 {
-    return (strcmp(PfxQ(self)->prefix, PfxQ(o)->prefix) == 0) 
-        && (strcmp(PfxQ(self)->field,  PfxQ(o)->field) == 0);
+    return (strcmp(PfxQ(self)->prefix, PfxQ(o)->prefix) == 0)
+        && (PfxQ(self)->field == PfxQ(o)->field);
 }
 
-Query *prefixq_new(const char *field, const char *prefix)
+Query *prefixq_new(Symbol field, const char *prefix)
 {
     Query *self = q_new(PrefixQuery);
 
-    PfxQ(self)->field       = estrdup(field);
+    PfxQ(self)->field       = field;
     PfxQ(self)->prefix      = estrdup(prefix);
     MTQMaxTerms(self)       = PREFIX_QUERY_MAX_TERMS;
 

data/ext/q_range.c

@@ -1,5 +1,7 @@
 #include <string.h>
 #include "search.h"
+#include "symbol.h"
+#include "internal.h"
 
 /*****************************************************************************
  *
@@ -9,26 +11,27 @@
 
 typedef struct Range
 {
-    char *field;
+    Symbol field;
     char *lower_term;
     char *upper_term;
     bool include_lower : 1;
     bool include_upper : 1;
 } Range;
 
-static char *range_to_s(Range *range, const char *field, float boost)
+static char *range_to_s(Range *range, Symbol default_field, float boost)
 {
     char *buffer, *b;
     size_t flen, llen, ulen;
+    const char *field = S(range->field);
 
-    flen = strlen(range->field);
+    flen = strlen(field);
     llen = range->lower_term ? strlen(range->lower_term) : 0;
     ulen = range->upper_term ? strlen(range->upper_term) : 0;
     buffer = ALLOC_N(char, flen + llen + ulen + 40);
     b = buffer;
 
-    if (strcmp(field, range->field)) {
-        memcpy(buffer, range->field, flen * sizeof(char));
+    if (default_field != range->field) {
+        memcpy(buffer, field, flen * sizeof(char));
         b += flen;
         *b = ':';
         b++;
@@ -68,7 +71,6 @@ static char *range_to_s(Range *range, const char *field, float boost)
 
 static void range_destroy(Range *range)
 {
-    free(range->field);
     free(range->lower_term);
     free(range->upper_term);
     free(range);
@@ -77,26 +79,26 @@ static void range_destroy(Range *range)
 static unsigned long range_hash(Range *filt)
 {
     return filt->include_lower | (filt->include_upper << 1)
-        | ((str_hash(filt->field)
+        | ((sym_hash(filt->field)
             ^ (filt->lower_term ? str_hash(filt->lower_term) : 0)
             ^ (filt->upper_term ? str_hash(filt->upper_term) : 0)) << 2);
 }
 
-static int str_eq(char *s1, char *s2)
+static int str_eq(const char *s1, const char *s2)
 {
     return (s1 && s2 && (strcmp(s1, s2) == 0)) || (s1 == s2);
 }
 
 static int range_eq(Range *filt, Range *o)
 {
-    return (str_eq(filt->field, o->field)
+    return ((filt->field == o->field)
             && str_eq(filt->lower_term, o->lower_term)
             && str_eq(filt->upper_term, o->upper_term)
             && (filt->include_lower == o->include_lower)
             && (filt->include_upper == o->include_upper));
 }
 
-Range *range_new(const char *field, const char *lower_term,
+static Range *range_new(Symbol field, const char *lower_term,
                  const char *upper_term, bool include_lower,
                  bool include_upper)
 {
@@ -123,7 +125,61 @@ Range *range_new(const char *field, const char *lower_term,
 
     range = ALLOC(Range);
 
-    range->field = estrdup((char *)field);
+    range->field = field;
+    range->lower_term = lower_term ? estrdup(lower_term) : NULL;
+    range->upper_term = upper_term ? estrdup(upper_term) : NULL;
+    range->include_lower = include_lower;
+    range->include_upper = include_upper;
+    return range;
+}
+
+static Range *trange_new(Symbol field, const char *lower_term,
+                  const char *upper_term, bool include_lower,
+                  bool include_upper)
+{
+    Range *range; 
+    int len;
+    double upper_num, lower_num;
+
+    if (!lower_term && !upper_term) {
+        RAISE(ARG_ERROR, "Nil bounds for range. A range must include either "
+              "lower bound or an upper bound");
+    }
+    if (include_lower && !lower_term) {
+        RAISE(ARG_ERROR, "Lower bound must be non-nil to be inclusive. That "
+              "is, if you specify :include_lower => true when you create a "
+              "range you must include a :lower_term");
+    }
+    if (include_upper && !upper_term) {
+        RAISE(ARG_ERROR, "Upper bound must be non-nil to be inclusive. That "
+              "is, if you specify :include_upper => true when you create a "
+              "range you must include a :upper_term");
+    }
+    if (upper_term && lower_term) {
+        if ((!lower_term ||
+             (sscanf(lower_term, "%lg%n", &lower_num, &len) &&
+              (int)strlen(lower_term) == len)) &&
+            (!upper_term ||
+             (sscanf(upper_term, "%lg%n", &upper_num, &len) &&
+              (int)strlen(upper_term) == len)))
+        {
+            if (upper_num < lower_num) {
+                RAISE(ARG_ERROR, "Upper bound must be greater than lower bound."
+                      " numbers \"%lg\" < \"%lg\"", upper_num, lower_num);
+            }
+        }
+        else {
+            if (upper_term && lower_term &&
+                (strcmp(upper_term, lower_term) < 0)) {
+                RAISE(ARG_ERROR, "Upper bound must be greater than lower bound."
+                      " \"%s\" < \"%s\"", upper_term, lower_term);
+            }
+        }
+    }
+
+    range = ALLOC(Range);
+
+    range->field = field;
     range->lower_term = lower_term ? estrdup(lower_term) : NULL;
     range->upper_term = upper_term ? estrdup(upper_term) : NULL;
     range->include_lower = include_lower;
@@ -153,7 +209,7 @@ static void rfilt_destroy_i(Filter *filt)
 
 static char *rfilt_to_s(Filter *filt)
 {
-    char *rstr = range_to_s(RF(filt)->range, "", 1.0);
+    char *rstr = range_to_s(RF(filt)->range, NULL, 1.0);
     char *rfstr = strfmt("RangeFilter< %s >", rstr);
     free(rstr);
     return rfstr;
@@ -229,7 +285,7 @@ static int rfilt_eq(Filter *filt, Filter *o)
     return range_eq(RF(filt)->range, RF(o)->range);
 }
 
-Filter *rfilt_new(const char *field,
+Filter *rfilt_new(Symbol field,
                   const char *lower_term, const char *upper_term,
                   bool include_lower, bool include_upper)
 {
@@ -245,6 +301,143 @@ Filter *rfilt_new(const char *field,
     return filt;
 }
 
+/***************************************************************************
+ *
+ * RangeFilter
+ *
+ ***************************************************************************/
+
+static char *trfilt_to_s(Filter *filt)
+{
+    char *rstr = range_to_s(RF(filt)->range, NULL, 1.0);
+    char *rfstr = strfmt("TypedRangeFilter< %s >", rstr);
+    free(rstr);
+    return rfstr;
+}
+
+typedef enum {
+    TRC_NONE    = 0x00,
+    TRC_LE      = 0x01,
+    TRC_LT      = 0x02,
+    TRC_GE      = 0x04,
+    TRC_GE_LE   = 0x05,
+    TRC_GE_LT   = 0x06,
+    TRC_GT      = 0x08,
+    TRC_GT_LE   = 0x09,
+    TRC_GT_LT   = 0x0a
+} TypedRangeCheck;
+
+#define SET_DOCS(cond)\
+do {\
+    if (term[0] > '9') break; /* done */\
+    sscanf(term, "%lg%n", &num, &len);\
+    if (len == te->curr_term_len) { /* We have a number */\
+        if (cond) {\
+            tde->seek_te(tde, te);\
+            while (tde->next(tde)) {\
+                bv_set(bv, tde->doc_num(tde));\
+            }\
+        }\
+    }\
+} while (te->next(te))
+  
+
+static BitVector *trfilt_get_bv_i(Filter *filt, IndexReader *ir)
+{
+    Range *range = RF(filt)->range;
+    double lnum = 0.0, unum = 0.0;
+    int len = 0;
+    const char *lt = range->lower_term;
+    const char *ut = range->upper_term;
+    if ((!lt || (sscanf(lt, "%lg%n", &lnum, &len) && (int)strlen(lt) == len)) &&
+        (!ut || (sscanf(ut, "%lg%n", &unum, &len) && (int)strlen(ut) == len)))
+    {
+        BitVector *bv = bv_new_capa(ir->max_doc(ir));
+        FieldInfo *fi = fis_get_field(ir->fis, range->field);
+        /* the field info exists we need to add docs to the bit vector,
+         * otherwise we just return an empty bit vector */
+        if (fi) {
+            const int field_num = fi->number;
+            char *term;
+            double num;
+            TermEnum* te;
+            TermDocEnum *tde;
+            TypedRangeCheck check = TRC_NONE;
+
+            te = ir->terms(ir, field_num);
+            if (te->skip_to(te, "+.") == NULL) {
+                te->close(te);
+                return bv;
+            }
+
+            tde = ir->term_docs(ir);
+            term = te->curr_term;
+
+            if (lt) {
+                check = range->include_lower ? TRC_GE : TRC_GT;
+            }
+            if (ut) {
+               check = (TypedRangeCheck)(check | (range->include_upper
+                                                  ? TRC_LE
+                                                  : TRC_LT));
+            }
+
+            switch(check) {
+                case TRC_LE:
+                    SET_DOCS(num <= unum);
+                    break;
+                case TRC_LT:
+                    SET_DOCS(num <  unum);
+                    break;
+                case TRC_GE:
+                    SET_DOCS(num >= lnum);
+                    break;
+                case TRC_GE_LE:
+                    SET_DOCS(num >= lnum && num <= unum);
+                    break;
+                case TRC_GE_LT:
+                    SET_DOCS(num >= lnum && num <  unum);
+                    break;
+                case TRC_GT:
+                    SET_DOCS(num >  lnum);
+                    break;
+                case TRC_GT_LE:
+                    SET_DOCS(num >  lnum && num <= unum);
+                    break;
+                case TRC_GT_LT:
+                    SET_DOCS(num >  lnum && num <  unum);
+                    break;
+                case TRC_NONE:
+                    /* should never happen. Error should have been raised */
+                    assert(false);
+            }
+            tde->close(tde);
+            te->close(te);
+        }
+
+        return bv;
+    }
+    else {
+        return rfilt_get_bv_i(filt, ir);
+    }
+}
+
+Filter *trfilt_new(Symbol field,
+                   const char *lower_term, const char *upper_term,
+                   bool include_lower, bool include_upper)
+{
+    Filter *filt = filt_new(RangeFilter);
+    RF(filt)->range =  trange_new(field, lower_term, upper_term,
+                                  include_lower, include_upper); 
+
+    filt->get_bv_i  = &trfilt_get_bv_i;
+    filt->hash      = &rfilt_hash;
+    filt->eq        = &rfilt_eq;
+    filt->to_s      = &trfilt_to_s;
+    filt->destroy_i = &rfilt_destroy_i;
+    return filt;
+}
+
 /*****************************************************************************
  *
  * RangeQuery
@@ -258,7 +451,7 @@ typedef struct RangeQuery
     Range *range;
 } RangeQuery;
 
-static char *rq_to_s(Query *self, const char *field)
+static char *rq_to_s(Query *self, Symbol field)
 {
     return range_to_s(RQ(self)->range, field, self->boost);
 }
@@ -273,23 +466,29 @@ static MatchVector *rq_get_matchv_i(Query *self, MatchVector *mv,
                                     TermVector *tv)
 {
     Range *range = RQ(((ConstantScoreQuery *)self)->original)->range;
-    if (strcmp(tv->field, range->field) == 0) {
+    if (tv->field == range->field) {
+        const int term_cnt = tv->term_cnt;
         int i, j;
         char *upper_text = range->upper_term;
         char *lower_text = range->lower_term;
         int upper_limit = range->include_upper ? 1 : 0;
-        int lower_limit = range->include_lower ? 1 : 0;
 
-        for (i = tv->term_cnt - 1; i >= 0; i--) {
+        i = lower_text ? tv_scan_to_term_index(tv, lower_text) : 0;
+        if (i < term_cnt && !range->include_lower && lower_text
+            && 0 == strcmp(lower_text, tv->terms[i].text)) {
+            i++;
+        }
+
+        for (; i < term_cnt; i++) {
             TVTerm *tv_term = &(tv->terms[i]);
             char *text = tv_term->text;
-            if ((!upper_text || strcmp(text, upper_text) < upper_limit) && 
-                (!lower_text || strcmp(lower_text, text) < lower_limit)) {
-
-                for (j = 0; j < tv_term->freq; j++) {
-                    int pos = tv_term->positions[j];
-                    matchv_add(mv, pos, pos);
-                }
+            const int tv_term_freq = tv_term->freq;
+            if (upper_text && strcmp(text, upper_text) >= upper_limit) {
+                break;
+            } 
+            for (j = 0; j < tv_term_freq; j++) {
+                int pos = tv_term->positions[j];
+                matchv_add(mv, pos, pos);
             }
         }
     }
@@ -319,25 +518,26 @@ static int rq_eq(Query *self, Query *o)
     return range_eq(RQ(self)->range, RQ(o)->range);
 }
 
-Query *rq_new_less(const char *field, const char *upper_term,
+Query *rq_new_less(Symbol field, const char *upper_term,
                    bool include_upper)
 {
     return rq_new(field, NULL, upper_term, false, include_upper);
 }
 
-Query *rq_new_more(const char *field, const char *lower_term,
+Query *rq_new_more(Symbol field, const char *lower_term,
                    bool include_lower)
 {
     return rq_new(field, lower_term, NULL, include_lower, false);
 }
 
-Query *rq_new(const char *field, const char *lower_term,
+Query *rq_new(Symbol field, const char *lower_term,
               const char *upper_term, bool include_lower, bool include_upper)
 {
-    Query *self     = q_new(RangeQuery);
-
-    RQ(self)->range = range_new(field, lower_term, upper_term,
-                                include_lower, include_upper); 
+    Query *self;
+    Range *range            = range_new(field, lower_term, upper_term,
+                                        include_lower, include_upper); 
+    self                    = q_new(RangeQuery);
+    RQ(self)->range         = range;
 
     self->type              = RANGE_QUERY;
     self->rewrite           = &rq_rewrite;
@@ -348,3 +548,135 @@ Query *rq_new(const char *field, const char *lower_term,
     self->create_weight_i   = &q_create_weight_unsup;
     return self;
 }
+
+/*****************************************************************************
+ *
+ * TypedRangeQuery
+ *
+ *****************************************************************************/
+
+#define SET_TERMS(cond)\
+for (i = tv->term_cnt - 1; i >= 0; i--) {\
+    TVTerm *tv_term = &(tv->terms[i]);\
+    char *text = tv_term->text;\
+    double num;\
+    sscanf(text, "%lg%n", &num, &len);\
+    if ((int)strlen(text) == len) { /* We have a number */\
+        if (cond) {\
+            const int tv_term_freq = tv_term->freq;\
+            for (j = 0; j < tv_term_freq; j++) {\
+                int pos = tv_term->positions[j];\
+                matchv_add(mv, pos, pos);\
+            }\
+        }\
+    }\
+}\
+
+static MatchVector *trq_get_matchv_i(Query *self, MatchVector *mv,
+                                     TermVector *tv)
+{
+    Range *range = RQ(((ConstantScoreQuery *)self)->original)->range;
+    if (tv->field == range->field) {
+        double lnum = 0.0, unum = 0.0;
+        int len = 0;
+        const char *lt = range->lower_term;
+        const char *ut = range->upper_term;
+        if ((!lt
+             || (sscanf(lt,"%lg%n",&lnum,&len) && (int)strlen(lt) == len))
+            &&
+            (!ut
+             || (sscanf(ut,"%lg%n",&unum,&len) && (int)strlen(ut) == len)))
+        {
+            TypedRangeCheck check = TRC_NONE;
+            int i = 0, j = 0;
+
+            if (lt) {
+               check = range->include_lower ? TRC_GE : TRC_GT;
+            }
+            if (ut) {
+               check = (TypedRangeCheck)(check | (range->include_upper
+                                                  ? TRC_LE
+                                                  : TRC_LT));
+            }
+
+            switch(check) {
+                case TRC_LE:
+                    SET_TERMS(num <= unum);
+                    break;
+                case TRC_LT:
+                    SET_TERMS(num <  unum);
+                    break;
+                case TRC_GE:
+                    SET_TERMS(num >= lnum);
+                    break;
+                case TRC_GE_LE:
+                    SET_TERMS(num >= lnum && num <= unum);
+                    break;
+                case TRC_GE_LT:
+                    SET_TERMS(num >= lnum && num <  unum);
+                    break;
+                case TRC_GT:
+                    SET_TERMS(num >  lnum);
+                    break;
+                case TRC_GT_LE:
+                    SET_TERMS(num >  lnum && num <= unum);
+                    break;
+                case TRC_GT_LT:
+                    SET_TERMS(num >  lnum && num <  unum);
+                    break;
+                case TRC_NONE:
+                    /* should never happen. Error should have been raised */
+                    assert(false);
+            }
+
+        }
+        else {
+            return rq_get_matchv_i(self, mv, tv);
+        }
+    }
+    return mv;
+}
+
+static Query *trq_rewrite(Query *self, IndexReader *ir)
+{
+    Query *csq;
+    Range *r = RQ(self)->range;
+    Filter *filter = trfilt_new(r->field, r->lower_term, r->upper_term,
+                                r->include_lower, r->include_upper);
+    (void)ir;
+    csq = csq_new_nr(filter);
+    ((ConstantScoreQuery *)csq)->original = self;
+    csq->get_matchv_i = &trq_get_matchv_i;
+    return (Query *)csq;
+}
+
+Query *trq_new_less(Symbol field, const char *upper_term,
+                    bool include_upper)
+{
+    return trq_new(field, NULL, upper_term, false, include_upper);
+}
+
+Query *trq_new_more(Symbol field, const char *lower_term,
+                    bool include_lower)
+{
+    return trq_new(field, lower_term, NULL, include_lower, false);
+}
+
+Query *trq_new(Symbol field, const char *lower_term,
+               const char *upper_term, bool include_lower, bool include_upper)
+{
+    Query *self;
+    Range *range            = trange_new(field, lower_term, upper_term,
+                                         include_lower, include_upper); 
+    self                    = q_new(RangeQuery);
+    RQ(self)->range         = range;
+
+    self->type              = TYPED_RANGE_QUERY;
+    self->rewrite           = &trq_rewrite;
+    self->to_s              = &rq_to_s;
+    self->hash              = &rq_hash;
+    self->eq                = &rq_eq;
+    self->destroy_i         = &rq_destroy;
+    self->create_weight_i   = &q_create_weight_unsup;
+    return self;
+}