np-ferret 0.11.6

data/ext/q_const_score.c

@@ -0,0 +1,165 @@
+#include "search.h"
+#include <string.h>
+
+/***************************************************************************
+ *
+ * ConstantScoreScorer
+ *
+ ***************************************************************************/
+
+#define CScQ(query) ((ConstantScoreQuery *)(query))
+#define CScSc(scorer) ((ConstantScoreScorer *)(scorer))
+
+typedef struct ConstantScoreScorer
+{
+    Scorer      super;
+    BitVector  *bv;
+    float       score;
+} ConstantScoreScorer;
+
+static float cssc_score(Scorer *self)
+{
+    return CScSc(self)->score;
+}
+
+static bool cssc_next(Scorer *self)
+{
+    return ((self->doc = bv_scan_next(CScSc(self)->bv)) >= 0);
+}
+
+static bool cssc_skip_to(Scorer *self, int doc_num)
+{
+    return ((self->doc = bv_scan_next_from(CScSc(self)->bv, doc_num)) >= 0);
+}
+
+static Explanation *cssc_explain(Scorer *self, int doc_num)
+{
+    (void)self; (void)doc_num;
+    return expl_new(1.0, "ConstantScoreScorer");
+}
+
+static Scorer *cssc_new(Weight *weight, IndexReader *ir)
+{
+    Scorer *self    = scorer_new(ConstantScoreScorer, weight->similarity);
+    Filter *filter  = CScQ(weight->query)->filter;
+
+    CScSc(self)->score  = weight->value;
+    CScSc(self)->bv     = filt_get_bv(filter, ir);
+    
+    // JRG - Make sure we reset the filter bitvector, since we're using the
+    // cached one.  Otherwise the second call wouldn't return any results!
+    bv_scan_reset(CScSc(self)->bv);
+
+    self->score     = &cssc_score;
+    self->next      = &cssc_next;
+    self->skip_to   = &cssc_skip_to;
+    self->explain   = &cssc_explain;
+    self->destroy   = &scorer_destroy_i;
+    return self;
+}
+
+/***************************************************************************
+ *
+ * ConstantScoreWeight
+ *
+ ***************************************************************************/
+
+static char *csw_to_s(Weight *self)
+{
+    return strfmt("ConstantScoreWeight(%f)", self->value);
+}
+
+static Explanation *csw_explain(Weight *self, IndexReader *ir, int doc_num)
+{
+    Filter *filter = CScQ(self->query)->filter;
+    Explanation *expl;
+    char *filter_str = filter->to_s(filter);
+    BitVector *bv = filt_get_bv(filter, ir);
+
+    if (bv_get(bv, doc_num)) {
+        expl = expl_new(self->value,
+                        "ConstantScoreQuery(%s), product of:", filter_str);
+        expl_add_detail(expl, expl_new(self->query->boost, "boost"));
+        expl_add_detail(expl, expl_new(self->qnorm, "query_norm"));
+    }
+    else {
+        expl = expl_new(self->value,
+                        "ConstantScoreQuery(%s), does not match id %d",
+                        filter_str, doc_num);
+    }
+    free(filter_str);
+    return expl;
+}
+
+static Weight *csw_new(Query *query, Searcher *searcher)
+{
+    Weight *self        = w_new(Weight, query);
+
+    self->scorer        = &cssc_new;
+    self->explain       = &csw_explain;
+    self->to_s          = &csw_to_s;
+
+    self->similarity    = query->get_similarity(query, searcher);
+    self->idf           = 1.0;
+
+    return self;
+}
+
+/***************************************************************************
+ *
+ * ConstantScoreQuery
+ *
+ ***************************************************************************/
+
+static char *csq_to_s(Query *self, const char *field)
+{
+    Filter *filter = CScQ(self)->filter;
+    char *filter_str = filter->to_s(filter);
+    char *buffer;
+    (void)field;
+    if (self->boost == 1.0) {
+        buffer = strfmt("ConstantScore(%s)", filter_str);
+    }
+    else {
+        buffer = strfmt("ConstantScore(%s)^%f", filter_str, self->boost);
+    }
+    free(filter_str);
+    return buffer;;
+}
+
+static void csq_destroy(Query *self)
+{
+    filt_deref(CScQ(self)->filter);
+    q_destroy_i(self);
+}
+
+static unsigned long csq_hash(Query *self)
+{
+    return filt_hash(CScQ(self)->filter);
+}
+
+static int csq_eq(Query *self, Query *o)
+{
+    return filt_eq(CScQ(self)->filter, CScQ(o)->filter);
+}
+
+Query *csq_new_nr(Filter *filter)
+{
+    Query *self = q_new(ConstantScoreQuery);
+    CScQ(self)->filter = filter;
+
+    self->type              = CONSTANT_QUERY;
+    self->to_s              = &csq_to_s;
+    self->hash              = &csq_hash;
+    self->eq                = &csq_eq;
+    self->destroy_i         = &csq_destroy;
+    self->create_weight_i   = &csw_new;
+
+    return self;
+}
+
+Query *csq_new(Filter *filter)
+{
+    REF(filter);
+    return csq_new_nr(filter);
+}

data/ext/q_filtered_query.c

@@ -0,0 +1,209 @@
+#include "search.h"
+#include <string.h>
+
+/***************************************************************************
+ *
+ * FilteredQueryScorer
+ *
+ ***************************************************************************/
+
+#define FQSc(scorer) ((FilteredQueryScorer *)(scorer))
+#define FQQ(query) ((FilteredQuery *)(query))
+
+typedef struct FilteredQueryScorer
+{
+    Scorer      super;
+    Scorer     *sub_scorer;
+    BitVector  *bv;
+} FilteredQueryScorer;
+
+float fqsc_score(Scorer *self)
+{
+    Scorer *sub_sc = FQSc(self)->sub_scorer;
+    return sub_sc->score(sub_sc);
+}
+
+bool fqsc_next(Scorer *self)
+{
+    Scorer *sub_sc = FQSc(self)->sub_scorer;
+    BitVector *bv = FQSc(self)->bv;
+    while (sub_sc->next(sub_sc)) {
+        self->doc = sub_sc->doc;
+        if (bv_get(bv, self->doc)) return true;
+    }
+    return false;
+}
+
+bool fqsc_skip_to(Scorer *self, int doc_num)
+{
+    Scorer *sub_sc = FQSc(self)->sub_scorer;
+    BitVector *bv = FQSc(self)->bv;
+    if (sub_sc->skip_to(sub_sc, doc_num)) {
+        self->doc = sub_sc->doc;
+        do {
+            if (bv_get(bv, self->doc)) return true;
+        } while (sub_sc->next(sub_sc));
+    }
+    return false;
+}
+
+Explanation *fqsc_explain(Scorer *self, int doc_num)
+{
+    Scorer *sub_sc = FQSc(self)->sub_scorer;
+    return sub_sc->explain(sub_sc, doc_num);
+}
+
+void fqsc_destroy(Scorer *self)
+{
+    FilteredQueryScorer *fqsc = FQSc(self);
+    fqsc->sub_scorer->destroy(fqsc->sub_scorer);
+    scorer_destroy_i(self);
+}
+
+Scorer *fqsc_new(Scorer *scorer, BitVector *bv, Similarity *sim)
+{
+    Scorer *self            = scorer_new(FilteredQueryScorer, sim);
+
+    FQSc(self)->sub_scorer  = scorer;
+    FQSc(self)->bv          = bv;
+
+    self->score   = &fqsc_score;
+    self->next    = &fqsc_next;
+    self->skip_to = &fqsc_skip_to;
+    self->explain = &fqsc_explain;
+    self->destroy = &fqsc_destroy;
+
+    return self;
+}
+
+/***************************************************************************
+ *
+ * Weight
+ *
+ ***************************************************************************/
+
+#define FQW(weight) ((FilteredQueryWeight *)(weight))
+typedef struct FilteredQueryWeight
+{
+    Weight  super;
+    Weight *sub_weight;
+} FilteredQueryWeight;
+
+static char *fqw_to_s(Weight *self)
+{
+    return strfmt("FilteredQueryWeight(%f)", self->value);
+}
+
+static float fqw_sum_of_squared_weights(Weight *self)
+{
+    Weight *sub_weight = FQW(self)->sub_weight;
+    return sub_weight->sum_of_squared_weights(sub_weight);
+}
+
+static void fqw_normalize(Weight *self, float normalization_factor)
+{
+    Weight *sub_weight = FQW(self)->sub_weight;
+    sub_weight->normalize(sub_weight, normalization_factor);
+}
+
+static float fqw_get_value(Weight *self)
+{
+    Weight *sub_weight = FQW(self)->sub_weight;
+    return sub_weight->get_value(sub_weight);
+}
+
+static Explanation *fqw_explain(Weight *self, IndexReader *ir, int doc_num)
+{
+    Weight *sub_weight = FQW(self)->sub_weight;
+    return sub_weight->explain(sub_weight, ir, doc_num);
+}
+
+static Scorer *fqw_scorer(Weight *self, IndexReader *ir)
+{
+    Weight *sub_weight = FQW(self)->sub_weight;
+    Scorer *scorer = sub_weight->scorer(sub_weight, ir);
+    Filter *filter = FQQ(self->query)->filter;
+
+    return fqsc_new(scorer, filt_get_bv(filter, ir), self->similarity);
+}
+
+static void fqw_destroy(Weight *self)
+{
+    Weight *sub_weight = FQW(self)->sub_weight;
+    sub_weight->destroy(sub_weight);
+    w_destroy(self);
+}
+
+static Weight *fqw_new(Query *query, Weight *sub_weight, Similarity *sim)
+{
+    Weight *self = w_new(FilteredQueryWeight, query);
+
+    FQW(self)->sub_weight           = sub_weight;
+
+    self->get_value                 = &fqw_get_value;
+    self->normalize                 = &fqw_normalize;
+    self->scorer                    = &fqw_scorer;
+    self->explain                   = &fqw_explain;
+    self->to_s                      = &fqw_to_s;
+    self->destroy                   = &fqw_destroy;
+    self->sum_of_squared_weights    = &fqw_sum_of_squared_weights;
+
+    self->similarity                = sim;
+    self->idf                       = 1.0;
+    self->value                     = sub_weight->value;
+
+    return self;
+}
+
+/***************************************************************************
+ *
+ * FilteredQuery
+ *
+ ***************************************************************************/
+
+static char *fq_to_s(Query *self, const char *field)
+{
+    FilteredQuery *fq = FQQ(self);
+    char *filter_str = fq->filter->to_s(fq->filter);
+    char *query_str = fq->query->to_s(fq->query, field);
+    char *buffer;
+    if (self->boost == 1.0) {
+        buffer = strfmt("FilteredQuery(query:%s, filter:%s)",
+                        query_str, filter_str);
+    } else {
+        buffer = strfmt("FilteredQuery(query:%s, filter:%s)^%f",
+                        query_str, filter_str, self->boost);
+    }
+    free(filter_str);
+    free(query_str);
+    return buffer;;
+}
+
+void fq_destroy(Query *self)
+{
+    filt_deref(FQQ(self)->filter);
+    q_deref(FQQ(self)->query);
+    q_destroy_i(self);
+}
+
+Weight *fq_new_weight(Query *self, Searcher *searcher)
+{
+    Query *sub_query = FQQ(self)->query;
+    return fqw_new(self, q_weight(sub_query, searcher),
+                      searcher->similarity);
+}
+
+Query *fq_new(Query *query, Filter *filter)
+{
+    Query *self = q_new(FilteredQuery);
+
+    FQQ(self)->query        = query;
+    FQQ(self)->filter       = filter;
+
+    self->type              = FILTERED_QUERY;
+    self->to_s              = &fq_to_s;
+    self->destroy_i         = &fq_destroy;
+    self->create_weight_i   = &fq_new_weight;
+
+    return self;
+}

data/ext/q_fuzzy.c

@@ -0,0 +1,335 @@
+#include <string.h>
+#include "search.h"
+#include "helper.h"
+
+/****************************************************************************
+ *
+ * FuzzyStuff
+ *
+ * The main method here is the fuzq_score method which scores a term against
+ * another term. The other methods all act in support.
+ *
+ ****************************************************************************/
+
+static INLINE int fuzq_calculate_max_distance(FuzzyQuery *fuzq, int m) 
+{
+    return (int)((1.0 - fuzq->min_sim) * (MIN(fuzq->text_len, m) + fuzq->pre_len));
+}
+
+static void fuzq_initialize_max_distances(FuzzyQuery *fuzq)
+{
+    int i;
+    for (i = 0; i < TYPICAL_LONGEST_WORD; i++) {
+        fuzq->max_distances[i] = fuzq_calculate_max_distance(fuzq, i);
+    }
+}
+
+static INLINE int fuzq_get_max_distance(FuzzyQuery *fuzq, int m)
+{
+    return (m < TYPICAL_LONGEST_WORD) ? fuzq->max_distances[m]
+        : fuzq_calculate_max_distance(fuzq, m);
+}
+
+/**
+ * The following algorithm is taken from Bob Carpenter's FuzzyTermEnum
+ * implentation here;
+ *
+ * http://mail-archives.apache.org/mod_mbox/lucene-java-dev/200606.mbox/%3c448F0E8C.3050901@alias-i.com%3e
+ */
+float fuzq_score(FuzzyQuery *fuzq, const char *target)
+{
+    const int m = (int)strlen(target);
+    const int n = fuzq->text_len;
+
+    if (n == 0)  {
+        /* we don't have anything to compare.  That means if we just add
+         * the letters for m we get the new word */
+        return fuzq->pre_len == 0 ? 0.0f : 1.0f - ((float) m / fuzq->pre_len);
+    }
+    else if (m == 0) {
+        return fuzq->pre_len == 0 ? 0.0f : 1.0f - ((float) n / fuzq->pre_len);
+    }
+    else {
+        int i, j, prune;
+        int *d_curr, *d_prev;
+        const char *text = fuzq->text;
+        const int max_distance = fuzq_get_max_distance(fuzq, m);
+
+        /*
+         printf("n%dm%dmd%ddiff%d<%s><%s>\n", n, m, max_distance, m-n,
+               fuzq->text, target);
+         */
+         
+        if (max_distance < ((m > n) ? (m-n) : (n-m))) { /* abs */
+            /* Just adding the characters of m to n or vice-versa results in
+             * too many edits for example "pre" length is 3 and "prefixes"
+             * length is 8. We can see that given this optimal circumstance,
+             * the edit distance cannot be less than 5 which is 8-3 or more
+             * precisesly Math.abs(3-8). If our maximum edit distance is 4,
+             * then we can discard this word without looking at it. */
+            return 0.0f;
+        }
+
+        d_curr = fuzq->da;
+        d_prev = d_curr + n + 1;
+
+        /* init array */
+        for (j = 0; j <= n; j++) {
+            d_curr[j] = j;
+        }
+
+        /* start computing edit distance */
+        for (i = 0; i < m;) {
+           char s_i = target[i];
+           /* swap d_current into d_prev */
+           int *d_tmp = d_prev;
+           d_prev = d_curr;
+           d_curr = d_tmp;
+           prune = (d_curr[0] = ++i) > max_distance;
+
+           for (j = 0; j < n; j++) {
+               d_curr[j + 1] = (s_i == text[j])
+                   ? min3(d_prev[j + 1] + 1, d_curr[j] + 1, d_prev[j])
+                   : min3(d_prev[j + 1], d_curr[j], d_prev[j]) + 1;
+               if (prune && d_curr[j + 1] <= max_distance) {
+                   prune = false;
+               }
+           }
+           if (prune) {
+               return 0.0f;
+           }
+        }
+
+        /*
+        printf("<%f, d_curr[n] = %d min_len = %d>",
+               1.0f - ((float)d_curr[m] / (float) (fuzq->pre_len + min2(n, m))),
+               d_curr[m], fuzq->pre_len + min2(n, m));
+               */
+
+        /* this will return less than 0.0 when the edit distance is greater
+         * than the number of characters in the shorter word.  but this was
+         * the formula that was previously used in FuzzyTermEnum, so it has
+         * not been changed (even though min_sim must be greater than 0.0) */
+        return 1.0f - ((float)d_curr[n] / (float) (fuzq->pre_len + min2(n, m)));
+    }
+}
+
+/****************************************************************************
+ *
+ * FuzzyQuery
+ *
+ ****************************************************************************/
+
+#define FzQ(query) ((FuzzyQuery *)(query))
+
+static char *fuzq_to_s(Query *self, const char *curr_field) 
+{
+    char *buffer, *bptr;
+    char *term = FzQ(self)->term;
+    char *field = FzQ(self)->field;
+    int tlen = (int)strlen(term);
+    int flen = (int)strlen(field);
+    bptr = buffer = ALLOC_N(char, tlen + flen + 70);
+
+    if (strcmp(curr_field, field) != 0) {
+        sprintf(bptr, "%s:", field);
+        bptr += flen + 1;
+    }
+
+    sprintf(bptr, "%s~", term);
+    bptr += tlen + 1;
+    // JRG -- Forcing it to put out the number, even if it's 0.5, because the query parser
+    // can't re-interpret it without the number.
+    // if (FzQ(self)->min_sim != 0.5) {
+        dbl_to_s(bptr, FzQ(self)->min_sim);
+        bptr += strlen(bptr);
+    //}
+
+    if (self->boost != 1.0) {
+        *bptr = '^';
+        dbl_to_s(++bptr, self->boost);
+    }
+
+    return buffer;
+}
+
+static Query *fuzq_rewrite(Query *self, IndexReader *ir)
+{
+    Query *q;
+    FuzzyQuery *fuzq = FzQ(self);
+
+    const char *term = fuzq->term;
+    const char *field = fuzq->field;
+    const int field_num = fis_get_field_num(ir->fis, field);
+
+    if (field_num < 0) {
+        q = bq_new(true);
+    }
+    else if (fuzq->pre_len >= (int)strlen(term)) {
+        q = tq_new(field, term);
+    }
+    else {
+        TermEnum *te;
+        char *prefix = NULL;
+        int pre_len = fuzq->pre_len;
+
+        q = multi_tq_new_conf(fuzq->field, MTQMaxTerms(self), fuzq->min_sim);
+
+        if (pre_len > 0) {
+            prefix = ALLOC_N(char, pre_len + 1);
+            strncpy(prefix, term, pre_len);
+            prefix[pre_len] = '\0';
+            te = ir->terms_from(ir, field_num, prefix);
+        }
+        else {
+            te = ir->terms(ir, field_num);
+        }
+
+        fuzq->scale_factor = (float)(1.0 / (1.0 - fuzq->min_sim));
+        fuzq->text = term + pre_len;
+        fuzq->text_len = (int)strlen(fuzq->text);
+        fuzq->da = REALLOC_N(fuzq->da, int, fuzq->text_len * 2 + 2);
+        fuzq_initialize_max_distances(fuzq);
+
+        if (te) {
+            const char *curr_term = te->curr_term;
+            const char *curr_suffix = curr_term + pre_len;
+            float score = 0.0;
+
+<<<<<<< HEAD
+=======
+            /* JRG -- Keeping track of terms added, so we can break out of the loop once
+            we've hit the max_terms limit.  Otherwise this can be really really slow. For
+            fuzzy, we'll actually go up to double the max terms, though, to allow the
+            priority queue to a bit of its job. */
+            // int terms_added = 0;
+            // int max_terms = MTQMaxTerms(self) << 1;
+<<<<<<< HEAD
+>>>>>>> 518a88d... Put fuzzy searches back to running through the whole list of terms, but make a slight optimization that only ones over the min_similarity are added to the priority queue.
+
+=======
+            
+<<<<<<< HEAD
+            /* JRG -- Pre-calculate what the min and max word lengths will be, so we
+            can optimize a bit by not calculating levenshtein distances on them.  The
+            levenshtein algorithm does this a bit internally, but it's more efficient
+            to just do some basic comparisons right off the bat. */
+            int term_len = (int)strlen(term);
+            int min_word_length = pre_len + 1;
+            while (term_len - min_word_length - fuzq_get_max_distance(fuzq, min_word_length) > 0) min_word_length++;
+            int max_word_length = term_len + fuzq_get_max_distance(fuzq, term_len);
+            
+>>>>>>> 48f2e0a... Try to optimize the fuzzy query stuff a bit more.
+=======
+>>>>>>> fda5be3... Ignoring earlier optimizations that didn't help enough.
+            do { 
+                if ((prefix && strncmp(curr_term, prefix, pre_len) != 0)) {
+                    break;
+                }
+<<<<<<< HEAD
+
+                score = fuzq_score(fuzq, curr_suffix);
+<<<<<<< HEAD
+                /*
+                 printf("%s:%s:%f < %f\n", curr_term, term, score, min_score);
+                 */
+                multi_tq_add_term_boost(q, curr_term, score);
+
+=======
+                if (score >= fuzq->min_sim) {
+                  multi_tq_add_term_boost(q, curr_term, score);
+                  // terms_added++;
+=======
+                
+<<<<<<< HEAD
+                if (min_word_length <= te->curr_term_len && te->curr_term_len <= max_word_length) {
+
+                  // Perturb score ever so slightly, so that we don't have problems
+                  // with 0.9999999999 not being >= 1.0, etc.
+                  score = fuzq_score(fuzq, curr_suffix) + 0.00001;
+                  /*
+                   printf("%s:%s:%f < %f\n", curr_term, term, score, min_score);
+                 */
+                  if (score >= fuzq->min_sim) {
+                    multi_tq_add_term_boost(q, curr_term, score);
+                    // printf("  Added %s, length = %d, score = %f\n", curr_term, (int)strlen(curr_term), score);
+                    // terms_added++;
+                  }
+>>>>>>> 48f2e0a... Try to optimize the fuzzy query stuff a bit more.
+=======
+                // Perturb score ever so slightly, so that we don't have problems
+                // with 0.9999999999 not being >= 1.0, etc.
+                score = fuzq_score(fuzq, curr_suffix) + 0.00001;
+                /*
+                 printf("%s:%s:%f < %f\n", curr_term, term, score, min_score);
+               */
+                if (score >= fuzq->min_sim) {
+                  multi_tq_add_term_boost(q, curr_term, score);
+                  // printf("  Added %s, length = %d, score = %f\n", curr_term, (int)strlen(curr_term), score);
+                  // terms_added++;
+>>>>>>> fda5be3... Ignoring earlier optimizations that didn't help enough.
+                }
+
+            // } while (te->next(te) != NULL && terms_added < max_terms);
+>>>>>>> 518a88d... Put fuzzy searches back to running through the whole list of terms, but make a slight optimization that only ones over the min_similarity are added to the priority queue.
+            } while (te->next(te) != NULL);
+
+            te->close(te);
+        }
+        free(prefix);
+    }
+
+    return q;
+}
+
+static void fuzq_destroy(Query *self)
+{
+    free(FzQ(self)->term);
+    free(FzQ(self)->field);
+    free(FzQ(self)->da);
+    q_destroy_i(self);
+}
+
+static unsigned long fuzq_hash(Query *self)
+{
+    return str_hash(FzQ(self)->term) ^ str_hash(FzQ(self)->field)
+        ^ float2int(FzQ(self)->min_sim) ^ FzQ(self)->pre_len;
+}
+
+static int fuzq_eq(Query *self, Query *o)
+{
+    FuzzyQuery *fq1 = FzQ(self);
+    FuzzyQuery *fq2 = FzQ(o);
+
+    return (strcmp(fq1->term, fq2->term) == 0)
+        && (strcmp(fq1->field, fq2->field) == 0)
+        && (fq1->pre_len == fq2->pre_len)
+        && (fq1->min_sim == fq2->min_sim);
+}
+
+Query *fuzq_new_conf(const char *field, const char *term,
+                     float min_sim, int pre_len, int max_terms)
+{
+    Query *self = q_new(FuzzyQuery);
+
+    FzQ(self)->field      = estrdup(field);
+    FzQ(self)->term       = estrdup(term);
+    FzQ(self)->pre_len    = pre_len ? pre_len : DEF_PRE_LEN;
+    FzQ(self)->min_sim    = min_sim ? min_sim : DEF_MIN_SIM;
+    MTQMaxTerms(self)     = max_terms ? max_terms : DEF_MAX_TERMS;
+
+    self->type            = FUZZY_QUERY;
+    self->to_s            = &fuzq_to_s;
+    self->hash            = &fuzq_hash;
+    self->eq              = &fuzq_eq;
+    self->rewrite         = &fuzq_rewrite;
+    self->destroy_i       = &fuzq_destroy;
+    self->create_weight_i = &q_create_weight_unsup;
+
+    return self;
+}
+
+Query *fuzq_new(const char *field, const char *term)
+{
+    return fuzq_new_conf(field, term, 0.0f, 0, 0);
+}