ferret 0.9.6 → 0.10.0

data/ext/q_fuzzy.c

@@ -1,5 +1,6 @@
 #include <string.h>
 #include "search.h"
+#include "helper.h"
 
 /****************************************************************************
  *
@@ -10,101 +11,106 @@
  *
  ****************************************************************************/
 
-
-int fuzq_calculate_max_distance(FuzzyQuery *fuzq, int m) 
+static inline int fuzq_calculate_max_distance(FuzzyQuery *fuzq, int m) 
 {
-  return (int)((1.0 - fuzq->min_sim) * (MIN(fuzq->text_len, m) + fuzq->pre_len));
+    return (int)((1.0 - fuzq->min_sim) * (MIN(fuzq->text_len, m) + fuzq->pre_len));
 }
 
-void fuzq_initialize_max_distances(FuzzyQuery *fuzq)
+static void fuzq_initialize_max_distances(FuzzyQuery *fuzq)
 {
-  int i;
-  for (i = 0; i < TYPICAL_LONGEST_WORD; i++) {
-    fuzq->max_distances[i] = fuzq_calculate_max_distance(fuzq, i);
-  }
+    int i;
+    for (i = 0; i < TYPICAL_LONGEST_WORD; i++) {
+        fuzq->max_distances[i] = fuzq_calculate_max_distance(fuzq, i);
+    }
 }
 
-int fuzq_get_max_distance(FuzzyQuery *fuzq, int m)
+static inline int fuzq_get_max_distance(FuzzyQuery *fuzq, int m)
 {
-  return (m < TYPICAL_LONGEST_WORD) ? fuzq->max_distances[m]
-                                    : fuzq_calculate_max_distance(fuzq, m);
+    return (m < TYPICAL_LONGEST_WORD) ? fuzq->max_distances[m]
+        : fuzq_calculate_max_distance(fuzq, m);
 }
 
-float fuzq_score(FuzzyQuery *fuzq, char *target)
+/**
+ * The following algorithm is taken from Bob Carpenter's FuzzyTermEnum
+ * implentation here;
+ *
+ * http://mail-archives.apache.org/mod_mbox/lucene-java-dev/200606.mbox/%3c448F0E8C.3050901@alias-i.com%3e
+ */
+float fuzq_score(FuzzyQuery *fuzq, const char *target)
 {
-  int i, j;
-  int max_distance;
-  int m = (int)strlen(target);
-  int n = fuzq->text_len;
-  int *d = fuzq->da;
-  char *text = fuzq->text;
-  if (n == 0)  {
-    /* we don't have anything to compare.  That means if we just add
-     * the letters for m we get the new word */
-    return fuzq->pre_len == 0 ? 0.0f : 1.0f - ((float) m / fuzq->pre_len);
-  }
-  if (m == 0) {
-    return fuzq->pre_len == 0 ? 0.0f : 1.0f - ((float) n / fuzq->pre_len);
-  }
-
-  max_distance = fuzq_get_max_distance(fuzq, m);
-
-  //printf("n%dm%dmd%ddiff%d<%s><%s>\n", n, m, max_distance, m-n, fuzq->text, target);
-  if (max_distance < ((m > n) ? (m-n) : (n-m))) { /* abs */
-    /* Just adding the characters of m to n or vice-versa results in too many
-     * edits for example "pre" length is 3 and "prefixes" length is 8. We can
-     * see that given this optimal circumstance, the edit distance cannot be
-     * less than 5 which is 8-3 or more precisesly Math.abs(3-8). If our
-     * maximum edit distance is 4, then we can discard this word without
-     * looking at it. */
-    return 0.0f;
-  }
-
-  /* Let's make sure we have enough room in our array to do the distance
-   * calculations. */
-  if (((m+1) * (n+1)) >= fuzq->da_capa) {
-    fuzq->da_capa = ((m+1) * (n+1)) * 2;
-    REALLOC_N(fuzq->da, int, fuzq->da_capa);
-    d = fuzq->da;
-  }
-
-  /* init matrix d */
-  for (i = 0; i <= n; i++) d[i + n * 0] = i;
-  for (j = 0; j <= m; j++) d[0 + n * j] = j;
-  
-  /* start computing edit distance */
-  for (i = 1; i <= n; i++) {
-    int best_pos_ed_dist = m;
-    char s_i = text[i - 1];
-    for (j = 1; j <= m; j++) {
-      if (s_i != target[j-1]) {
-        d[i + n*j] = min3(d[i-1 + n*j], d[i + n*(j-1)], d[i-1 + n*(j-1)])+1;
-      } else {
-        d[i + n*j] = min3(d[i-1 + n*j]+1, d[i + n*(j-1)]+1, d[i-1 + n*(j-1)]);
-      }
-      best_pos_ed_dist = min2(best_pos_ed_dist, d[i + n*j]);
+    const int m = (int)strlen(target);
+    const int n = fuzq->text_len;
+
+    if (n == 0)  {
+        /* we don't have anything to compare.  That means if we just add
+         * the letters for m we get the new word */
+        return fuzq->pre_len == 0 ? 0.0f : 1.0f - ((float) m / fuzq->pre_len);
+    }
+    else if (m == 0) {
+        return fuzq->pre_len == 0 ? 0.0f : 1.0f - ((float) n / fuzq->pre_len);
     }
-    //printf("(bped = %d, i = %d, md = %d)", best_pos_ed_dist, i, max_distance);
-
-    /* After calculating row i, the best possible edit distance can be found
-     * by found by finding the smallest value in a given column.  If the
-     * best_pos_ed_dist is greater than the max distance, abort.
-     */
-
-    if ((i > max_distance) && (best_pos_ed_dist > max_distance)) {
-      /* equal is okay, but not greater
-       * the closest the target can be to the text is just too far away.
-       * this target is leaving the party early. */
-      return 0.0f;
+    else {
+        int i, j, prune;
+        int *d_curr, *d_prev;
+        const char *text = fuzq->text;
+        const int max_distance = fuzq_get_max_distance(fuzq, m);
+
+        /*
+         printf("n%dm%dmd%ddiff%d<%s><%s>\n", n, m, max_distance, m-n,
+               fuzq->text, target);
+         */
+        if (max_distance < ((m > n) ? (m-n) : (n-m))) { /* abs */
+            /* Just adding the characters of m to n or vice-versa results in
+             * too many edits for example "pre" length is 3 and "prefixes"
+             * length is 8. We can see that given this optimal circumstance,
+             * the edit distance cannot be less than 5 which is 8-3 or more
+             * precisesly Math.abs(3-8). If our maximum edit distance is 4,
+             * then we can discard this word without looking at it. */
+            return 0.0f;
+        }
+
+        d_curr = fuzq->da;
+        d_prev = d_curr + n + 1;
+
+        /* init array */
+        for (j = 0; j <= n; j++) {
+            d_curr[j] = j;
+        }
+
+        /* start computing edit distance */
+        for (i = 0; i < m;) {
+           char s_i = target[i];
+           /* swap d_current into d_prev */
+           int *d_tmp = d_prev;
+           d_prev = d_curr;
+           d_curr = d_tmp;
+           prune = (d_curr[0] = ++i) > max_distance;
+
+           for (j = 0; j < n; j++) {
+               d_curr[j + 1] = (s_i == text[j])
+                   ? min3(d_prev[j + 1] + 1, d_curr[j] + 1, d_prev[j])
+                   : min3(d_prev[j + 1], d_curr[j], d_prev[j]) + 1;
+               if (prune && d_curr[j + 1] <= max_distance) {
+                   prune = false;
+               }
+           }
+           if (prune) {
+               return 0.0f;
+           }
+        }
+
+        /*
+        printf("<%f, d_curr[n] = %d min_len = %d>",
+               1.0f - ((float)d_curr[m] / (float) (fuzq->pre_len + min2(n, m))),
+               d_curr[m], fuzq->pre_len + min2(n, m));
+               */
+
+        /* this will return less than 0.0 when the edit distance is greater
+         * than the number of characters in the shorter word.  but this was
+         * the formula that was previously used in FuzzyTermEnum, so it has
+         * not been changed (even though min_sim must be greater than 0.0) */
+        return 1.0f - ((float)d_curr[n] / (float) (fuzq->pre_len + min2(n, m)));
     }
-  }
-  //printf("<%f, d[n + m*m] = %d min_len = %d>", 1.0f - ((float)d[n + m*m] / (float) (fuzq->pre_len + min(n, m))), d[n + m*m], fuzq->pre_len + min(n, m));
-
-  /* this will return less than 0.0 when the edit distance is greater than the
-   * number of characters in the shorter word.  but this was the formula that
-   * was previously used in FuzzyTermEnum, so it has not been changed (even
-   * though min_sim must be greater than 0.0) */
-  return 1.0f - ((float)d[n + n*m] / (float) (fuzq->pre_len + min2(n, m)));
 }
 
 /****************************************************************************
@@ -113,192 +119,150 @@ float fuzq_score(FuzzyQuery *fuzq, char *target)
  *
  ****************************************************************************/
 
-char *fuzq_to_s(Query *self, char *field) 
-{
-  char *buffer, *bptr;
-  FuzzyQuery *fuzq = (FuzzyQuery *)self->data;
-  Term *term = fuzq->term;
-  int tlen = (int)strlen(term->text);
-  int flen = (int)strlen(term->field);
-  bptr = buffer = ALLOC_N(char, tlen + flen + 35);
-
-  if (strcmp(term->field, field) != 0) {
-    sprintf(bptr, "%s:", term->field);
-    bptr += strlen(term->field) + 1;
-  }
-  sprintf(bptr, "%s~", term->text);
-  bptr += strlen(bptr);
-  if (fuzq->min_sim != 0.5) {
-    dbl_to_s(bptr, fuzq->min_sim);
-    bptr += strlen(bptr);
-  }
-  if (self->boost != 1.0) {
-    *bptr = '^';
-    dbl_to_s(++bptr, self->boost);
-  }
-  return buffer;
-}
+#define FzQ(query) ((FuzzyQuery *)(query))
 
-typedef struct ScoredTerm {
-  Term *term;
-  float score;
-} ScoredTerm;
-
-bool scored_term_less_than(void *p1, void *p2)
+static char *fuzq_to_s(Query *self, const char *curr_field) 
 {
-  ScoredTerm *st1 = (ScoredTerm *)p1;
-  ScoredTerm *st2 = (ScoredTerm *)p2;
+    char *buffer, *bptr;
+    char *term = FzQ(self)->term;
+    char *field = FzQ(self)->field;
+    int tlen = (int)strlen(term);
+    int flen = (int)strlen(field);
+    bptr = buffer = ALLOC_N(char, tlen + flen + 70);
+
+    if (strcmp(curr_field, field) != 0) {
+        sprintf(bptr, "%s:", field);
+        bptr += flen + 1;
+    }
 
-  if (st1->score == st2->score)
-    return (strcmp(st1->term->text, st2->term->text) < 0);
+    sprintf(bptr, "%s~", term);
+    bptr += tlen + 1;
+    if (FzQ(self)->min_sim != 0.5) {
+        dbl_to_s(bptr, FzQ(self)->min_sim);
+        bptr += strlen(bptr);
+    }
 
-  return (st1->score < st2->score);
-}
+    if (self->boost != 1.0) {
+        *bptr = '^';
+        dbl_to_s(++bptr, self->boost);
+    }
 
-void scored_term_destroy(ScoredTerm *self)
-{
-  term_destroy(self->term);
-  free(self);
+    return buffer;
 }
 
-ScoredTerm *scored_term_create(Term *term, float score)
+static Query *fuzq_rewrite(Query *self, IndexReader *ir)
 {
-  ScoredTerm *self = ALLOC(ScoredTerm);
-  self->term = term;
-  self->score = score;
-  return self;
-}
+    Query *q;
+    FuzzyQuery *fuzq = FzQ(self);
 
-Query *fuzq_rewrite(Query *self, IndexReader *ir)
-{
-  Query *q;
-  Query *tq;
-  FuzzyQuery *fuzq = (FuzzyQuery *)self->data;
-
-  Term *term = fuzq->term;
-  char *text = term->text;
-  char *field = term->field;
-  Term prefix_term;
-  prefix_term.field = field;
-  if (fuzq->pre_len >= (int)strlen(text)) {
-    q = tq_create(term_clone(term));
-  } else {
-    PriorityQueue *term_pq;
-    TermEnum *te;
-    Term prefix_term;
-    char *prefix = NULL;
-    int pre_len = fuzq->pre_len;
-    ScoredTerm *scored_term;
-
-    q = bq_create(true);
-
-    term_pq = pq_create(((BooleanQuery *)q->data)->max_clause_cnt,
-        &scored_term_less_than);
-    term_pq->free_elem = (free_ft)&scored_term_destroy;
-
-    prefix_term.field = field;
-    prefix_term.text = (char *)EMPTY_STRING;
-    if (pre_len >= 0) {
-      prefix = ALLOC_N(char, pre_len + 1);
-      strncpy(prefix, text, pre_len);
-      prefix_term.text = prefix;
-      prefix_term.text[pre_len] = '\0';
+    const char *term = fuzq->term;
+    const char *field = fuzq->field;
+    const int field_num = fis_get_field_num(ir->fis, field);
+
+    if (field_num < 0) {
+        q = bq_new(true);
     }
-    te = ir->terms_from(ir, &prefix_term);
-
-    fuzq->scale_factor = (float)(1.0 / (1.0 - fuzq->min_sim));
-    fuzq->text = fuzq->term->text + pre_len;
-    fuzq->text_len = (int)strlen(fuzq->text);
-    fuzq_initialize_max_distances(fuzq);
-
-    if (te) {
-      TermBuffer *tb = te->tb_curr;
-      float score = 0.0, min_score = fuzq->min_sim;
-
-      TRY
-        do { 
-          if (strcmp(tb->field, field) != 0 ||
-              (prefix && strncmp(tb->text, prefix, pre_len) != 0))
-            break;
-
-          score = fuzq_score(fuzq, tb->text + pre_len);
-          //printf("%s:%s:%f\n", tb->text, fuzq->text, score);
-
-          if (score > min_score) {
-            pq_insert(term_pq, scored_term_create(tb_get_term(tb), score));
-            if (pq_full(term_pq))
-              min_score = ((ScoredTerm *)pq_top(term_pq))->score;
-          }
-        } while ((tb = te->next(te)) != NULL);
-      XFINALLY
-        te->close(te);
-      XENDTRY
+    else if (fuzq->pre_len >= (int)strlen(term)) {
+        q = tq_new(field, term);
     }
-    free(prefix);
-
-    while ((scored_term = pq_pop(term_pq)) != NULL) {
-      tq = tq_create(scored_term->term); /* found match */
-      tq->boost = self->boost;           /* set boost */
-      bq_add_query(q, tq, BC_SHOULD);    /* add query */
-      free(scored_term); /* no need to free the term as it's in the query */
+    else {
+        TermEnum *te;
+        char *prefix = NULL;
+        int pre_len = fuzq->pre_len;
+
+        q = multi_tq_new_conf(fuzq->field, MTQMaxTerms(self), fuzq->min_sim);
+
+        if (pre_len > 0) {
+            prefix = ALLOC_N(char, pre_len + 1);
+            strncpy(prefix, term, pre_len);
+            prefix[pre_len] = '\0';
+            te = ir->terms_from(ir, field_num, prefix);
+        }
+        else {
+            te = ir->terms(ir, field_num);
+        }
+
+        fuzq->scale_factor = (float)(1.0 / (1.0 - fuzq->min_sim));
+        fuzq->text = term + pre_len;
+        fuzq->text_len = (int)strlen(fuzq->text);
+        fuzq->da = REALLOC_N(fuzq->da, int, fuzq->text_len * 2 + 2);
+        fuzq_initialize_max_distances(fuzq);
+
+        if (te) {
+            const char *curr_term = te->curr_term;
+            const char *curr_suffix = curr_term + pre_len;
+            float score = 0.0;
+
+
+            do { 
+                if ((prefix && strncmp(curr_term, prefix, pre_len) != 0)) {
+                    break;
+                }
+
+                score = fuzq_score(fuzq, curr_suffix);
+                /*
+                 printf("%s:%s:%f < %f\n", curr_term, term, score, min_score);
+                 */
+                multi_tq_add_term_boost(q, curr_term, score);
+
+            } while (te->next(te) != NULL);
+
+            te->close(te);
+        }
+        free(prefix);
     }
-    pq_destroy(term_pq);
-  }
 
-  return q;
+    return q;
 }
 
-void fuzq_destroy(Query *self)
+static void fuzq_destroy(Query *self)
 {
-  FuzzyQuery *fuzq = (FuzzyQuery *)self->data;
-  if (self->destroy_all) term_destroy((Term *)fuzq->term);
-  free(fuzq->da);
-  free(fuzq);
-  q_destroy_i(self);
+    free(FzQ(self)->term);
+    free(FzQ(self)->field);
+    free(FzQ(self)->da);
+    q_destroy_i(self);
 }
 
-static uint fuzq_hash(Query *self)
+static ulong fuzq_hash(Query *self)
 {
-  FuzzyQuery *fq = (FuzzyQuery *)self->data;
-  return term_hash(fq->term) ^ *((int *)&fq->min_sim) ^ fq->pre_len;
+    return str_hash(FzQ(self)->term) ^ str_hash(FzQ(self)->field)
+        ^ float2int(FzQ(self)->min_sim) ^ FzQ(self)->pre_len;
 }
 
 static int fuzq_eq(Query *self, Query *o)
 {
-  FuzzyQuery *fq1 = (FuzzyQuery *)self->data;
-  FuzzyQuery *fq2 = (FuzzyQuery *)o->data;
-  return term_eq(fq1->term, fq2->term) &&
-    (fq1->pre_len == fq2->pre_len) && 
-    (fq1->min_sim == fq2->min_sim);
+    FuzzyQuery *fq1 = FzQ(self);
+    FuzzyQuery *fq2 = FzQ(o);
+
+    return (strcmp(fq1->term, fq2->term) == 0)
+        && (strcmp(fq1->field, fq2->field) == 0)
+        && (fq1->pre_len == fq2->pre_len)
+        && (fq1->min_sim == fq2->min_sim);
 }
 
-Query *fuzq_create(Term *term)
+Query *fuzq_new_conf(const char *field, const char *term,
+                     float min_sim, int pre_len, int max_terms)
 {
-  Query *self = q_create();
-
-  FuzzyQuery *fq = ALLOC(FuzzyQuery);
-  ZEROSET(fq, FuzzyQuery, 1);
-  fq->term = term;
-  fq->pre_len = DEF_PRE_LEN;
-  fq->min_sim = DEF_MIN_SIM;
-  self->data = fq;
-
-  self->type = FUZZY_QUERY;
-  self->to_s = &fuzq_to_s;
-  self->hash = &fuzq_hash;
-  self->eq = &fuzq_eq;
-  self->rewrite = &fuzq_rewrite;
-  self->destroy_i = &fuzq_destroy;
-  self->create_weight_i = &q_create_weight_unsup;
-
-  return self;
+    Query *self = q_new(FuzzyQuery);
+
+    FzQ(self)->field      = estrdup(field);
+    FzQ(self)->term       = estrdup(term);
+    FzQ(self)->pre_len    = pre_len ? pre_len : DEF_PRE_LEN;
+    FzQ(self)->min_sim    = min_sim ? min_sim : DEF_MIN_SIM;
+    MTQMaxTerms(self)     = max_terms ? max_terms : DEF_MAX_TERMS;
+
+    self->type            = FUZZY_QUERY;
+    self->to_s            = &fuzq_to_s;
+    self->hash            = &fuzq_hash;
+    self->eq              = &fuzq_eq;
+    self->rewrite         = &fuzq_rewrite;
+    self->destroy_i       = &fuzq_destroy;
+    self->create_weight_i = &q_create_weight_unsup;
+
+    return self;
 }
 
-Query *fuzq_create_mp(Term *term, float min_sim, int pre_len)
+Query *fuzq_new(const char *field, const char *term)
 {
-  Query *self = fuzq_create(term);
-  FuzzyQuery *fuzq = (FuzzyQuery *)self->data;
-  if (pre_len) fuzq->pre_len = pre_len;
-  if (min_sim) fuzq->min_sim = min_sim;
-  return self;
+    return fuzq_new_conf(term, field, 0.0f, 0, 0);
 }