ferret 0.3.2 → 0.9.0

data/ext/q_const_score.c

@@ -0,0 +1,154 @@
+#include "search.h"
+#include <string.h>
+
+/***************************************************************************
+ *
+ * Weight
+ *
+ ***************************************************************************/
+
+char *csw_to_s(Weight *self)
+{
+  char dbuf[32];
+  dbl_to_s(dbuf, self->value);
+  return epstrdup("ConstantScoreWeight(%s)", strlen(dbuf), dbuf);
+}
+
+void csw_destroy(void *p)
+{
+  free(p);
+}
+
+Explanation *csw_explain(Weight *self, IndexReader *ir, int doc_num)
+{
+  Filter *filter = (Filter *)self->query->data;
+  Explanation *expl;
+  char *filter_str = filter->to_s(filter);
+  BitVector *bv = filt_get_bv(filter, ir);
+
+  if (bv_get(bv, doc_num)) {
+    expl = expl_create(self->value,
+        epstrdup("ConstantScoreQuery(%s), product of:",
+          strlen(filter_str), filter_str));
+    expl_add_detail(expl, expl_create(self->query->boost, estrdup("boost")));
+    expl_add_detail(expl, expl_create(self->qnorm, estrdup("query_norm")));
+  } else {
+    expl = expl_create(self->value,
+        epstrdup("ConstantScoreQuery(%s), does not match id %d",
+          strlen(filter_str) + 20, filter_str, doc_num));
+  }
+  free(filter_str);
+  return expl;
+}
+
+Weight *csw_create(Query *query, Searcher *searcher)
+{
+  Weight *self = ALLOC(Weight);
+  ZEROSET(self, Weight, 1);
+  self->get_query = &w_get_query;
+  self->get_value = &w_get_value;
+  self->normalize = &w_normalize;
+  self->scorer    = &cssc_create;
+  self->explain   = &csw_explain;
+  self->to_s      = &csw_to_s;
+  self->destroy   = &csw_destroy;
+  self->sum_of_squared_weights = &w_sum_of_squared_weights;
+
+  self->similarity = query->get_similarity(query, searcher);
+  self->idf = 1.0;
+  self->query = query;
+  self->value = 0.0;
+
+  return self;
+}
+
+/***************************************************************************
+ *
+ * ConstantScoreQuery
+ *
+ ***************************************************************************/
+
+char *csq_to_s(Query *self, char *field)
+{
+  Filter *filter = (Filter *)self->data;
+  char *filter_str = filter->to_s(filter);
+  char *buffer;
+  if (self->boost == 1.0) {
+    buffer = epstrdup("ConstantScore(%s)", strlen(filter_str), filter_str);
+  } else {
+    char dbuf[32];
+    dbl_to_s(dbuf, self->boost);
+    buffer = epstrdup("ConstantScore(%s)^%s",
+        strlen(filter_str) + strlen(dbuf), filter_str, dbuf);
+  }
+  free(filter_str);
+  return buffer;;
+}
+
+void csq_destroy(void *p)
+{
+  Query *self = (Query *)p;
+  if (self->destroy_all) {
+    Filter *filter = (Filter *)self->data;
+    filter->destroy(filter);
+  }
+  q_destroy(self);
+}
+
+Query *csq_create(Filter *filter)
+{
+  Query *self = q_create();
+  self->type = CONSTANT_QUERY;
+  self->data = filter;
+  self->create_weight = &csw_create;
+  self->to_s = &csq_to_s;
+  self->destroy = &csq_destroy;
+
+  return self;
+}
+
+/***************************************************************************
+ *
+ * ConstantScoreScorer
+ *
+ ***************************************************************************/
+
+float cssc_score(Scorer *self)
+{
+  return ((ConstantScoreScorer *)self->data)->score;
+}
+
+bool cssc_next(Scorer *self)
+{
+  BitVector *bv = ((ConstantScoreScorer *)self->data)->bv;
+  return ((self->doc = bv_scan_next(bv)) >= 0);
+}
+
+bool cssc_skip_to(Scorer *self, int doc_num)
+{
+  BitVector *bv = ((ConstantScoreScorer *)self->data)->bv;
+  return ((self->doc = bv_scan_next_from(bv, doc_num)) >= 0);
+}
+
+Explanation *cssc_explain(Scorer *self, int doc_num)
+{
+  return expl_create(1.0, estrdup("ConstantScoreScorer"));
+}
+
+Scorer *cssc_create(Weight *weight, IndexReader *ir)
+{
+  Scorer *self = scorer_create(weight->similarity);
+  Filter *filter = (Filter *)weight->query->data;
+  ConstantScoreScorer *cssc = ALLOC(ConstantScoreScorer);
+  ZEROSET(cssc, ConstantScoreScorer, 1);
+  self->data = cssc;
+  cssc->score = weight->value;
+  cssc->bv = filt_get_bv(filter, ir);
+
+  self->score = &cssc_score;
+  self->next = &cssc_next;
+  self->skip_to = &cssc_skip_to;
+  self->explain = &cssc_explain;
+  self->destroy = &scorer_destroy;
+  return self;
+}

data/ext/q_fuzzy.c

@@ -0,0 +1,287 @@
+#include <string.h>
+#include "search.h"
+
+/****************************************************************************
+ *
+ * FuzzyStuff
+ *
+ * The main method here is the fuzq_score method which scores a term against
+ * another term. The other methods all act in support.
+ *
+ ****************************************************************************/
+
+
+int fuzq_calculate_max_distance(FuzzyQuery *fuzq, int m) 
+{
+  return (int)((1.0 - fuzq->min_sim) * (MIN(fuzq->text_len, m) + fuzq->pre_len));
+}
+
+void fuzq_initialize_max_distances(FuzzyQuery *fuzq)
+{
+  int i;
+  for (i = 0; i < TYPICAL_LONGEST_WORD; i++) {
+    fuzq->max_distances[i] = fuzq_calculate_max_distance(fuzq, i);
+  }
+}
+
+float fuzq_get_max_distance(FuzzyQuery *fuzq, int m)
+{
+  return (m < TYPICAL_LONGEST_WORD) ? fuzq->max_distances[m]
+                                    : fuzq_calculate_max_distance(fuzq, m);
+}
+
+float fuzq_score(FuzzyQuery *fuzq, char *target)
+{
+  int i, j;
+  int max_distance;
+  int m = strlen(target);
+  int n = fuzq->text_len;
+  int *d = fuzq->da;
+  char *text = fuzq->text;
+  if (n == 0)  {
+    /* we don't have anything to compare.  That means if we just add
+     * the letters for m we get the new word */
+    return fuzq->pre_len == 0 ? 0.0f : 1.0f - ((float) m / fuzq->pre_len);
+  }
+  if (m == 0) {
+    return fuzq->pre_len == 0 ? 0.0f : 1.0f - ((float) n / fuzq->pre_len);
+  }
+
+  max_distance = fuzq_get_max_distance(fuzq, m);
+
+  //printf("n%dm%dmd%ddiff%d<%s><%s>\n", n, m, max_distance, m-n, fuzq->text, target);
+  if (max_distance < ((m > n) ? (m-n) : (n-m))) { /* abs */
+    /* Just adding the characters of m to n or vice-versa results in too many
+     * edits for example "pre" length is 3 and "prefixes" length is 8. We can
+     * see that given this optimal circumstance, the edit distance cannot be
+     * less than 5 which is 8-3 or more precisesly Math.abs(3-8). If our
+     * maximum edit distance is 4, then we can discard this word without
+     * looking at it. */
+    return 0.0f;
+  }
+
+  /* Let's make sure we have enough room in our array to do the distance
+   * calculations. */
+  if (((m+1) * (n+1)) >= fuzq->da_capa) {
+    fuzq->da_capa = (m * (fuzq->text_len+1)) * 2;
+    REALLOC_N(fuzq->da, int, fuzq->da_capa);
+    d = fuzq->da;
+  }
+
+  /* init matrix d */
+  for (i = 0; i <= n; i++) d[i + m * 0] = i;
+  for (j = 0; j <= m; j++) d[0 + m * j] = j;
+  
+  /* start computing edit distance */
+  for (i = 1; i <= n; i++) {
+    int best_pos_ed_dist = m;
+    char s_i = text[i - 1];
+    for (j = 1; j <= m; j++) {
+      if (s_i != target[j-1]) {
+        d[i + m*j] = min3(d[i-1 + m*j], d[i + m*(j-1)], d[i-1 + m*(j-1)])+1;
+      } else {
+        d[i + m*j] = min3(d[i-1 + m*j]+1, d[i + m*(j-1)]+1, d[i-1 + m*(j-1)]);
+      }
+      best_pos_ed_dist = min(best_pos_ed_dist, d[i + m*j]);
+    }
+    //printf("(bped = %d, i = %d, md = %d)", best_pos_ed_dist, i, max_distance);
+
+    /* After calculating row i, the best possible edit distance can be found
+     * by found by finding the smallest value in a given column.  If the
+     * best_pos_ed_dist is greater than the max distance, abort.
+     */
+
+    if ((i > max_distance) && (best_pos_ed_dist > max_distance)) {
+      /* equal is okay, but not greater
+       * the closest the target can be to the text is just too far away.
+       * this target is leaving the party early. */
+      return 0.0f;
+    }
+  }
+  //printf("<%f, d[n + m*m] = %d min_len = %d>", 1.0f - ((float)d[n + m*m] / (float) (fuzq->pre_len + min(n, m))), d[n + m*m], fuzq->pre_len + min(n, m));
+
+  /* this will return less than 0.0 when the edit distance is greater than the
+   * number of characters in the shorter word.  but this was the formula that
+   * was previously used in FuzzyTermEnum, so it has not been changed (even
+   * though min_sim must be greater than 0.0) */
+  return 1.0f - ((float)d[n + m*m] / (float) (fuzq->pre_len + min(n, m)));
+}
+
+/****************************************************************************
+ *
+ * FuzzyQuery
+ *
+ ****************************************************************************/
+
+char *fuzq_to_s(Query *self, char *field) 
+{
+  char *buffer, *bptr;
+  FuzzyQuery *fuzq = (FuzzyQuery *)self->data;
+  Term *term = fuzq->term;
+  int tlen = strlen(term->text);
+  int flen = strlen(term->field);
+  bptr = buffer = ALLOC_N(char, tlen + flen + 35);
+
+  if (strcmp(term->field, field) != 0) {
+    sprintf(bptr, "%s:", term->field);
+    bptr += strlen(term->field) + 1;
+  }
+  sprintf(bptr, "%s~", term->text);
+  bptr += strlen(bptr);
+  if (fuzq->min_sim != 0.5) {
+    dbl_to_s(bptr, fuzq->min_sim);
+    bptr += strlen(bptr);
+  }
+  if (self->boost != 1.0) {
+    *bptr = '^';
+    dbl_to_s(++bptr, self->boost);
+  }
+  return buffer;
+}
+
+typedef struct ScoredTerm {
+  Term *term;
+  float score;
+} ScoredTerm;
+
+bool scored_term_less_than(void *p1, void *p2)
+{
+  ScoredTerm *st1 = (ScoredTerm *)p1;
+  ScoredTerm *st2 = (ScoredTerm *)p2;
+
+  if (st1->score == st2->score)
+    return (strcmp(st1->term->text, st2->term->text) < 0);
+
+  return (st1->score < st2->score);
+}
+
+void scored_term_destroy(void *p)
+{
+  ScoredTerm *st = (ScoredTerm *)p;
+  term_destroy(st->term);
+  free(st);
+}
+
+ScoredTerm *scored_term_create(Term *term, float score)
+{
+  ScoredTerm *self = ALLOC(ScoredTerm);
+  self->term = term;
+  self->score = score;
+  return self;
+}
+
+Query *fuzq_rewrite(Query *self, IndexReader *ir)
+{
+  Query *q;
+  Query *tq;
+  FuzzyQuery *fuzq = (FuzzyQuery *)self->data;
+
+  Term *term = fuzq->term;
+  char *text = term->text;
+  char *field = term->field;
+  Term prefix_term;
+  prefix_term.field = field;
+  if (fuzq->pre_len >= strlen(text)) {
+    q = tq_create(term_clone(term));
+  } else {
+    PriorityQueue *term_pq;
+    TermEnum *te;
+    Term prefix_term;
+    char *prefix = NULL;
+    int pre_len = fuzq->pre_len;
+    ScoredTerm *scored_term;
+
+    q = bq_create(true);
+
+    term_pq = pq_create(((BooleanQuery *)q->data)->max_clause_cnt,
+        &scored_term_less_than);
+    term_pq->free_elem = &scored_term_destroy;
+
+    prefix_term.field = field;
+    prefix_term.text = (char *)EMPTY_STRING;
+    if (pre_len >= 0) {
+      prefix = ALLOC_N(char, pre_len + 1);
+      strncpy(prefix, text, pre_len);
+      prefix_term.text = prefix;
+      prefix_term.text[pre_len] = '\0';
+    }
+    te = ir->terms_from(ir, &prefix_term);
+
+    fuzq->scale_factor = 1.0 / (1.0 - fuzq->min_sim);
+    fuzq->text = fuzq->term->text + pre_len;
+    fuzq->text_len = strlen(fuzq->text);
+    fuzq_initialize_max_distances(fuzq);
+
+    if (te) {
+      TermBuffer *tb = te->tb_curr;
+      float score = 0.0, min_score = fuzq->min_sim;
+
+      do { 
+        if (strcmp(tb->field, field) != 0 ||
+            (prefix && strncmp(tb->text, prefix, pre_len) != 0))
+          break;
+
+        score = fuzq_score(fuzq, tb->text + pre_len);
+        //printf("%s:%s:%f\n", tb->text, fuzq->text, score);
+
+        if (score > min_score) {
+          pq_insert(term_pq, scored_term_create(tb_get_term(tb), score));
+          if (pq_full(term_pq))
+            min_score = ((ScoredTerm *)pq_top(term_pq))->score;
+        }
+      } while ((tb = te->next(te)) != NULL);
+      te->close(te);
+    }
+    free(prefix);
+
+    while ((scored_term = pq_pop(term_pq)) != NULL) {
+      tq = tq_create(scored_term->term); /* found match */
+      tq->boost = self->boost;           /* set boost */
+      bq_add_query(q, tq, BC_SHOULD);    /* add query */
+      free(scored_term); /* no need to free the term as it's in the query */
+    }
+    pq_destroy(term_pq);
+  }
+
+  if (self->rewritten) self->rewritten->destroy(self->rewritten);
+  return self->rewritten = q;
+}
+
+void fuzq_destroy(void *p)
+{
+  Query *self = (Query *)p;
+  FuzzyQuery *fuzq = (FuzzyQuery *)self->data;
+  if (self->destroy_all) term_destroy((Term *)fuzq->term);
+  free(fuzq->da);
+  free(fuzq);
+  q_destroy(self);
+}
+
+Query *fuzq_create(Term *term)
+{
+  Query *self = q_create();
+  FuzzyQuery *fq = ALLOC(FuzzyQuery);
+  ZEROSET(fq, FuzzyQuery, 1);
+
+  fq->term = term;
+  fq->pre_len = DEF_PRE_LEN;
+  fq->min_sim = DEF_MIN_SIM;
+  self->data = fq;
+  self->type = FUZZY_QUERY;
+  self->create_weight = NULL;
+  self->to_s = &fuzq_to_s;
+  self->rewrite = &fuzq_rewrite;
+  self->destroy = &fuzq_destroy;
+  self->rewritten = NULL;
+
+  return self;
+}
+
+Query *fuzq_create_mp(Term *term, float min_sim, int pre_len)
+{
+  Query *self = fuzq_create(term);
+  FuzzyQuery *fuzq = (FuzzyQuery *)self->data;
+  if (pre_len) fuzq->pre_len = pre_len;
+  if (min_sim) fuzq->min_sim = min_sim;
+  return self;
+}

data/ext/q_match_all.c

@@ -0,0 +1,142 @@
+#include "search.h"
+#include <string.h>
+
+/***************************************************************************
+ *
+ * Weight
+ *
+ ***************************************************************************/
+
+char *maw_to_s(Weight *self)
+{
+  char dbuf[32];
+  dbl_to_s(dbuf, self->value);
+  return epstrdup("MatchAllWeight(%s)", strlen(dbuf), dbuf);
+}
+
+void maw_destroy(void *p)
+{
+  free(p);
+}
+
+Explanation *maw_explain(Weight *self, IndexReader *ir, int doc_num)
+{
+  Explanation *expl;
+  if (!ir->is_deleted(ir, doc_num)) {
+    expl = expl_create(self->value, estrdup("MatchAllQuery: product of:"));
+    expl_add_detail(expl, expl_create(self->query->boost, estrdup("boost")));
+    expl_add_detail(expl, expl_create(self->qnorm, estrdup("query_norm")));
+  } else {
+    expl = expl_create(self->value,
+        epstrdup("MatchAllQuery: doc %d was deleted", 20, doc_num));
+  }
+
+  return expl;
+}
+
+Weight *maw_create(Query *query, Searcher *searcher)
+{
+  Weight *self = ALLOC(Weight);
+  ZEROSET(self, Weight, 1);
+  self->get_query = &w_get_query;
+  self->get_value = &w_get_value;
+  self->normalize = &w_normalize;
+  self->scorer    = &masc_create;
+  self->explain   = &maw_explain;
+  self->to_s      = &maw_to_s;
+  self->destroy   = &maw_destroy;
+  self->sum_of_squared_weights = &w_sum_of_squared_weights;
+
+  self->similarity = query->get_similarity(query, searcher);
+  self->idf = 1.0;
+  self->query = query;
+  self->value = 0.0;
+
+  return self;
+}
+
+/***************************************************************************
+ *
+ * MatchAllQuery
+ *
+ ***************************************************************************/
+
+char *maq_to_s(Query *self, char *field)
+{
+  if (self->boost == 1.0) {
+    return estrdup("MatchAll");
+  } else {
+    char dbuf[32];
+    dbl_to_s(dbuf, self->boost);
+    return epstrdup("MatchAll^%s", strlen(dbuf), dbuf);
+  }
+}
+
+void maq_destroy(void *p)
+{
+  Query *self = (Query *)p;
+  q_destroy(self);
+}
+
+Query *maq_create()
+{
+  Query *self = q_create();
+  self->type = MATCH_ALL_QUERY;
+  self->create_weight = &maw_create;
+  self->to_s = &maq_to_s;
+  self->destroy = &maq_destroy;
+
+  return self;
+}
+
+/***************************************************************************
+ *
+ * MatchAllScorer
+ *
+ ***************************************************************************/
+
+float masc_score(Scorer *self)
+{
+  return ((MatchAllScorer *)self->data)->score;
+}
+
+bool masc_next(Scorer *self)
+{
+  MatchAllScorer *mas = (MatchAllScorer *)self->data;
+  while (self->doc < (mas->max_doc - 1)) {
+    self->doc++;
+    if (!mas->ir->is_deleted(mas->ir, self->doc)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool masc_skip_to(Scorer *self, int doc_num)
+{
+  self->doc = doc_num - 1;
+  return masc_next(self);
+}
+
+Explanation *masc_explain(Scorer *self, int doc_num)
+{
+  return expl_create(1.0, estrdup("MatchAllScorer"));
+}
+
+Scorer *masc_create(Weight *weight, IndexReader *ir)
+{
+  Scorer *self = scorer_create(weight->similarity);
+  MatchAllScorer *mas = ALLOC(MatchAllScorer);
+  mas->ir = ir;
+  mas->max_doc = ir->max_doc(ir);
+  mas->score = weight->value;
+  self->data = mas;
+
+  self->doc = -1;
+  self->score = &masc_score;
+  self->next = &masc_next;
+  self->skip_to = &masc_skip_to;
+  self->explain = &masc_explain;
+  self->destroy = &scorer_destroy;
+  return self;
+}